o
    $i                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ e Ze Zg dZd	e	fd
dZd	e	deeef fddZdS )    N)Dict)TagKeyrecord_extra_usage_tag)LogicalOperator)AbstractUDFMap)Read)Write)-ReadBigQuery	ReadRange	ReadMongoReadParquetReadParquetBulk	ReadImageReadJSONReadCSVReadText	ReadNumpyReadTFRecord
ReadBinary	ReadTorchReadAvroReadWebDatasetReadSQLReadDatabricksUC	ReadLanceReadHuggingFace
ReadCustom	FromArrow	FromItems	FromNumpy
FromPandasWriteBigQueryWriteParquet	WriteJSONWriteCSVWriteTFRecord
WriteNumpy
WriteMongoWriteWebDatasetWriteSQLWriteCustomMap
MapBatchesFilterFlatMapRandomizeBlockOrderRandomShuffleRepartitionSort	AggregateZipUnionopc                 C   s   t  }t| | d}t$ | D ]\} }t| d t|   |7  < qtt}W d   n1 s3w   Y  tt	j
| dS )z1Record logical operator usage with Ray telemetry. r   N)dict_collect_operators_to_dict_recorded_operators_lockitems_recorded_operators
setdefaultjsondumpsr   r   DATA_LOGICAL_OPS)r6   ops_dictops_json_strcount rD   \/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/data/_internal/logical/util.pyrecord_operators_usageH   s   
rF   rA   c                 C   s   | j D ]}t|| q| j}t| tr"d| j  }|tvr!d}n t| tr6d| j	  }|tvr5d}nt| t
rBtdd|}|tvrHd}||d ||  d	7  < d
S )z<Collect the logical operator name and count into `ops_dict`.r   r   r   r*   z\(.*\)$r7   Unknownr      N)input_dependenciesr9   name
isinstancer   _datasourceget_name_op_name_white_listr   _datasink_or_legacy_datasourcer   resubr=   )r6   rA   childop_namerD   rD   rE   r9   V   s&   



r9   )r>   rP   	threadingtypingr   ray._common.usage.usage_libr   r   %ray.data._internal.logical.interfacesr   1ray.data._internal.logical.operators.map_operatorr   2ray.data._internal.logical.operators.read_operatorr   3ray.data._internal.logical.operators.write_operatorr   r8   r<   Lockr:   rN   rF   strintr9   rD   rD   rD   rE   <module>   s    7