o
    .i_                     @   s~   d dl Z d dlmZ d dlmZ d dlZd dlZddl	m
Z
 ddlmZ ddlmZ er0d dlZG d	d
 d
eedef ZdS )    N)Mapping)TYPE_CHECKING   )config)
map_nested   )TensorFormatterc                       s   e Zd Zd fdd	Zdd Zdd Zdd	 Zd
efddZde	j
defddZde	j
ddfddZde	j
defddZ  ZS )TFFormatterNc                    s"   t  j||d || _dd l}d S )N)featurestoken_per_repo_idr   )super__init__tf_tensor_kwargs
tensorflow)selfr
   r   r   tf	__class__ T/home/ubuntu/.local/lib/python3.10/site-packages/datasets/formatting/tf_formatter.pyr   !   s   zTFFormatter.__init__c                    s`   dd l t tr. r.t fdd D r S t fdd D r.j S  S )Nr   c                 3   s<    | ]}t |jo|j d  jko|j d  jkV  qdS )r   N)
isinstanceTensorshapedtype.0xcolumnr   r   r   	<genexpr>*   s    ,
z+TFFormatter._consolidate.<locals>.<genexpr>c                 3   s<    | ]}t |jjfo|jd ko|j d jkV  qdS )r   r   N)r   r   RaggedTensorndimr   r   r   r   r   r   .   s
    *
)r   r   listallstackragged)r   r   r   r   r   _consolidate&   s   
zTFFormatter._consolidatec                 C   s   dd l }|d u r
|S i }t|tjtjfr#t|jtjr#d|ji}nt|tjtjfr9t|jtj	r9d|j
i}tjrQdtjv rQdd l}t||jjrQt|}tjrfdtjv rfddlm} t||rf|S |j|fi i || jS )Nr   r   PILtorchvision)VideoReader)r   r   npnumberndarray
issubdtyper   integerint64floatingfloat32r   PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayTORCHVISION_AVAILABLEtorchvision.ior)   convert_to_tensorr   )r   valuer   default_dtyper'   r)   r   r   r   
_tensorize7   s"   ""


zTFFormatter._tensorizec                    s   dd l }tjr#dtjv r#dd l}t||jr# |	 
  d S t|dr2t||js2| }t|tjrJ|jtkrI  fdd|D S nt|ttfr]  fdd|D S  |S )Nr   torchr   	__array__c                       g | ]}  |qS r   recursive_tensorizer   	substructr   r   r   
<listcomp>_       z4TFFormatter._recursive_tensorize.<locals>.<listcomp>c                    r@   r   rA   rC   rE   r   r   rF   a   rG   )r   r   TORCH_AVAILABLEr3   r4   r>   r   r   r=   detachcpunumpyhasattrr?   r*   r,   r   objectr&   r"   tuple)r   data_structr   r>   r   rE   r   _recursive_tensorizeQ   s   

z TFFormatter._recursive_tensorizerO   c                 C   s   t | j|ddS )NF)map_list)r   rP   )r   rO   r   r   r   rB   d   s   zTFFormatter.recursive_tensorizepa_tablereturnc                 C   s$   |   |}| j|}| |S N)numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowrB   )r   rR   rowr   r   r   
format_rowg   s   
zTFFormatter.format_row	tf.Tensorc                 C   s:   |   |}| j||jd }| |}| |}|S )Nr   )rU   extract_columnrW   decode_columncolumn_namesrB   r&   )r   rR   r   r   r   r   format_columnl   s
   

zTFFormatter.format_columnc                 C   sD   |   |}| j|}| |}|D ]}| || ||< q|S rT   )rU   extract_batchrW   decode_batchrB   r&   )r   rR   batchcolumn_namer   r   r   format_batchs   s   
zTFFormatter.format_batch)NN)__name__
__module____qualname__r   r&   r=   rP   dictrB   paTabler   rZ   r_   rd   __classcell__r   r   r   r   r	       s    r	   r[   )r3   collections.abcr   typingr   rK   r*   pyarrowri    r   utils.py_utilsr   
formattingr   r   r   r	   r   r   r   r   <module>   s   