o
    .i                     @   s~   d dl Z d dlmZ d dlmZ d dlZd dlZddl	m
Z
 ddlmZ ddlmZ er0d dlZG d	d
 d
eedef ZdS )    N)Mapping)TYPE_CHECKING   )config)
map_nested   )TensorFormatterc                       s   e Zd Zd fdd	Zdd Zdd Zdd	 Zd
efddZde	j
defddZde	j
ddfddZde	j
defddZ  ZS )TorchFormatterNc                    s"   t  j||d || _dd l}d S )N)featurestoken_per_repo_idr   )super__init__torch_tensor_kwargstorch)selfr
   r   r   r   	__class__ W/home/ubuntu/.local/lib/python3.10/site-packages/datasets/formatting/torch_formatter.pyr   !   s   zTorchFormatter.__init__c                    s<   dd l t tr rt fdd D r S  S )Nr   c                 3   s<    | ]}t |jo|j d  jko|j d  jkV  qdS )r   N)
isinstanceTensorshapedtype).0xcolumnr   r   r   	<genexpr>*   s
    *
z.TorchFormatter._consolidate.<locals>.<genexpr>)r   r   listallstack)r   r   r   r   r   _consolidate&   s   
zTorchFormatter._consolidatec                 C   sh  dd l }t|tttd fr|S t|tjtjfr%t|j	tjr%|
 S i }t|tjtjfrMt|j	tjrMd|ji}|j	tjtjfv rL|tj}nt|tjtjfrct|j	tjrcd|ji}tjrdtjv rdd l}t||jjrt|}|jdkr|d d d d tjf }|d}tjrdtjv rddlm} t||r|S |j |fi i || j!S )Nr   r   PILr   )r   r   r   torchvision)VideoReader)"r   r   strbytestypenp	characterndarray
issubdtyper   tolistnumberintegerint64uint16uint32astypefloatingfloat32r   PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayndimnewaxis	transposeTORCHVISION_AVAILABLEtorchvision.ior$   tensorr   )r   valuer   default_dtyper"   r$   r   r   r   
_tensorize1   s2   ""
"




zTorchFormatter._tensorizec                    s   dd l }t|drt||js| }t|tjr+|jtkr* 	 fdd|D S nt|t
tfr> 	 fdd|D S  |S )Nr   	__array__c                       g | ]}  |qS r   recursive_tensorizer   	substructr   r   r   
<listcomp>`       z7TorchFormatter._recursive_tensorize.<locals>.<listcomp>c                    rE   r   rF   rH   rJ   r   r   rK   b   rL   )r   hasattrr   r   rD   r(   r*   r   objectr!   r   tuplerC   )r   data_structr   r   rJ   r   _recursive_tensorizeW   s   

z#TorchFormatter._recursive_tensorizerP   c                 C   s   t | j|ddS )NF)map_list)r   rQ   )r   rP   r   r   r   rG   e   s   z"TorchFormatter.recursive_tensorizepa_tablereturnc                 C   s$   |   |}| j|}| |S N)numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowrG   )r   rS   rowr   r   r   
format_rowh   s   
zTorchFormatter.format_rowtorch.Tensorc                 C   s:   |   |}| j||jd }| |}| |}|S )Nr   )rV   extract_columnrX   decode_columncolumn_namesrG   r!   )r   rS   r   r   r   r   format_columnm   s
   

zTorchFormatter.format_columnc                 C   sD   |   |}| j|}| |}|D ]}| || ||< q|S rU   )rV   extract_batchrX   decode_batchrG   r!   )r   rS   batchcolumn_namer   r   r   format_batcht   s   
zTorchFormatter.format_batch)NN)__name__
__module____qualname__r   r!   rC   rQ   dictrG   paTabler   r[   r`   re   __classcell__r   r   r   r   r	       s    &r	   r\   )r6   collections.abcr   typingr   numpyr(   pyarrowrj    r   utils.py_utilsr   
formattingr   r   r	   r   r   r   r   <module>   s   