o
    ॵiI                     @   sT   d dl mZ d dlmZ d dlmZmZmZmZ ddl	m
Z
 e
 Z	G dd dZdS )	    )OrderedDict)Mapping)AnyListOptionalTuple   )
get_loggerc                   @   sd   e Zd ZdZ		ddee dee dee fddZded	e	ee
f fd
dZdee fddZdS )RemoveColumnsCollatora  Remove specified columns from the input mini-batch, and convert them to attributes.

    For example: if columns_to_remove = ['id'], then user should call batch.id instead of batch['id'].

    Args:
        data_collator: An inner data collator to collate the mini-batch
        columns_to_remove(`List[str]`): The redundant columns to be removed from the mini-batch
        model_name(`Optional[str]`): An optional model name to print into log
        description(`Optional[str]`): An optional description to print into log
    Ncolumns_to_remove
model_namedescriptionc                 C   s"   || _ || _|| _|| _d| _d S )NF)data_collatorr   r   r   message_logged)selfr   r   r   r    r   S/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/utils/data_collators.py__init__   s
   
zRemoveColumnsCollator.__init__featurereturnc                    s   t |ts	|d fS  jsW jrWtt| t j }t|dkrW j	d u r)dnd j	 d}t
d| d j dd| d	d j d
d| d j d d _ fdd| D } fdd| D }||fS )Nr    zin the z setzThe following columns z) don't have a corresponding argument in `z!.forward` and have been ignored: z, z.Legal columns: z. If z are not expected by `z/.forward`,  you can safely ignore this message.Tc                    s    i | ]\}}| j v r||qS r   r   .0kvr   r   r   
<dictcomp>5       z9RemoveColumnsCollator._remove_columns.<locals>.<dictcomp>c                    s    i | ]\}}| j vr||qS r   r   r   r   r   r   r   9   r   )
isinstancer   r   r   listsetkeysr   lenr   loggerinfojoinitems)r   r   ignored_columnsdset_descriptionfeature_cleanfeature_unusedr   r   r   _remove_columns&   s:   



z%RemoveColumnsCollator._remove_columnsfeaturesc                    s~   g }g }|D ]}|  |\}}|| || qt| |}|d d ur=|d  D ] t|  fdd|D  q-|S )Nr   c                    s   g | ]}|  qS r   r   )r   r+   keyr   r   
<listcomp>I   s    z2RemoveColumnsCollator.__call__.<locals>.<listcomp>)r,   appendr   r   r"   setattr)r   r-   features_cleanfeatures_unusedr   r+   datar   r.   r   __call__?   s   

zRemoveColumnsCollator.__call__)NN)__name__
__module____qualname____doc__r   strr   r   r   r   r   r,   r6   r   r   r   r   r
      s    
r
   N)collectionsr   collections.abcr   typingr   r   r   r   r$   r	   r
   r   r   r   r   <module>   s   