o
    wi                     @   s   d dl mZmZ d dlmZmZmZmZ d dlZd dl	m
Z
 d dlmZmZ eG dd dZeG dd	 d	eZeG d
d deZeG dd deZeG dd dZeG dd deZeG dd dZeG dd deZeG dd dZdS )    )	dataclassfield)ListOptionalTupleUnionNPackedSeqParams)BaseConversationTemplateConfigLLaVATemplateConfigc                   @   s&   e Zd ZU eed< eed< eed< dS )MultiModalToken	token_strtoken_id
media_typeN)__name__
__module____qualname__str__annotations__int r   r   l/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/nemo/collections/multimodal/data/energon/config.pyr      s   
 r   c                   @   2   e Zd ZU dZeed< dZeed< dZeed< dS )
ImageTokenz<image>r   i8r   imager   N	r   r   r   r   r   r   r   r   r   r   r   r   r   r          
 r   c                   @   r   )
AudioTokenz<audio>r   ir   audior   Nr   r   r   r   r   r   '   r   r   c                   @   r   )
VideoTokenz<video>r   ipr   videor   Nr   r   r   r   r   r   .   r   r   c                   @   s   e Zd ZU dZdZeed< edd dZe	j
ed< edd dZe	j
ed	< ed
d dZe	j
ed< edd dZe	j
ed< dZeee  ed< dS )ImageTextSamplez8Sample type for template formatted raw image text sample __key__c                   C   
   t dS Nr   torchemptyr   r   r   r   <lambda>:      
 zImageTextSample.<lambda>default_factoryimagesc                   C      t jdt jdS Nr   )dtyper'   r(   longr   r   r   r   r)   ;       tokensc                   C   r.   r/   r1   r   r   r   r   r)   <   r3   labelsc                   C   r.   r/   r'   r(   floatr   r   r   r   r)   =   r3   	loss_maskNnum_image_tiles)r   r   r   __doc__r#   r   r   r   r-   r'   Tensorr4   r5   r8   r9   r   r   r   r   r   r   r   r!   5   s   
 r!   c                   @   s^   e Zd ZU dZdZeeeee	f df e
d< edd dZeje
d< ed	d dZee
d
< dS )PackedImageTextSamplez(Sample type for packed image text sampler   .__restore_key__c                   C   r.   r/   r6   r   r   r   r   r)   F   r3   zPackedImageTextSample.<lambda>r+   position_idsc                   C      t  S Nr   r   r   r   r   r)   G       packed_seq_paramsN)r   r   r   r:   r=   r   r   r   r   tupler   r   r>   r'   r;   rB   r	   r   r   r   r   r<   A   s
   
 r<   c                   @   s   e Zd ZU dZeedZee e	d< edd dZ
eje	d< edd dZeje	d< ed	d dZeje	d
< edd dZeje	d< dZeee  e	d< dS )ImageTextRawBatch$Sample type for image text raw batchr+   __keys__c                   C   r$   r%   r&   r   r   r   r   r)   P   r*   zImageTextRawBatch.<lambda>r-   c                   C   r.   r/   r1   r   r   r   r   r)   R   r3   r4   c                   C   r.   r/   r1   r   r   r   r   r)   S   r3   r5   c                   C   r.   r/   r6   r   r   r   r   r)   T   r3   r8   Nr9   )r   r   r   r:   r   listrF   r   r   r   r-   r'   r;   r4   r5   r8   r9   r   r   r   r   r   r   rD   J   s   
 rD   c                   @   s@   e Zd ZU dZedd dZejed< edd dZ	e
ed< dS )	PackedImageTextRawBatchrE   c                   C   r.   r/   r6   r   r   r   r   r)   \   r3   z PackedImageTextRawBatch.<lambda>r+   r>   c                   C   r?   r@   r   r   r   r   r   r)   ]   rA   rB   N)r   r   r   r:   r   r>   r'   r;   r   rB   r	   r   r   r   r   rH   X   s   
 rH   c                   @   sJ   e Zd ZU eedZeed< dZeed< ee	dZ
eed< dZeed< dS )	MultiModalSampleConfigr+   image_tokeniignore_place_holderconversation_template_configTimage_following_textN)r   r   r   r   r   rJ   r   rK   r   r   rL   r
   rM   boolr   r   r   r   rI   `   s
   
 rI   )dataclassesr   r   typingr   r   r   r   r'   megatron.core.packed_seq_paramsr	   5nemo.collections.multimodal.data.energon.conversationr
   r   r   r   r   r   r!   r<   rD   rH   rI   r   r   r   r   <module>   s.   