o
    }oi7                     @   s   d dl Z d dlZd dlmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZmZmZmZ d dlmZ dgZG dd dee	ZdS )	    N)DictOptional)NeuralModule)
Exportable)ChannelType	FloatTypeIntTypeMaskType
NeuralType
StringTypeVoidType)logging	GPTModulec                   @   s^   e Zd Zedeeeef  fddZedeeeef  fddZ	defddZ
d	d
 ZdS )r   returnc                 C   s   t dt t dt ddt dt ddt dt ddt d t ddggt d t ddt dt ddt d t ddt d t ddt d t ddt d t dddS )N)BTToptional)	input_idstoken_type_idsattention_masklabelspast_key_values	use_cacheposition_idsreturn_dictoutput_attentionsoutput_hidden_states
max_length)r
   r   r	   r   r   r   self r!   b/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/nlp/modules/common/gpt_module.pyinput_types   s   
zGPTModule.input_typesc                 C   s   t d t ddt dt dS )NTr   )r   r   D)losshidden_states)r
   r   r   r   r!   r!   r"   output_types.   s   
zGPTModule.output_typesrestore_pathc                    s   t d|  tj|st d| d dS t|}d| v r(|d }t	
dt| d }|rF|dfdd	| D }|    fd
d	| D }d v rcd|vrc d |d<  | |   t dt| j d|  dS )zRestores module/model's weightszRestoring weights from zPath z
 not foundN
state_dictz	^gpt.*?\.r   c                    s"   i | ]\}}|t  d  |qS )N)len.0kv)prefixr!   r"   
<dictcomp>G   s   " z-GPTModule.restore_weights.<locals>.<dictcomp>c                    s   i | ]\}}| v r||qS r!   r!   r+   )
model_dictr!   r"   r0   I   s    zembeddings.position_idszWeights for z restored from )r   infoospathexistswarningtorchloadkeysrematchlistgroupitemsr)   updateload_state_dicttype__name__)r    r(   pretrained_dictmr!   )r1   r/   r"   restore_weights5   s$   



 zGPTModule.restore_weightsc                 C   sV   t |  }tjddd|jd}tjddd|jd}tjddd|jd}t|||gS )zs
        Generates input examples for tracing etc.
        Returns:
            A tuple of input examples.
        r   i   )      )lowhighsizedevice   )next
parametersr7   randintrK   tuple)r    sampler   r   r   r!   r!   r"   input_exampleT   s
   zGPTModule.input_exampleN)rB   
__module____qualname__propertyr   r   strr
   r#   r'   rE   rR   r!   r!   r!   r"   r      s    )r3   r:   typingr   r   r7   nemo.core.classesr   nemo.core.classes.exportabler   nemo.core.neural_typesr   r   r   r	   r
   r   r   
nemo.utilsr   __all__r   r!   r!   r!   r"   <module>   s   $