o
    Ti                     @   sD   d dl T ddlmZ ddlmZ G dd deZG dd	 d	eZd
S )   )*    )DeepSpeedGPTInference   )TransformerPolicyc                       s&   e Zd Z fddZdddZ  ZS )DS_GPT2Containerc                    s   t  jdi | d S )N )super__init__)selfkwargs	__class__r   [/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/module_inject/containers/gpt2.pyr
      s   zDS_GPT2Container.__init__Nc                 C   s4   |d ur|n| j }t|| jd| _| j| jj_| jS )N)mp_group)ds_model_configr   r   modulescale_attentionconfig)r   r   _configr   r   r   create_module   s   zDS_GPT2Container.create_moduleN)__name__
__module____qualname__r
   r   __classcell__r   r   r   r   r      s    r   c                       sF   e Zd ZdZd fdd	Zdd Zddd	Zdd
dZdd Z  Z	S )HFGPT2LayerPolicyNTc                    sF   t  j|dd || _zdd l}|jjjjt_	W d S    d t_	Y d S )NF)linear_layerr   )
r	   r
   client_moduletransformersmodelsgpt2modeling_gpt2	GPT2Blockr   _orig_layer_class)r   r   	inferencer   r   r   r   r
      s   zHFGPT2LayerPolicy.__init__c                 C   s   | j jj| j jj| j jjtfS r   )r   attn	embed_dim	num_headsln_1epsDEFAULT_INTERMEDIATE_SIZEr   r   r   r   get_hidden_heads&   s
   z"HFGPT2LayerPolicy.get_hidden_headsFc                 C   ,   | j jjj| j jjj| j jjj| j jjjfS r   )r   r&   c_attnweightbiasc_projr   enable_trainingr   r   r   	attention,   
   



zHFGPT2LayerPolicy.attentionc                 C   r.   r   )r   mlpc_fcr0   r1   r2   r3   r   r   r   r7   2   r6   zHFGPT2LayerPolicy.mlpc                 C   s$   | j jj| j jj| j jj| j jjfS r   )r   ln_2r0   r1   r)   r,   r   r   r   	layernorm8   s
   zHFGPT2LayerPolicy.layernorm)T)F)
r   r   r   r$   r
   r-   r5   r7   r:   r   r   r   r   r   r      s    


r   N)base3deepspeed.model_implementations.transformers.ds_gptr   policyr   BaseTransformerContainerr   r   r   r   r   r   <module>   s
   