o
    Tio                     @   s6   d dl Z d dlZd dlmZmZ dddZdd ZdS )	    N)DeepSpeedTransformerLayerDeepSpeedTransformerConfigTc                 C   s  |  D ]\}}	t|	| rtd t|||j|j|j|j|j|j	|||d}
t
|
}|	jjjj}|	jjjj}|	jjjj}|	jjjj}|	jjjj}|	jjjj}t|||fd}t|||fd}||j_||j_|	jjjj|j_|	jjjj|j_|r~|	j}n|	jjj}|j|j_|j|j_|r|	jj }n|	jj}|j|j!_|j|j"_|	jjj|j#_|	jjj|j$_|r|	j%}n|	jj}|j|j&_|j|j'_t(||t)*| qt+| |	|||||| q|S )NzREPLACING BertLayer)
batch_sizemax_seq_lengthhidden_sizeheadsattn_dropout_ratiohidden_dropout_rationum_hidden_layersinitializer_rangeseedfp16pre_layer_normr   ),named_children
isinstanceprintr   r   num_attention_headsattention_probs_dropout_probhidden_dropout_probr
   r   r   	attentionselfqueryweightbiaskeyvaluetorchcat	attn_qkvwdata	attn_qkvboutputdenseattn_owattn_obPostAttentionLayerNorm	LayerNormattn_nwattn_nbintermediate	dense_actinter_winter_boutput_woutput_bPreAttentionLayerNormnorm_wnorm_bsetattrcopydeepcopymodule_inject)	layer_objmodelconfigmicro_batch_sizer   r   prelnr   namechildcuda_config
new_moduleqwqbkwkbvwvbqkvwqkvbattention_layerNormintermediate_FFtransformer_LayerNorm rJ   R/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/module_inject/inject.pyr5      s`   








r5   c                  C   s   ddl m}  ddl m} ddl m} ddddd	d
dddddddd}| di |}||d d}t|}t|||ddd}td| td| d S )Nr   )
BertConfig)BertForQuestionAnswering)	BertLayeri i         i   gelug?i      g{Gz?)vocab_size_or_config_json_filer   r
   r   intermediate_size
hidden_actr   r   r   r   max_position_embeddingstype_vocab_sizer   )args   i  i  BASETESTrJ   )turing.nvidia_modelingprelnrL   rM   rN   r3   r4   r5   r   )BertConfigPreLNBertForQuestionAnsweringPreLNrN   bert_model_configbert_config
base_model
test_modelrJ   rJ   rK   test_hiJ   s.   

rc   )T)r3   r   deepspeed.ops.transformerr   r   r5   rc   rJ   rJ   rJ   rK   <module>   s
   
?