o ߥiB@s<ddlZddlmZddlmZeZGdddeZdS)N)PretrainedConfig)loggingc$sxeZdZdZdddddddddd d d d d d d d d d dd dd d dddddddgd dd d df$fdd ZeddZZS) GPTMoEConfigzgpt-moeidiN i gelug?ig-q=TFg{Gz?drg?standardc%( stjdd| i|%||_||_|durd|n||_||_||_||_||_||_ | |_ | |_ | |_ | |_ | |_||_||_||_||_|rL|rLJ||_||_|durc||dks^J|||_||_||_||_||_||_||_||_||_||_||_||_ | |_!|!|_"|"|_#|#|_$|$|_%|j dt&j'(krt&j'(|_)n|j d|_)t*t&j+,dd}&t*t&j+,dd}'|&dkp|&dko|'dk|_-dS)Nlayer_norm_epsr. ).super__init__ vocab_size hidden_sizeffn_hidden_sizenum_hidden_layersnum_attention_heads hidden_actintermediate_sizehidden_dropout_probattention_probs_dropout_probmax_position_embeddingstype_vocab_sizelayernorm_epsilonbias_gelu_fusionfp32_residual_connectionsequence_parallelfp16bf16apply_query_key_layer_scalingattention_softmax_in_fp32 kv_channelsmasked_softmax_fusionattention_dropoutbias_dropout_fusion(apply_residual_connection_post_layernormhidden_dropoutinit_method_stdeod_idtokens_to_generatetop_ktop_p num_experts use_tuteltop_k_linear_strategyuse_expert_residual_network load_ds_ckpts model_dirtorchcuda device_countmoe_expert_parallel_sizeint __version__splitno_persist_layer_norm)(selfrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6kwargs TORCH_MAJOR TORCH_MINOR __class__r_/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/gpt_moe/configuration.pyrsf(   zGPTMoEConfig.__init__cCs|jrtjS|jr tjStjS)N)r"r7halfr#bfloat16float)r?rrrE params_dtypeys zGPTMoEConfig.params_dtype)__name__ __module__ __qualname__ model_typerpropertyrI __classcell__rrrCrErsR^r)r7 transformers.configuration_utilsrtransformers.utilsr get_loggerloggerrrrrrEs