o
    ß¥µi€	  ã                   @   s6   d dl mZ d dlmZ eƒ Zi ZG dd„ deƒZdS )é    )ÚPretrainedConfig)Ú
get_loggerc                       sd   e Zd ZdZdgZdddddœZ				
	
																			d‡ fdd„	Z‡  ZS )Ú
QWenConfigÚqwenÚpast_key_valuesÚn_embdÚn_headÚn_positionsÚn_layer)Úhidden_sizeÚnum_attention_headsÚmax_position_embeddingsÚnum_hidden_layersé+Q é   é    Nç        çñhãˆµøä>ç{®Gáz”?Té[P Fé€   ç      ð?é'  é V  c                    s¦   || _ tƒ jd||dœ|¤Ž || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _d S )N)Úeos_token_idÚtie_word_embeddings© )r   ÚsuperÚ__init__Ú
vocab_sizer   r
   r   Ún_innerÚ
embd_pdropÚ
attn_pdropÚlayer_norm_epsilonÚinitializer_rangeÚscale_attn_weightsÚ	use_cacheÚ(apply_residual_connection_post_layernormÚbf16Úkv_channelsÚ
rotary_pctÚrotary_emb_baseÚuse_dynamic_ntkÚuse_logn_attnÚuse_flash_attnÚffn_hidden_sizeÚno_biasr   )Úselfr   r   r
   r   r    r!   r"   r#   r$   r%   r&   r   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r   Úkwargs©Ú	__class__r   ú\/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/qwen/configuration.pyr      s:   þý
zQWenConfig.__init__)r   r   r   r   Nr   r   r   r   TTr   FTr   r   r   FFTr   TF)Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr   Ú__classcell__r   r   r3   r5   r      s@    ü	èr   N)Útransformersr   Úmodelscope.utils.loggerr   ÚloggerÚ"QWEN_PRETRAINED_CONFIG_ARCHIVE_MAPr   r   r   r   r5   Ú<module>   s
   