o
    ˜à·iŒ  ã                   @   s&   d dl mZ G dd„ deƒZdgZdS )é    )ÚPretrainedConfigc                A       sø   e Zd ZdZ										
																										d:dedededededededed	B d ed!ed"ed#ed$ed%ed&ed'ed	B d(ed	B d)ed*ed+ed,ed-ed.ed/ed0ed1ed2ed3e	e d	B d4ed5ed6ed7ef@‡ fd8d9„Z
‡  ZS );ÚAfmoeConfigÚafmoeé  é   é   é€  é    é   é   Né€   Úsilué   ç{®Gáz”?çñhãˆµøä>TFé@   é   é   Úsigmoidç      ð?é   ç        Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmoe_intermediate_sizeÚnum_hidden_layersÚnum_dense_layersÚnum_attention_headsÚnum_key_value_headsÚhead_dimÚ
hidden_actÚmax_position_embeddingsÚinitializer_rangeÚrms_norm_epsÚ	use_cacheÚtie_word_embeddingsÚrope_parametersÚrope_scalingÚnum_expertsÚnum_experts_per_tokÚnum_shared_expertsÚnum_expert_groupsÚnum_limited_groupsÚ
score_funcÚ
route_normÚroute_scaleÚglobal_attn_every_n_layersÚsliding_windowÚlayer_typesÚattention_dropoutÚmup_enabledÚn_groupÚ
topk_groupc!           #         sö   || _ || _|| _|| _|| _|| _|p|| _|	| _|
| _|| _	|| _
|| _|| _|! dd¡}"|d u r8d|"dœ}|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _| | _t ƒ j!dd|i|!¤Ž d S )NÚ
rope_thetag     ˆÃ@Údefault)Ú	rope_typer8   r&   © )"r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   Úpopr'   r(   r   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   ÚsuperÚ__init__)#Úselfr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   Úkwargsr8   ©Ú	__class__r;   ú[/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/transformers_utils/configs/afmoe.pyr>   
   sF   $

zAfmoeConfig.__init__) r   r   r   r   r	   r
   r   Nr   r   r   r   r   TFNNr   r   r   r
   r
   r   Tr   r   r   Nr   Fr
   r
   )Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚintÚstrÚfloatÚboolÚdictÚlistr>   Ú__classcell__r;   r;   rA   rC   r      sÆ    ßþýüûúùø	÷
öõôóòñðïîíìëêéèçæåä
ãâá à!ßr   N)Ú transformers.configuration_utilsr   r   Ú__all__r;   r;   r;   rC   Ú<module>   s   
P