o
    ÔÙ¾i¦  ã                   @   s0   d dl mZmZ d dlmZ G dd„ deƒZdS )é    )ÚListÚOptional)ÚPretrainedConfigc                E       s  e Zd ZdZ												
																							d:dededededededee dee deded ed!ed"ed#ee d$ed%ed&ed'ed(ee	 d)ed*ed+ee d,ee d-ed.ed/ed0ed1ed2ed3ed4ee d5ee
e  d6ed7efD‡ fd8d9„Z‡  ZS );ÚAfmoeConfigÚafmoeé }  é   é +  é   é    NÚsilué   ç{®Gáz”?çñhãˆµøä>Té   é   Fç     ˆÃ@ç        r   Úsigmoidç      ð?é   Ú
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmoe_intermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚhead_dimÚ
hidden_actÚmax_position_embeddingsÚinitializer_rangeÚrms_norm_epsÚ	use_cacheÚpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsÚ
rope_thetaÚrope_scalingÚattention_biasÚattention_dropoutÚnum_expertsÚnum_experts_per_tokÚnum_shared_expertsÚnum_dense_layersÚ
score_funcÚ
route_normÚroute_scaleÚn_groupÚ
topk_groupÚsliding_windowÚlayer_typesÚglobal_attn_every_n_layersÚmup_enabledc#           $         sð   || _ |
| _|| _|| _|| _|| _|| _|d u r|}|| _|d ur$|n|| | _|	| _	|| _
|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _| | _|!| _|"| _tƒ jd||||dœ|#¤Ž d S )N)r$   r%   r&   r'   © ) r   r    r   r   r   r   r   r   r   r   r!   r"   r#   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   ÚsuperÚ__init__)$Úselfr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   Úkwargs©Ú	__class__r9   úL/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/configs/afmoe.pyr;   	   sR   *ÿü
ûzAfmoeConfig.__init__)"r   r   r	   r
   r   r   NNr   r   r   r   TNr   r   Fr   NFr   NNr   r   r   Tr   r   r   NNr   F)Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚintr   ÚstrÚfloatÚboolÚdictr   r;   Ú__classcell__r9   r9   r>   r@   r      sÒ    Ùþýüûúùø	÷
öõôóòñðïîíìëêèçæåãâá à!ß#Ý
$Ü%Û'Ùr   N)Útypingr   r   Útransformersr   r   r9   r9   r9   r@   Ú<module>   s    