o
    -i                     @   s&   d dl mZ G dd deZdgZdS )    )PretrainedConfigc                A       s   e Zd ZdZ										
																										d:dedededededededed	B d ed!ed"ed#ed$ed%ed&ed'ed	B d(ed	B d)ed*ed+ed,ed-ed.ed/ed0ed1ed2ed3e	e d	B d4ed5ed6ed7ef@ fd8d9Z
  ZS );AfmoeConfigafmoe                    N   silu   {Gz?h㈵>TF@         sigmoid      ?           
vocab_sizehidden_sizeintermediate_sizemoe_intermediate_sizenum_hidden_layersnum_dense_layersnum_attention_headsnum_key_value_headshead_dim
hidden_actmax_position_embeddingsinitializer_rangerms_norm_eps	use_cachetie_word_embeddingsrope_parametersrope_scalingnum_expertsnum_experts_per_toknum_shared_expertsnum_expert_groupsnum_limited_groups
score_func
route_normroute_scaleglobal_attn_every_n_layerssliding_windowlayer_typesattention_dropoutmup_enabledn_group
topk_groupc!           #         s   || _ || _|| _|| _|| _|| _|p|| _|	| _|
| _|| _	|| _
|| _|| _|!dd}"|d u r8d|"d}|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _| | _t  j!dd|i|! d S )N
rope_thetag     @default)	rope_typer8   r&    )"r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   popr'   r(   r   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   super__init__)#selfr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   kwargsr8   	__class__r;   b/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/transformers_utils/configs/afmoe.pyr>   
   sF   $

zAfmoeConfig.__init__) r   r   r   r   r	   r
   r   Nr   r   r   r   r   TFNNr   r   r   r
   r
   r   Tr   r   r   Nr   Fr
   r
   )__name__
__module____qualname__
model_typeintstrfloatbooldictlistr>   __classcell__r;   r;   rA   rC   r      s    	

 !r   N) transformers.configuration_utilsr   r   __all__r;   r;   r;   rC   <module>   s   
P