o
    ÔÙ¾i  ã                   @   s:   d dl mZ d dlmZ e e¡Zi ZG dd„ deƒZdS )é    )ÚPretrainedConfig)Úloggingc                       sz   e Zd ZdZdgZ										
													
		
																	
d‡ fdd„	Z‡  ZS )ÚLongcatFlashConfigÚlongcat_flashÚpast_key_valuesé   é   Né 0  é   é   é@   é   é   é   é€   é   Fçñhãˆµøä>Té   ç    ÐcAç        Úbfloat16Úfloat32ç      @é   Úidentityc*           +         sô   t ƒ jd||||| |!|"|$|#|(|)dœ|*¤Ž || _|| _|| _|d ur$|n|| _|d ur-|n|| _|| _|| _|	| _	|
| _
|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|| _|&| _|'| _|%| _d| _d S )N)Úpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsÚtorch_dtypeÚparams_dtypeÚrounter_params_dtypeÚtopk_methodÚrouter_biasÚnextn_use_scmoeÚnum_nextn_predict_layersÚsilu© )ÚsuperÚ__init__Ú
vocab_sizeÚmax_position_embeddingsÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚmoe_intermediate_sizeÚnum_attention_headsÚep_sizeÚkv_lora_rankÚq_lora_rankÚqk_rope_head_dimÚ
v_head_dimÚqk_nope_head_dimÚn_routed_expertsÚmoe_topkÚnorm_topk_probÚrms_norm_epsÚpretraining_tpÚ	use_cacheÚ
rope_thetaÚrope_scalingÚattention_biasÚattention_dropoutÚmla_scale_q_loraÚmla_scale_kv_loraÚzero_expert_numÚzero_expert_typeÚrouted_scaling_factorÚ
hidden_act)+Úselfr*   r,   r.   Úffn_hidden_sizeÚexpert_ffn_hidden_sizeÚ
num_layersr-   r0   r1   r2   r3   r4   r6   r5   r7   r8   r9   r+   r:   r<   r   r   r   r;   r   r=   r>   r?   r@   rA   rB   r   r    r!   r#   r"   rE   rC   rD   r$   r%   Úkwargs©Ú	__class__r'   úT/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/configs/longcat_flash.pyr)      s\   -õôÿÿ
zLongcatFlashConfig.__init__))r   r   Nr	   r
   r   Nr   r   r   r   r   r   r   r   r   Fr   r   TNr   r   r   Fr   NFr   TTr   r   r   FNr   r   r   Fr   )Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚkeys_to_ignore_at_inferencer)   Ú__classcell__r'   r'   rL   rN   r   	   sZ    Ör   N)	Ú transformers.configuration_utilsr   Útransformers.utilsr   Ú
get_loggerrO   ÚloggerÚ#FLASH_PRETRAINED_CONFIG_ARCHIVE_MAPr   r'   r'   r'   rN   Ú<module>   s
    
