o
    پi
                     @   s   d dl mZmZ d dlmZmZ dedefddZdedefddZ	dedefd	d
Z
eG dd deZeG dd deZdS )    )	dataclassfield)TextEncoderArchConfigTextEncoderConfignreturnc                 C   s   d| v ot | dd S )Nlayers.)strisdigitsplitr   m r   g/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/configs/models/encoders/llama.py_is_transformer_layer   s   r   c                 C   
   |  dS )Nembed_tokensendswithr   r   r   r   _is_embeddings      
r   c                 C   r   )Nnormr   r   r   r   r   _is_final_norm   r   r   c                   @   st  e Zd ZU dZeed< dZeed< dZeed< dZeed< dZ	eed	< d
Z
ed
B ed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed < d
Zed
B ed!< dZeed"< d#Zeed$< dZeed%< d
Zed
B ed&< dZeed'< d(Zeed)< e d*d+ d,Z!e"e#eeef  ed-< e d.d+ d,Z$e"ed/< d
S )0LlamaArchConfigi }  
vocab_sizei   hidden_sizei +  intermediate_size    num_hidden_layersnum_attention_headsNnum_key_value_headssilu
hidden_acti   max_position_embeddingsg{Gz?initializer_rangegư>rms_norm_epsT	use_cacher   pad_token_id   bos_token_id   eos_token_idpretraining_tpFtie_word_embeddingsg     @
rope_thetarope_scalingattention_biasg        attention_dropoutmlp_biashead_dimhidden_state_skip_layer   text_lenc                   C   s   g dS )N))	.qkv_projz.q_projq)r9   z.k_projk)r9   z.v_projv).gate_up_projz
.gate_projr   )r=   z.up_projr*   r   r   r   r   r   <lambda>3   s    zLlamaArchConfig.<lambda>default_factorystacked_params_mappingc                   C   s
   t ttgS )N)r   r   r   r   r   r   r   r>   =   s   
 _fsdp_shard_conditions)%__name__
__module____qualname__r   int__annotations__r   r   r    r!   r"   r$   r   r%   r&   floatr'   r(   boolr)   r+   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r8   r   rA   listtuplerB   r   r   r   r   r      s>   
 
r   c                   @   s,   e Zd ZU eedZeed< dZe	ed< dS )LlamaConfigr?   arch_configllamaprefixN)
rC   rD   rE   r   r   rM   r   rG   rO   r   r   r   r   r   rL   A   s   
 rL   N)dataclassesr   r   2sglang.multimodal_gen.configs.models.encoders.baser   r   r   rI   r   r   r   r   rL   r   r   r   r   <module>   s   (