o
    
۾iU                     @   s,   d dl mZ d dlmZ G dd deZdS )    )Any)PretrainedConfigc                       sh   e Zd ZdZdgZ											
													ddeeef dB f fddZ  Z	S )FlexOlmoConfig	flex_olmopast_key_values      +      Nsilu{Gz?ư>T鵇 顇 F              {Gz?rope_parametersc                    s  d|vr	dg|d< t  jd
||||d| || _|| _|| _|| _|| _|| _|d u r/|}|| _|| _	|	| _
|
| _|| _|dd }|pK|pKddi}|dd}d|vrZ||d< || _|| _|| _|| _|| _|| _|| _|| _| jd urd	| jv r| jd	 | jd< d S d S d S )NarchitecturesFlexOlmoForCausalLM)pad_token_idbos_token_ideos_token_idtie_word_embeddingsrope_scaling	rope_typedefault
rope_thetag    Atype )super__init__
vocab_sizemax_position_embeddingshidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actinitializer_rangerms_norm_eps	use_cachepopr   attention_biasattention_dropoutnum_experts_per_toknum_expertsoutput_router_logitsrouter_aux_loss_coefnorm_topk_prob)selfr#   r%   r&   r'   r(   r)   r*   r$   r+   r,   r-   r   r   r   r   r   r/   r0   r1   r2   r3   r4   r5   kwargsr   r   	__class__r    ]/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/flex_olmo.pyr"      sN   
zFlexOlmoConfig.__init__)r   r   r	   r
   r
   Nr   r   r   r   Tr   Nr   FNFr   r   r   Fr   F)
__name__
__module____qualname__
model_typekeys_to_ignore_at_inferencedictstrr   r"   __classcell__r    r    r8   r:   r      s:    r   N)typingr    transformers.configuration_utilsr   r   r    r    r    r:   <module>   s   