o
    
Û¾iy  ã                   @   s$   d Z ddlmZ G dd„ deƒZdS )zFalcon configurationé    )ÚPretrainedConfigc                       sr   e Zd ZdZdgZddddœZ					
														d	d‡ fdd„Zedd„ ƒZedd„ ƒZ	‡  Z
S )ÚRWConfigÚfalconÚpast_key_valuesÚn_layerÚn_headÚ	n_head_kv)Únum_hidden_layersÚnum_attention_headsÚnum_kv_headsé Ô é@   é   é   çñhãˆµøä>ç{®Gáz”?Té   ç        NFÚreturnc                    s¶   || _ | dd ¡}|d u r|n|| _|| _|| _|| _|| _|| _|
| _|| _	|| _
|	| _|| _|d u r6dn|| _|| _|| _|| _|| _| jdkrMd| _tƒ jd||	dœ|¤Ž d S )NÚn_embedr   i    T)Úbos_token_idÚeos_token_id© )Ú
vocab_sizeÚpopÚhidden_sizer   r   Úlayer_norm_epsilonÚinitializer_rangeÚ	use_cacheÚhidden_dropoutÚattention_dropoutr   r   Úmulti_queryr   ÚalibiÚbiasÚparallel_attnÚnew_decoder_architectureÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   r    r!   r   r"   r#   r$   r%   Úkwargsr   ©Ú	__class__r   úZ/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/falcon.pyr'   #   s*   
zRWConfig.__init__c                 C   s   | j | j S ©N)r   r   ©r(   r   r   r,   Úhead_dimS   s   zRWConfig.head_dimc                 C   s   | j  S r-   )r"   r.   r   r   r,   ÚrotaryW   s   zRWConfig.rotary)r   r   r   r   r   r   Tr   r   r   r   TNFFFF)r   N)Ú__name__Ú
__module__Ú__qualname__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr'   Úpropertyr/   r0   Ú__classcell__r   r   r*   r,   r      s>    ýîì0
r   N)Ú__doc__Ú transformers.configuration_utilsr   r   r   r   r   r,   Ú<module>   s   