o
    -iy                     @   s$   d Z ddlmZ G dd deZdS )zFalcon configuration    )PretrainedConfigc                       sr   e Zd ZdZdgZddddZ					
														d	d fddZedd Zedd Z	  Z
S )RWConfigfalconpast_key_valuesn_layern_head	n_head_kv)num_hidden_layersnum_attention_headsnum_kv_heads  @         h㈵>{Gz?T           NFreturnc                    s   || _ |dd }|d u r|n|| _|| _|| _|| _|| _|| _|
| _|| _	|| _
|	| _|| _|d u r6dn|| _|| _|| _|| _|| _| jdkrMd| _t jd||	d| d S )Nn_embedr   i    T)bos_token_ideos_token_id )
vocab_sizepophidden_sizer   r   layer_norm_epsiloninitializer_range	use_cachehidden_dropoutattention_dropoutr   r   multi_queryr   alibibiasparallel_attnnew_decoder_architecturesuper__init__)selfr   r   r   r   r   r   r   r   r   r   r    r!   r   r"   r#   r$   r%   kwargsr   	__class__r   c/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/transformers_utils/configs/falcon.pyr'   #   s*   
zRWConfig.__init__c                 C   s   | j | j S N)r   r   r(   r   r   r,   head_dimS   s   zRWConfig.head_dimc                 C   s   | j  S r-   )r"   r.   r   r   r,   rotaryW   s   zRWConfig.rotary)r   r   r   r   r   r   Tr   r   r   r   TNFFFF)r   N)__name__
__module____qualname__
model_typekeys_to_ignore_at_inferenceattribute_mapr'   propertyr/   r0   __classcell__r   r   r*   r,   r      s>    0
r   N)__doc__ transformers.configuration_utilsr   r   r   r   r   r,   <module>   s   