o
    iA                     @   sH   d dl Z d dlZd dlmZmZ G dd de jZG dd dejZ	dS )    N)common_spec
model_specc                   @   s   e Zd ZdZdZdZdZdS )RotaryScalingTypezRoPE scaling type.r         N)__name__
__module____qualname____doc__LinearSuLlama3 r   r   ]/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/ctranslate2/specs/attention_spec.pyr   	   s
    r   c                   @   s2   e Zd Z														d	ddZdS )
MultiHeadAttentionSpecFNTr   '  r   c                 C   st  t j| _tj|d| _dd t|rdndD | _|r!d | _d | _	|r)d | _
d | _|
dkr6td|
| _|dkrCtd|| _|d urtd|| _|| _td|	| _|d uritd	|| _|tju rxtd|| _n|tju rd | _d | _n|tju rd | _d | _|d urtd|| _|d urtd|| _|d urtd|| _ d S d S )
N)rms_normc                 S   s   g | ]}t  qS r   )r   
LinearSpec).0_r   r   r   
<listcomp>&   s    z3MultiHeadAttentionSpec.__init__.<locals>.<listcomp>r      r   int32float32int8)!r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrangelinearrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distancenpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_window)selfself_attentionrelative_positionr#   r   r*   r+   r-   r.   r,   r(   r)   r3   r4   r5   r   r   r   __init__   sT   





zMultiHeadAttentionSpec.__init__)FFFFNTNr   r   r   r   NNN)r   r   r	   r9   r   r   r   r   r      s     r   )
enumnumpyr%   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r   r   r   r   <module>   s
    