o
    TiB                     @   sH   d dl Z d dlZd dlmZmZ G dd de jZG dd dejZ	dS )    N)common_spec
model_specc                   @   s   e Zd ZdZdZdZdZdS )RotaryScalingTypezRoPE scaling type.r         N)__name__
__module____qualname____doc__LinearSuLlama3 r   r   T/home/ubuntu/.local/lib/python3.10/site-packages/ctranslate2/specs/attention_spec.pyr   	   s
    r   c                   @   s:   e Zd Z																		d	ddZdS )
MultiHeadAttentionSpecFNTr   '  r   c                 C   s  t j| _|rtj|d| _dd t|rdndD | _|r+tj|d| _tj|d| _	|r3d | _
d | _|r;d | _d | _|rFd | _d | _d | _|dkrStd|| _|dkr`td|| _|d urtd|| _|| _td|
| _|d urtd	|| _|tju rtd|	| _n|tju rd | _d | _n|tj u rd | _!d | _"|d urtd|| _#|d urtd|| _$|d urtd|| _%d S d S )
N)rms_normc                 S   s   g | ]}t  qS r   )r   
LinearSpec).0_r   r   r   
<listcomp>+   s    z3MultiHeadAttentionSpec.__init__.<locals>.<listcomp>r      r   int32float32int8)&r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrangelinearq_normk_normrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_window)selfself_attentionrelative_positionrelative_asymmetric_positionr%   r   r/   r0   r2   r3   r1   r-   r.   r8   r9   r:   qk_normqk_norm_rmshas_normr   r   r   __init__   sd   





zMultiHeadAttentionSpec.__init__)FFFFFNTNr   r   r   r   NNNFTT)r   r   r	   rB   r   r   r   r   r      s(    r   )
enumnumpyr*   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r   r   r   r   <module>   s
    