o
    i*                     @   s,  d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZ ejjejjejjdZh dZG dd	 d	e
Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd0ddZdd Zdd  Zd0d!d"Z d1d$d%Z!d&d' Z"d2d(d)Z#d3d+d,Z$d-d. Z%e&d/kre%  dS dS )4    N)List)utils)	Converter)common_spectransformer_spec)gelureluswish>    dnndc                   @   s.   e Zd ZdZdedee fddZdd ZdS )	MarianConverterz$Converts models trained with Marian.
model_pathvocab_pathsc                 C   s   || _ || _dS )zInitializes the Marian converter.

        Arguments:
          model_path: Path to the Marian model (.npz file).
          vocab_paths: Paths to the vocabularies (.yml files).
        N)_model_path_vocab_paths)selfr   r    r   Z/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/ctranslate2/converters/marian.py__init__   s   
zMarianConverter.__init__c              	   C   s  t | j}t|}ttt| j}|d }d|d v }|d }t	 }||d dkd ||d d	kd
 ||d  d ||t
v d|dt
 f  ||tv d|dtf  |rv||d dkor|d dkor|dddkd n||d dko|d dko|dddkd |  |d }|dkrdnt|d }d|v }	tjj|d |d f|d |t
| |d|	d}
t|
| |
|d  |
|d  d |
j_|
S )!Nztransformer-ffn-activationr   ztransformer-preprocessztransformer-postprocess-embtypetransformerz#Option --type must be 'transformer'ztransformer-decoder-autoregzself-attentionz=Option --transformer-decoder-autoreg must be 'self-attention'ztransformer-no-projectionz3Option --transformer-no-projection is not supportedzWOption --transformer-ffn-activation %s is not supported (supported activations are: %s)z, zSOption --transformer-postprocess-emb %s is not supported (supported values are: %s)ztransformer-postprocessdaztransformer-postprocess-topr
   zUnsupported pre-norm Transformer architecture, expected the following combination of options: --transformer-preprocess n --transformer-postprocess da --transformer-postprocess-top ndanzUnsupported post-norm Transformer architecture, excepted the following combination of options: --transformer-preprocess '' --transformer-postprocess dan --transformer-postprocess-top ''z"transformer-guided-alignment-layerlast   z	enc-depthz	dec-depthztransformer-heads)pre_norm
activationalignment_layeralignment_headslayernorm_embeddingr   T)nploadr   _get_model_configlistmap
load_vocabr   r   ConfigurationChecker_SUPPORTED_ACTIVATIONSjoinkeys_SUPPORTED_POSTPROCESS_EMBgetvalidateintr   TransformerSpecfrom_configset_transformer_specregister_source_vocabularyregister_target_vocabularyconfigadd_source_eos)r   modelr6   vocabsr   r   postprocess_embcheckr    r"   
model_specr   r   r   _load#   sz   



	zMarianConverter._loadN)__name__
__module____qualname____doc__strr   r   r=   r   r   r   r   r      s    
r   c                 C   s&   | d }|d d   }t|}|S )Nzspecial:model.ymlr   )tobytesyaml	safe_load)r8   r6   r   r   r   r%   o   s   
r%   c                 C   s  t | dd}g }d }d }t|D ]\}}|d}|sq|dr)|dd  }n|d ur4|dd  }n|dd\}}|d ur|dri|dritd	d
|}|dd }|drhtt	|dd  dd}n|dr|dr|dd }|
dd}|d urzt	| }W n ty } ztd|d |f |d }~ww |||f d }d }qW d    n1 sw   Y  dd t|dd dD S )Nzutf-8)encodingz
z?    :r   "z\\([^x])z\1r   z\x   )base'z''z"Unexpected format at line %d: '%s'c                 S   s   g | ]\}}|qS r   r   ).0_tokenr   r   r   
<listcomp>       zload_vocab.<locals>.<listcomp>c                 S   s   | d S )Nr   r   )itemr   r   r   <lambda>   s    zload_vocab.<locals>.<lambda>)key)open	enumeraterstrip
startswithrsplitendswithresubchrr0   replacestrip
ValueErrorappendsorted)pathvocabtokensrO   idxilineer   r   r   r(   v   sR   


)r(   c                 C   s    t | j|d t| j|d d S )Nencoderdecoder)set_transformer_encoderrj   set_transformer_decoderrk   )specweightsr   r   r   r3      s   r3   c                 C   s<   t | || t| jD ]\}}t||d||d f  qd S )N%s_l%dr   )set_common_layersrV   layerset_transformer_encoder_layerrn   ro   scoperg   
layer_specr   r   r   rl      s   rl   c                 C   s\   d| _ t| || t| jD ]\}}t||d||d f  qt| j|d| | jjd d S )NTrp   r   z%s_ff_logit_out)reuse_weight)	start_from_zero_embeddingrq   rV   rr   set_transformer_decoder_layer
set_linear
projection
embeddingsweightrt   r   r   r   rm      s   
rm   c                 C   s   | j }t|ts|g}t|d || t| j||d jjd d t| dr1t	| j
|d| dd t| drAt	| j|d	|  d S d S )
Nr   r   )dimr"   z%s_embTr   
layer_normz%s_top)r|   
isinstancer&   set_embeddingsset_position_encodingsposition_encodingsr}   shapehasattrset_layer_normr"   r   )rn   ro   ru   embeddings_specsr   r   r   rq      s"   


rq   c                 C   s,   t | j|d|  t| j|d| dd d S )N%s_ffn%s_selfTself_attention)set_ffnffnset_multi_head_attentionr   rn   ro   ru   r   r   r   rs      s   
rs   c                 C   s>   t | j|d|  t| j|d| dd t| j|d|  d S )Nr   r   Tr   z
%s_context)r   r   r   r   	attentionr   r   r   r   ry      s
   ry   Fc                 C   s   dd t dD }t|d ||d t|d ||d t|d ||d	 |r0t| jd | n|d j| jd _|d j| jd _t| jd |dd   t| jd
 ||d t| j|d|  d S )Nc                 S   s   g | ]}t  qS r   )r   
LinearSpec)rM   rN   r   r   r   rP      rQ   z,set_multi_head_attention.<locals>.<listcomp>   r   qr   krG   vr   oz%s_Wo)	rangerz   r   fuse_linearlinearr}   biasset_layer_norm_autor   )rn   ro   ru   r   split_layersr   r   r   r      s   r   c                 C   s6   t | j|d|  t| j||d t| j||d d S )Nr   12)r   r   rz   linear_0linear_1r   r   r   r   r      s   r   c                 C   s8   zt | ||dd W d S  ty   t | || Y d S w )NTr   )r   KeyErrorr   r   r   r   r      s
   r   c                 C   s<   |rdnd}|d||f    | _|d||f    | _d S )N_prer
   z%s_ln_scale%sz%s_ln_bias%s)squeezegammabeta)rn   ro   ru   r   suffixr   r   r   r      s   r   r
   c                 C   sj   | d||f }|d u r| d||f |}n| }|| _| d||f }|d ur3| | _d S d S )Nz%s_W%sz%s_Wt%sz%s_b%s)r.   	transposer}   r   r   )rn   ro   ru   r   rw   r}   r   r   r   r   rz     s   rz   c                 C   s.   | d| | _| jd u r| d| _d S d S )Nz%s_WembWemb)r.   r}   r   r   r   r   r     s   
r   c                 C   s   | dt|| _d S )NWpos)r.   #_make_sinusoidal_position_encodings	encodings)rn   ro   r~   r   r   r   r     s   r      c                 C   s   t |}t ddt | d  |  }t |dt |d }t |}t |d d dd df |d d d | d f< t |d d dd df |d d | d d f< |S )Ni'  rG   r   r   )r#   arangepowerexpand_dims
zeros_likesincos)r~   num_positions	positions
timescalesposition_enctabler   r   r   r     s   

00r   c                  C   s^   t jt jd} | jdddd | jddddd	 t|  |  }t|j|j	}|
| d S )
N)formatter_classz--model_pathTzPath to the model .npz file.)requiredhelpz--vocab_paths+z'List of paths to the YAML vocabularies.)r   nargsr   )argparseArgumentParserArgumentDefaultsHelpFormatteradd_argumentr   declare_arguments
parse_argsr   r   r   convert_from_args)parserargs	converterr   r   r   main'  s    
r   __main__)F)r
   N)N)r   )'r   r[   typingr   numpyr#   rD   ctranslate2.convertersr    ctranslate2.converters.converterr   ctranslate2.specsr   r   
ActivationGELUSigmoidRELUSWISHr*   r-   r   r%   r(   r3   rl   rm   rq   rs   ry   r   r   r   r   rz   r   r   r   r   r>   r   r   r   r   <module>   sD    Y.






