o
    ॵi                     @   s  d dl mZ d dlmZmZ d dlZd dlZd dl	Z
d dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z m!Z! d dl"m#Z# e
j$dkroe
j%j&Z
e
'  e# Z(dgZ)ej*e!j+ej,dG dd deZ-dS )    N)AnyDict)MosesDetokenizerMosesPunctNormalizerMosesTokenizer)	apply_bpe)	Pipelines)Model)
OutputKeys)Pipeline)	PIPELINES)TranslationPipeline)Config)	ModelFileTasks)
get_loggerz2.0InteractiveTranslationPipeline)module_namec                       s   e Zd Zdef fddZdedeeef fddZdeeef deeef fdd	Z	d
eeef deeef fddZ
  ZS )r   modelc                    s  t  jdd|i| | jj}t  tt|tj	d}t
dd tt| jD | _t| j| _tjtjddgdd| _tjtjddgdd| _tjtjddgd	d| _i | _| jd
krct| _nt| jd| _t| jd| _| | jd| j| j}| j| tj dd}d|j!_"tj#|d| _$| j$% }t&'d|  tj()t* }|+|| W d   dS 1 sw   Y  dS )u  Build a interactive translation pipeline with a model dir or a model id in the model hub.

        Args:
            model (`str` or `Model` or module instance): A model instance or a model local dir
                or a model id in the model hub.

        Example:
            >>> from modelscope.pipelines import pipeline
            >>> pipeline_ins = pipeline(task=Tasks.translation,
                model='damo/nlp_imt_translation_zh2en')
            >>> input_sequence = 'Elon Musk, co-founder and chief executive officer of Tesla Motors.'
            >>> input_prefix = "特斯拉汽车公司"
            >>> print(pipeline_ins(input_sequence + "<PREFIX_SPLIT>" + input_prefix))
        r   zckpt-0c                 S   s   g | ]
\}}|  |fqS  )strip).0iwr   r   m/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/nlp/interactive_translation_pipeline.py
<listcomp>8   s    z;InteractiveTranslationPipeline.__init__.<locals>.<listcomp>N
input_wids)dtypeshapenameprefix_wids
prefix_hitzh)langT)allow_soft_placement)configzloading model from r   ),super__init__r   	model_dirtfreset_default_graphospjoinr   TF_CHECKPOINT_FOLDERdict	enumerateopen_trg_vocab_path
_trg_vocablen_trg_rvocab_len_tgt_vocabplaceholderint64r   r    boolr!   output	_tgt_langjieba_tgt_tokr   _tgt_punct_normalizerr   updateConfigProtogpu_optionsallow_growthSession_session
as_defaultloggerinfotrainSaverglobal_variablesrestore)selfr   kwargs
model_pathr9   	tf_configsessmodel_loader	__class__r   r   r'   #   sL   

"z'InteractiveTranslationPipeline.__init__inputreturnc                    s  | dd\}} jdkr j|}dt|}n j|} jj|ddd} j	dkrA j
|}dt|d d }n j|} j
j|ddd  }d|d d }tt|dkrjt|d }nd	} j|} j|}t fd
d|   D g}	t fdd|   D g}
dg jd  g}|d	krd} j D ]\}}||rd|d |< d}q|du rdg jd  g}|	|
t|d}|S )Nz<PREFIX_SPLIT>   r"    T)
return_straggressive_dash_splitsr    c                    .   g | ]}| j v r j | n jd  d qS )r   src_vocab_size)
_src_vocabcfgr   r   rK   r   r   r   y       

z=InteractiveTranslationPipeline.preprocess.<locals>.<listcomp>c                    r[   )r   trg_vocab_size)r2   r^   r_   r`   r   r   r      ra   F)	input_ids
prefix_idsr!   )split	_src_lang_tokcutr,   list_punct_normalizer	normalizetokenizer:   r<   lcutr=   r3   _bpeprocess_linenparrayr   r5   r4   items
startswith)rK   rS   	input_srcprefix	input_tok
prefix_toksubword	input_bpe
prefix_bperc   rd   r!   	hit_stater   r   resultr   r`   r   
preprocess_   sX   






z)InteractiveTranslationPipeline.preprocessc                 C   sf   | j  $ | j|d | j|d | j|d i}| j j| j|d}|W  d    S 1 s,w   Y  d S )Nrc   rd   r!   )	feed_dict)rC   rD   r   r    r!   runr9   )rK   rS   r~   sess_outputsr   r   r   forward   s   


$z&InteractiveTranslationPipeline.forwardinputsc                    sv   |d d }t |d dg }|d |d }d fdd|D dddd} j| }tj|i}|S )	Noutput_seqsr   rV   c                    s$   g | ]}| j v r j | nd qS )z<unk>)r4   )r   widr`   r   r   r      s    z>InteractiveTranslationPipeline.postprocess.<locals>.<listcomp>z@@ rZ   z@@)	ri   indexr,   replace_detok
detokenizere   r
   TRANSLATION)rK   r   r   widstranslation_outr|   r   r`   r   postprocess   s   
z*InteractiveTranslationPipeline.postprocess)__name__
__module____qualname__r	   r'   strr   r   r}   r   r   __classcell__r   r   rQ   r   r      s
    <"7*
).os.pathpathr+   typingr   r   r;   numpyrp   
tensorflowr)   
sacremosesr   r   r   subword_nmtr   modelscope.metainfor   modelscope.models.baser	   modelscope.outputsr
   modelscope.pipelines.baser   modelscope.pipelines.builderr   -modelscope.pipelines.nlp.translation_pipeliner   modelscope.utils.configr   modelscope.utils.constantr   r   modelscope.utils.loggerr   __version__compatv1disable_eager_executionrE   __all__register_moduletranslationinteractive_translationr   r   r   r   r   <module>   s2   
