o
    i#                     @   s"  d dl mZ d dlmZmZmZmZmZmZm	Z	m
Z
 d dlZd dlmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZmZ ddlmZm Z  dd	l!m"Z" dd
l#m$Z$m%Z% ddl&m'Z( errddl)m*Z* ddl+m,Z, de-de-de.dedegef fddZ/de-de-de-dedegef fddZ0dd Z1dd Z2	d8dede-de-dee- def
dd Z3d!ddede-de-def
d"d#Z4d!ddede-de-d$e-defd%d&Z5	'd9d!dd(ed)e6defd*d+Z7G d,d- d-Z8	'd9d.ed/ d0e8d)e6de	ej9ed/ f fd1d2Z:d:d4e.d0e8d5e.de.fd6d7Z;dS );    )partial)TYPE_CHECKINGAnyCallableIterableListOptionalTuplecastN)CosineDistance
L2Distance	LayerNormLinearMaxoutModelMultiSoftmaxSoftmaxchain
list2arrayto_categorical	zero_init)Loss)Floats2dInts1d   )IDORTH)Errors)OOV_RANKregistry)ModeDoc)Vocabmaxout_pieceshidden_sizelossreturnr#   c                    s4   dddt dt f fdd}dtffdd |S )	Nvocabr#   tok2vecr'   c                    s:   | j jd dkrttjt| |d}  |jd< |S )N   r   )r%   r$   r&   )vectorsshape
ValueErrorr   E875build_cloze_multi_task_modelattrsr(   r)   model)create_vectors_lossr%   r$    N/home/ubuntu/.local/lib/python3.10/site-packages/spacy/ml/models/multi_task.pycreate_vectors_objective$   s   
z9create_pretrain_vectors.<locals>.create_vectors_objectivec                     sR    dkrt ddd} tt| dS  dkrtdd} tt| dS ttjj dd)	NcosineT)	normalizeignore_zerosdistanceL2)r8   z'cosine', 'L2')found	supported)r   r   get_vectors_lossr   r-   r   E906formatr:   )r&   r4   r5   r3   -   s   
z4create_pretrain_vectors.<locals>.create_vectors_loss)r   r   )r$   r%   r&   r6   r4   )r3   r%   r&   r$   r5   create_pretrain_vectors!   s   	rB   n_charactersc                    s"   dddt dt f fdd}|S )Nr(   r#   r)   r'   c                    s(   t | | d}ttd|jd< |S )N)r%   r$   nr_charrD   r&   )'build_cloze_characters_multi_task_modelr   get_characters_lossr0   r1   r%   r$   rC   r4   r5   create_characters_objective>   s   z?create_pretrain_characters.<locals>.create_characters_objective)r   )r$   r%   rC   rI   r4   rH   r5   create_pretrain_characters;   s   rJ   c           
      C   s   |d j }|jjtjkr0| dd |D }|d j jj| }d||tk< |||\}}||fS |jjtjkrW| dd |D }	|j	|	}| 
|}|||\}}||fS ttjj|jjd)z^Compute a loss based on a distance between the documents' vectors and
    the prediction.
    r   c                 S   s   g | ]	}| t qS r4   )to_arrayr   ravel.0docr4   r4   r5   
<listcomp>V   s    z$get_vectors_loss.<locals>.<listcomp>c                 S   s   g | ]
}t t|tqS r4   )r
   r   rK   r   rM   r4   r4   r5   rP   [       )mode)r(   r+   rR   VectorsModedefaultflattendatar   floret	get_batch	as_contigr-   r   E850rA   )
opsdocs
predictionr;   r(   idstargetd_targetr&   keysr4   r4   r5   r?   L   s   

r?   c           	         sx   t  fdd|D }|d}| jt|dddd}|dd  f}|| }|d	  }|t|jd
  }||fS )zGCompute a loss based on a number of characters predicted from the docs.c                    s   g | ]}|j  d qS )rE   )to_utf8_arrayrM   rE   r4   r5   rP   f   s    z'get_characters_loss.<locals>.<listcomp>)   )	n_classesfdtyperc      r   )numpyvstackreshapeasarrayr   sumfloatr,   )	r[   r\   r]   rD   
target_idsr_   diffr&   r`   r4   rE   r5   rG   d   s   
rG   r)   token_vector_widthnOc                 C   sR   t ||d d}t| t|d ||ddt|d |}|d|  |d| |S )Nri   )rs   nI        )rs   rt   nPdropoutr)   output_layer)r   r   r   r   set_ref)r)   r$   rr   rs   softmaxr2   r4   r4   r5   build_multi_task_modelp   s   
r{   r(   c              	   C   sz   | j jd }ttttd tf t t||	d|dddt
||td}t||}t| |}|d| |d	| |S )
Nr*   r   rs   Tru   )rs   rt   rv   r8   rw   )rs   rt   init_Wr)   rx   )r+   r,   r   r
   r   r   r   r   r   get_dimr   r   build_masked_language_modelry   )r(   r)   r$   r%   rs   rx   r2   r4   r4   r5   r/      s"   

r/   rD   c                 C   sj   t tttd tf t t||dt|dtdg| |d}t	| t ||}|
d| |
d| |S )Nr   )rs   rv   )rt   rd   r)   rx   )r   r
   r   r   r   r   r   r   r   r~   ry   )r(   r)   r$   r%   rD   rx   r2   r4   r4   r5   rF      s   
rF   333333?wrapped_model	mask_probc                    s\   t |  fdd}ddtfdd}td||g|d|id	d
 |jD d}|d| |S )z7Convert a model into a BERT-style masked language modelc                    sX   t |d\}| jjd df| jd ||\}  fdd}||fS )N)r   r   r*   c                    s   | d 9 }  | S )Nr*   r4   )d_outputbackpropmaskr4   r5   mlm_backward   s   zFbuild_masked_language_model.<locals>.mlm_forward.<locals>.mlm_backward)_apply_maskr[   rm   rl   r,   layers)r2   r\   is_trainoutputr   r   random_wordsr   r5   mlm_forward   s
   z0build_masked_language_model.<locals>.mlm_forwardNr2   c                 S   sD   | j d }|j||d |jD ]}||r| ||| qd S )Nr   )XY)r   
initialize	dim_nameshas_dimset_dimr}   )r2   r   r   wrappeddimr4   r4   r5   mlm_initialize   s   


z3build_masked_language_model.<locals>.mlm_initializezmasked-language-modelr   c                 S   s   i | ]}|d qS Nr4   )rN   r   r4   r4   r5   
<dictcomp>   s    z/build_masked_language_model.<locals>.<dictcomp>)r   initrefsdims)NN)_RandomWordsr   r   ry   )r(   r   r   r   r   	mlm_modelr4   r   r5   r~      s   r~   c                   @   s$   e Zd Zd	ddZdefddZdS )
r   r(   r#   r'   Nc                 C   sj   dd |D | _ | j d d | _ dd |D }|d d }ttj|dd}||  }|| _g | _d S )Nc                 S   s   g | ]
}|j d kr|jqS ru   )probtextrN   lexr4   r4   r5   rP      rQ   z)_RandomWords.__init__.<locals>.<listcomp>'  c                 S   s   g | ]
}|j d kr|j qS r   )r   r   r4   r4   r5   rP      rQ   rf   rg   )wordsrj   exparrayrn   probs_cache)selfr(   r   r4   r4   r5   __init__   s   
z_RandomWords.__init__c                 C   s<   | j s| j tjjt| jd| jd | j  }| j| S )Nr   )p)	r   extendrj   randomchoicelenr   r   pop)r   indexr4   r4   r5   next   s   

z_RandomWords.next)r(   r#   r'   N)__name__
__module____qualname__r   strr   r4   r4   r4   r5   r      s    
r   r\   r"   r   c                 C   s   ddl m} tdd | D }tjdd|f}||k}d}g }| D ]2}g }	|D ]}
|| s5t|
j|}n|
j}|	| |d7 }q(d	d
 |D }|||j	|	|d q"||fS )Nr   r!   c                 s   s    | ]}t |V  qd S r   )r   rM   r4   r4   r5   	<genexpr>   s    z_apply_mask.<locals>.<genexpr>ru   g      ?r   r*   c                 S   s   g | ]}t |jqS r4   )boolwhitespace_)rN   wr4   r4   r5   rP      s    z_apply_mask.<locals>.<listcomp>)r   spaces)

tokens.docr"   rn   rj   r   uniform_replace_wordr   appendr(   )r\   r   r   r"   Nr   imasked_docsrO   r   tokenwordr   r4   r4   r5   r      s"   

r   [MASK]r   r   c                 C   s*   t j }|dk r|S |dk r| S | S )Ng?g?)rj   r   r   )r   r   r   rollr4   r4   r5   r     s   
r   r   )r   )r   )<	functoolsr   typingr   r   r   r   r   r   r	   r
   rj   	thinc.apir   r   r   r   r   r   r   r   r   r   r   r   
thinc.lossr   thinc.typesr   r   r0   r   r   errorsr   utilr   r   r+   r    rS   r   r"   r(   r#   intr   rB   rJ   r?   rG   r{   r/   rF   ro   r~   r   ndarrayr   r   r4   r4   r4   r5   <module>   s    (8





$
 