o
    iM                     @   s  d dl Z d dlZd dlmZ d dlZd dlZd dlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZ d dlmZ ddlmZ ejdd Zd	d
 Zdd Zdd Zdd Zdd Z dd Z!dd Z"dd Z#dd Z$dd Z%dd Z&dd  Z'd!d" Z(d#d$ Z)d%d& Z*d'd( Z+d)d* Z,d+d, Z-d-d. Z.d/d0 Z/d1d2 Z0ej12d3d43 d5d6 Z4d7d8 Z5ej1j6e d9d:d;d< Z7d=d> Z8d?d@ Z9dAdB Z:dCdD Z;dEdF Z<dGdH Z=dIdJ Z>dKdL Z?dMdN Z@dS )O    N)Counter)AdamCupyOpsDropoutLinearModelReluShimSoftmaxchainchange_attr_valuesconcatenateset_dropout_rateuse_ops
with_debugwrap_model_recursive)has_cupy_gpu   )make_tempdirc                   C   s   t  S N)r    r   r   P/home/ubuntu/.local/lib/python3.10/site-packages/thinc/tests/model/test_model.pymodel_with_no_args   s   r   c                 C   s   t | dd S )Nc                 S      | dd fS )Nc                 S      | S r   r   dYr   r   r   <lambda>$       z0create_model.<locals>.<lambda>.<locals>.<lambda>r   Xr   r   r   r   $       zcreate_model.<locals>.<lambda>r   namer   r   r   create_model#   s   r%   c                 C   s   t | jtrJ d S r   )
isinstanceopsr   )r   r   r   r   test_model_defaults_to_cpu'      r(   c                 C   s    t  }t  }|j|jksJ d S r   )r   id)r   model1model2r   r   r   test_models_get_different_ids+   s   r-   c               
   C   s  G dd dt } td}tddd dd dtd	d d
|d dddi| d g||gd}|ds4J |djd	ks>J |dd u sGJ t	t
 |d W d    n1 s\w   Y  t	t
 |d W d    n1 svw   Y  |dtd	 |dsJ |djd	ksJ ||jdftd	i t|dtd	sJ W d    n1 sw   Y  t|dtd	sJ |drJ |drJ t	t
 |d W d    n1 sw   Y  |d|jd |d|jd t	t |dtd W d    n	1 s$w   Y  |ds1J |ddks;J t	t
 |d W d    n	1 sQw   Y  t	t |d W d    n	1 slw   Y  |dsyJ |djdksJ |drJ t	t
 |d W d    n	1 sw   Y  |dd u sJ t	t |d W d    n	1 sw   Y  |d| |dsJ |djdksJ t	t |dtd W d    n	1 sw   Y  d|jv sJ d|jvsJ |jd dksJ t	t
 |jd  W d    n	1 s3w   Y  d|jd< | }|jdksIJ d S )Nc                   @      e Zd ZdZdS )ztest_model_init.<locals>.MyShimtestshimN__name__
__module____qualname__r$   r   r   r   r   MyShim2       r4   atestc                 S   r   )Nc                 S   r   r   r   r   r   r   r   r   8   r   z3test_model_init.<locals>.<lambda>.<locals>.<lambda>r   r   r   r   r   r   8   r!   z!test_model_init.<locals>.<lambda>
   nInOr8   Wbr6   r?   foobardimsparamsrefsattrsshimslayersr>   r?   r    xyz)   r   r:   r;   cbaz)r	   r%   r   numpyzeros	has_param	get_paramshapepytestraisesKeyError	set_param
use_paramsr*   onesarray_equalhas_gradget_gradr'   alloc1fset_grad
ValueErrorinc_gradhas_dimget_dimhas_refget_refr$   set_refrG   copy)r4   model_amodel
model_copyr   r   r   test_model_init1   s   

ri   c               
   C   s  G dd dt } td}tddd dd dd d d	|d d
ddi| d g||gd}tt |dd W d    n1 s@w   Y  |jdddd |d|j	d |
d|j	d |dshJ |ddksqJ tt |dd W d    n1 sw   Y  tt |dd W d    n1 sw   Y  tt |jdddd W d    d S 1 sw   Y  d S )Nc                   @   r.   )z"test_model_set_dim.<locals>.MyShimr/   Nr0   r   r   r   r   r4   s   r5   r4   r6   r7   c                 S   r   )Nc                 S   r   r   r   r   r   r   r   r   y   r   z6test_model_set_dim.<locals>.<lambda>.<locals>.<lambda>r   r   r   r   r   r   y   r!   z$test_model_set_dim.<locals>.<lambda>rK   r9   r=   r@   rA   rB   rC   r:   r8   T)forcer>   rJ      )r	   r%   r   rS   rT   r^   set_dimrV   r'   r\   r]   r`   ra   rU   )r4   rf   rg   r   r   r   test_model_set_dimr   s:   
"rm   c                  C   sP   t d} | jt ksJ | dd  | jdksJ | dd  | jdks&J d S )Ntmpparam1ro   param2)ro   rq   )r%   param_namestuplerV   rg   r   r   r   test_param_names   s   ru   c                  C   sZ   t d} | jt ksJ | d| jdd | d| jddd  | jdks+J d S )Nrn   ro         rp   )r%   
grad_namesrs   rV   r'   alloc2fr]   rt   r   r   r   test_grad_names   s
   rz   c                  C   s   t dd} | jdksJ d S )NrK      )r;   r:   )r   	dim_namesrt   r   r   r   test_dim_names   s   
r}   c                  C   s   t d} t d}t d}| j| | jt ksJ | d| | jdks'J | d|u s0J |j| tt	 | d W d    n1 sKw   Y  | d| | d|u s_J | 
| ||jvskJ | drrJ d S )Nparentchildkid)r   grandkid	grandkind)r%   rI   append	ref_namesrs   rd   rc   rS   rT   rU   remove_noderb   )r~   r   
grandchildr   r   r   test_model_set_reference   s"   
r   c                  C   sz   t d} | dd u sJ | dd | ddksJ | dd u s%J | dd u s.J |   | dd us;J d S )NrK   r:   rv   boor>   )r   maybe_get_dimrl   maybe_get_refmaybe_get_param
initializert   r   r   r   test_maybe_methods   s   r   c                 C   s:   t  }| |d  W d    d S 1 sw   Y  d S Nthinc_model)r   to_disk)r   pathr   r   r   test_model_can_save_to_disk   s   "r   c                 C   sZ   t  }| |d  | |d }W d    n1 sw   Y  |  | ks+J d S r   )r   r   	from_diskto_bytes)r   r   m2r   r   r   test_model_can_load_from_disk   s
   r   c                 C   s4   |d }|  | | |}|  | ksJ d S r   )r   r   r   )r   pathy_fixturer   r   r   r   r   +test_model_can_roundtrip_with_path_subclass   s   

r   c                 C   sL   | }d|_ d|jd< t|ddddi |jd du sJ d|jvs$J d S )NtargetFhas_varT)r   errorr   )r$   rG   r   r   rg   r   r   r   test_change_attr_values   s   
r   c                  C   s8   t  } | jd dksJ t| d | jd dksJ d S )Ndropout_rate        皙?)r   rG   r   rt   r   r   r   test_set_dropout   s   
r   c                 C   s4   | }d|_ d|jd< t|d |jd dksJ d S )Ndropoutr   r   r   )r$   rG   r   r   r   r   r   test_set_dropout_2   s
   

r   c                  C   sX   t ddd i tddtdd } | dksJ W d    d S 1 s%w   Y  d S )N+c                 S   s   | j |j fS r   r#   r@   r   r   r   r      r!   z test_bind_plus.<locals>.<lambda>r6   r#   r?   r@   )r   define_operatorsr%   mr   r   r   test_bind_plus   s   "r   c                  C   sn   t ddd i$ tddtdd tdd tdd } | jdks%J W d    d S 1 s0w   Y  d S )	Nr   c                 S   r   r   r   r@   r   r   r   r      r   z!test_plus_chain.<locals>.<lambda>r6   r#   r?   rL   d)r   r   r%   r$   r   r   r   r   test_plus_chain   s   "r   c                  C   s|   t jtdd} t jtdd}|  |   |   |  t jtdd} t jtdd}|  |   |   |  dS )zGTest we can create a model in a child thread with overloaded operators.)r   r   r   args)*rw   )r   rw   )r   r   N)	threadingThread_overload_plusstartjoin)worker1worker2r   r   r   $test_overload_operators_in_subthread  s   r   c                 C   s   t dd}t dd}t| dd i t| | dkr"|| }n|| }W d    n1 s0w   Y  |dks;J tj i ksDJ d S )Nr6   r#   r?   c                 S      | j |j  S r   r#   r@   r   r   r   r     r!   z _overload_plus.<locals>.<lambda>r   ab)r%   r   r   timesleep_context_operatorsget)operatorr   m1r   valuer   r   r   r     s   



r   c                  C   s   t dd} t dd}tj i ksJ tddd i | | }tt | | }W d    n1 s5w   Y  tddd ix tt | | }W d    n1 sXw   Y  | | }td	d
d i% tt | | }W d    n1 sw   Y  | | }W d    n1 sw   Y  tt | | }W d    n1 sw   Y  | | }W d    n1 sw   Y  | | }tt | | }W d    n1 sw   Y  W d    n1 sw   Y  |dksJ tj i ksJ d S )Nr6   r#   r?   r   c                 S   r   r   r#   r@   r   r   r   r   '  r!   z/test_nested_operator_contexts.<locals>.<lambda>r   c                 S   r   r   r#   r@   r   r   r   r   +  r!   -c                 S   r   r   r#   r@   r   r   r   r   /  r!   r   )r%   r   r   r   r   rS   rT   	TypeError)r   r   r   r   r   r   test_nested_operator_contexts#  s@   








r   opz+ - * @ / // % ** << >> & ^ |c              	   C   s2  t  }t  }t| dd i | dkr|| }ntt || }W d    n1 s.w   Y  | dkr<|| }ntt || }W d    n1 sPw   Y  | dkr^|| }ntt || }W d    n1 srw   Y  | dkr||}ntt ||}W d    n1 sw   Y  | dkr|| }ntt || }W d    n1 sw   Y  | dkr|| }ntt || }W d    n1 sw   Y  | d	kr||A }ntt ||A }W d    n1 sw   Y  | d
kr|| }ntt || }W d    n	1 s w   Y  | dkr/|| }ntt || }W d    n	1 sDw   Y  | dkrS||> }ntt ||> }W d    n	1 shw   Y  | dkrw||? }ntt ||? }W d    n	1 sw   Y  | dkr||@ }ntt ||@ }W d    n	1 sw   Y  | d	kr||A }ntt ||A }W d    n	1 sw   Y  | dkr||B }ntt ||B }W d    n	1 sw   Y  W d    n	1 sw   Y  tj i ksJ d S )Nc                 S   r   r   r#   r@   r   r   r   r   A  r!   z$test_all_operators.<locals>.<lambda>r   r   r   @/z//^%z**z<<z>>&|)	r   r   r   rS   rT   r   
__matmul__r   r   )r   r   r   r   r   r   r   test_all_operators=  s   
































Kr   c                  C   s   g } dd }d}t | dk rAg }tdD ]}tj|| |fd}|| |d7 }q|D ]}|  q+|D ]}|  q4t | dk st | t tt| ksOJ dS )	z<Create a bunch of threads and assert they all get unique IDsc                 S   s   |  td| dj d S )Nworkerr#   )r   r%   r*   )id_listindexr   r   r   get_model_id  s   z3test_unique_id_multithreading.<locals>.get_model_idr   i  2   r   rw   N)	lenranger   r   r   r   r   listset)list_of_idsr   counterworkersiwr   r   r   test_unique_id_multithreading  s   



 r   zneeds CuPy GPU)reasonc                  C   s  t d dd l} td d}d}|  \\}}\}}tt||dt||dt }|j	|}|j	|}|j	|}|j	|}|j
|d d |d d d td	}d
}	tdD ]R}
|jj|	||dd}|D ]\}}||\}}|||  || qjd}d}|j|	||D ] \}}||}||jdd|jddk 7 }||jd 7 }qq\W d    d S 1 sw   Y  d S )Nml_datasetsr   cupy    r   )r;   r   rK   r    YgMbP?   r   T)shufflerw   )axis)rS   importorskipr   r   mnistr   r   r
   r'   asarrayr   r   r   
multibatchbegin_updatefinish_updatepredictargmaxsumrR   )r   n_hiddenr   train_Xtrain_Ydev_Xdev_Yrg   	optimizer
batch_sizer   batchesr    r   Yhbackpropcorrecttotalr   r   r   test_model_gpu  sB   




 "r   c                  C   s  t d} t d}t| |}t| }tddd | || |g| || dd}|| | |jd |ks2J |jd |ks;J |jd	 |ksDJ |d
|ksMJ |d|ksVJ |d|ks_J |jd |kshJ |jd jd |kstJ |jd jd |ksJ d S )NrK   r7   c                 S   r   )Nc                 S   r   r   r   r   r   r   r   r     r   z5test_replace_node.<locals>.<lambda>.<locals>.<lambda>r   r   r   r   r   r     r!   z#test_replace_node.<locals>.<lambda>)relu1relu2relu3)rI   rF   r   rw   r   r   r   r   r{   )r   r   r   r   replace_noderI   rc   )r   r   
relu_chainrelu1_debugdebugr   r   r   test_replace_node  s(   


r   c                  C   s   dd } | dg }| d|g}t |}| d|g}|d| t||}d|_||| |jd jd |ks9J |jd jd jd |ksHJ |jd d|ksTJ d S )	Nc                 S      t | dd |dS )Nc                 S      dS N.r   rg   r    is_trainr   r   r   r     r   zOtest_replace_node_with_indirect_node_ref.<locals>.dummy_model.<locals>.<lambda>rI   r"   r$   rI   r   r   r   dummy_model     z=test_replace_node_with_indirect_node_ref.<locals>.dummy_modelyxr?   r6   r   rw   )r   rd   r   r$   r   rI   rc   )r  r
  r  y_debugr?   r6   r   r   r   (test_replace_node_with_indirect_node_ref  s   	

r  c                     s   t d dd l} |  \\}}\}}t   fdd} fdd} fdd}t }tt |||d	}	t||	|	}
|
j|d d
 |d d
 d |
|d d
 dd\}}||d d
   ddddksgJ d S )Nr   r   c                         d  d7  < d S )Ninitrw   r   _countsr   r   on_init  r)   z test_with_debug.<locals>.on_initc                     r  )Nforwardrw   r   r  r  r   r   
on_forward  r)   z#test_with_debug.<locals>.on_forwardc                     r  )Nr   rw   r   r  r  r   r   on_backprop  r)   z$test_with_debug.<locals>.on_backprop)r  r  r  rK   r   F)r    r  r   rv   )r  r  r   )	rS   r   r   r   r   r   r   r   r   )r   r   r   r   r   r  r  r  relur   chainedr  r   r   r  r   test_with_debug  s    

r  c                     s   dd  t d} t| | }t| fdd}|jdksJ |jd |u s&J |jd jd jdks3J |jd jd jd | u sBJ |jd jd	 jdksOJ |jd jd	 jd | u s^J d S )
Nc                 S   r  )Nc                 S   r  r  r   r  r   r   r   r   '  r   z:test_recursive_wrap.<locals>.dummy_model.<locals>.<lambda>r  r"   r  r   r   r   r  &  r	  z(test_recursive_wrap.<locals>.dummy_modelrK   c                        d| j  d| gS Nzdummy()r#   rt   r  r   r   r   1      z%test_recursive_wrap.<locals>.<lambda>zdummy(relu>>relu)r   dummy(relu)rw   )r   r   r   r$   rI   )r  r  chained_debugr   r  r   test_recursive_wrap%  s   
"r"  c                     s   dd  t d} t| | }t||| }t| fdd}d}| D ]}|jdr.|d7 }q"|d	ks5J |jd jd jd jd jd
ksHJ |jd jd jd jd jd
ks[J |jd jd jd jd jd
ksnJ |jd jd jd jd jd
ksJ |jd jd jd
ksJ d S )Nc                 S   r  )Nc                 S   r  r  r   r  r   r   r   r   >  r   zAtest_recursive_double_wrap.<locals>.dummy_model.<locals>.<lambda>r  r"   r  r   r   r   r  =  r	  z/test_recursive_double_wrap.<locals>.dummy_modelrK   c                    r  r  r#   rt   r  r   r   r   D  r  z,test_recursive_double_wrap.<locals>.<lambda>r   dummyrw   r{   r   r   )r   r   r   r   walkr$   
startswithrI   )r  r  concatconcat_wrappedn_debugrg   r   r  r   test_recursive_double_wrap<  s$   
&&&&r)  c                  C   s>   t d} t d}t| | }t||}|d|  t|t d S )NrK   r  )r   r   rd   r   r   )r  r   r  chained2r   r   r   test_wrap_non_child_referencesY  s   

r+  c                  C   sh   t d} t d}t| |}t||}t|jdd||| |gks"J t|jdd| |||gks2J d S )NrK   dfs_preorderdfs_post)r   r   r   r$  )r  r   inner_chainr  r   r   r   test_walk_dfsc  s   

 r1  c                  C   sH   t d} tjtdd | jdd W d    d S 1 sw   Y  d S )NrK   zInvalid order)matchdfs_post_orderr-  )r   rS   rT   r^   r$  )r  r   r   r   test_walk_bfs_post_order_failsq  s   "r4  c               
   C   s  G dd dt } td}| d }tddd dddtd	d d
|d dddi|g||gd}tddd dddtd	d d
|d dddi|g||gd}td}t||||}| }||	  |j
dkskJ |jd |jd kswJ t|jd jd t|jd jd ksJ d S )Nc                   @   s    e Zd ZdZdd Zdd ZdS )z)test_model_copy_with_loop.<locals>.MyShimr/   c                 S   s   t S r   )r  )selfr   r   r   r   {     z2test_model_copy_with_loop.<locals>.MyShim.to_bytesc                 S   s   d S r   r   )r5  bytesr   r   r   
from_bytes~  r6  z4test_model_copy_with_loop.<locals>.MyShim.from_bytesN)r1   r2   r3   r$   r   r8  r   r   r   r   r4   x  s    r4   r6   r7   c                 S   r   )Nc                 S   r   r   r   r   r   r   r   r     r   =test_model_copy_with_loop.<locals>.<lambda>.<locals>.<lambda>r   r   r   r   r   r     r!   z+test_model_copy_with_loop.<locals>.<lambda>rK   r9   r<   r=   r@   rA   rB   rC   test2c                 S   r   )Nc                 S   r   r   r   r   r   r   r   r     r   r9  r   r   r   r   r   r     r!   ztest>>relu>>test>>test2r   r   r{   )r	   r%   r   rN   rO   r   r   re   	from_dictto_dictr$   rI   r*   rH   )r4   rf   working_shimlayerlayer2r  rg   r,   r   r   r   test_model_copy_with_loopw  s<   	

0r@  )Ar   r   collectionsr   rN   rS   	thinc.apir   r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   thinc.compatr   utilr   fixturer   r%   r(   r-   ri   rm   ru   rz   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   markparametrizesplitr   r   skipifr   r   r  r  r"  r)  r+  r1  r4  r@  r   r   r   r   <module>   sZ    D
A 		
Q
&!
