o
    ir                     @   s  d dl Z d dlZd dlmZmZ d dlZd dlZd dlmZm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dlmZmZmZ d d	lmZm Z m!Z! d d
l"m#Z# d dl$m%Z%m&Z& ddl'm(Z( ddl(m)Z)m*Z* dZ+eeZ,e Z-eddZ.e-e,gZ/e-gZ0ere01e  e0e,g Z2ddgZ3ddgZ4g dZ5g dZ6dd Z7e re7 Z8ng Z8ej9:deegdd Z;ej9:de2ej9<ddd  Z=ej9:de2d!d" Z>ej9:de0d#d$ Z?ej9:de0d%d& Z@ej9:de0d'd( ZAej9:de2ej9:d)e3ej9:d*dd+gd,d- ZBej9:de0ee() d.d/d0 ZCej9:de2d1d2 ZDej9:de/d3d4 ZEej9:de2ej9:d)e3e	e+dd5ee(F d6d7d8 ZGej9:de2ej9:d)e3d9d: ZHej9:de2e	e+dd5ee() d6d;d< ZIej9:de0ej9:d)e3d=d> ZJej9:de0ej9:d)e3d?d@ ZKej9:de0ej9:d)e3dAdB ZLej9:de0ej9:d)e3dCdD ZMej9:de0ej9:d)e3dEdF ZNej9:de0ej9:d)e3dGdH ZOej9:de2ej9:d)e3e	e+dd5ee() d6dIdJ ZPej9:de0ej9:d)e3dKdL ZQej9:de0ej9:d)e3dMdN ZRej9:de0ej9:d)e3dOdP ZSej9:de0ej9:d)e3dQdR ZTej9jUe dSdTej9:dUdVdgdWdX ZVej9jUe dSdTej9:dUdVdgdYdZ ZWej9:de2ej9:d)e3e	e+dd5ee() d6d[d\ ZXej9:de2e	e+dd5ee() d6d]d^ ZYej9:de2e	e+dd5ee() d6d_d` ZZdae#dbe#dce[ddee#e#f fdedfZ\ej9jUe  dgdTej9:de2ej9:dcg dhdidj Z]ej9:dkg e/e.dldm Z^ej9:dkg e/e.dndo Z_ej9:dke/e	e+dd5ee() d6dpdq Z`ej9:de2ej9:d)e3e4 drds Zaej9:de2ej9:d)e3dtdu Zbej9:de2ej9:d)e3dvdw Zcej9:de2ej9:d)e3dxdy Zdej9:de2ej9:d)e3dzd{ Zeej9:de2ej9:d)e3d|d} Zfej9:de2ej9:d)e3d~d Zgej9:de2ej9:d)e3dd Zhej9:de2ej9:d)e3dd Ziej9:de2ej9:d)e3dd Zjej9:de2ej9:d)e3dd Zkej9:de2ej9:d)e3dd Zlej9:de2ej9:d)e3ej9:de5dd Zmej9:de2ej9:d)e3ej9:de5dd Znej9:de2ej9:d)e3ej9:de6dd Zoej9:de2e	e+dd5ee() d6dd Zpej9:de0ej9:d)e3ej9:dg ddd Zqej9:de2e	e+dd5ee() d6dd Zrej9:de2e	e+dd5ee() d6dd Zsej9:de2e	e+dd5ee() d6dd Ztej9:de2e	e+dd5ee() d6dd Zuej9:de2e	e+dd5ee() d6dd Zvej9:de2e	e+dd5ee() d6dd Zwej9:de2e	e+dd5ee() d6dd Zxej9:de2e	e+dd5ee() d6dd Zyej9:de2e	e+dd5ee() d6dd Zzej9:de2e	e+dd5ee() d6dd Z{dddZ|edd Z}ej9:de0ej9:dg ddd Z~ej9jddTej9:de0e	e+dd5ee} ddd Zdd Zdd Zdd Zdd Zdd Zej9jUe  dgdTej9jUe!edk ddTej9:de2ej9:d)ddgej9:de8e	e+dd5ee(jdddƍe(jddVdƍdȍddʄ Zej9:de2e	e+dd5ee(jdddƍd̍dd΄ Zej9:de2ej9:ddСe	e+dd5ee(jdddƍd̍dd҄ Zej9jUe dSdTddԄ Zej9:de2ddք ZdS )    N)Tuplecast)givensettings)	compositeintegers)assert_allclose)Version)LSTMCupyOpsNumpyOpsOpsfix_random_seedget_current_opsget_opsuse_ops)KERNELSKERNELS_LISTcompile_mmh)has_cupy_gpu	has_torchtorch_version)Floats2d)torch2xpxp2torch   )
strategies)	arrays_BIndarrays_of_shape
   T)use_blisfloat32float64int32int64)reduce_firstreduce_last
reduce_maxreduce_mean
reduce_sum))r%   T)r&   T)r'   T)r(   F)r)   Fc                     s   dd l  dd lfdd} fdd}fddfdd	}fd
d}fdd}fdd}fdd}fdd}dd } fdd}	fdd}
d| fd|fdfd|fd|fd|fd |fd!|fd"|fd#|	fd$|
fd%|fgS )&Nr   c                        j j| S N)nn
functionalreluxtorch Q/home/ubuntu/.local/lib/python3.10/site-packages/thinc/tests/backends/test_ops.py
torch_relu@      z(create_pytorch_funcs.<locals>.torch_reluc                    r*   r+   )r,   r-   relu6r/   r1   r3   r4   torch_relu_kC   r6   z*create_pytorch_funcs.<locals>.torch_relu_kc                    s     | d d ddS )N皙?      ?r      )clipr/   r1   r3   r4   torch_hard_sigmoidF   s   z0create_pytorch_funcs.<locals>.torch_hard_sigmoidc                    r*   r+   )r,   r-   hardtanhr/   r1   r3   r4   torch_hard_tanhI   r6   z-create_pytorch_funcs.<locals>.torch_hard_tanhc                    r*   r+   )r,   r-   mishr/   r1   r3   r4   
torch_mishL   r6   z(create_pytorch_funcs.<locals>.torch_mishc                    r*   r+   )r,   r-   silur/   r1   r3   r4   torch_swishO   r6   z)create_pytorch_funcs.<locals>.torch_swishc                    s   |  |  S r+   r3   r/   )r=   r3   r4   torch_hard_swishR   s   z.create_pytorch_funcs.<locals>.torch_hard_swishc                    r*   r+   )r,   r-   	hardswishr/   r1   r3   r4   torch_hard_swish_mobilenetU   r6   z8create_pytorch_funcs.<locals>.torch_hard_swish_mobilenetc                    s
     | S r+   )sigmoidr/   r1   r3   r4   torch_sigmoidX   s   
z+create_pytorch_funcs.<locals>.torch_sigmoidc                 S   s    d|  | d| |      d  S )Nr:   r;   )sqrtr/   r3   r3   r4   
torch_dish[   s    z(create_pytorch_funcs.<locals>.torch_dishc                    s6   d|  d  d j | d| d     S )Nr:         ?       @gHm?      @)tanhrI   pipowr/   mathr2   r3   r4   torch_gelu_approx_   s   "z/create_pytorch_funcs.<locals>.torch_gelu_approxc                    r*   r+   )r,   r-   gelur/   r1   r3   r4   
torch_geluk   r6   z(create_pytorch_funcs.<locals>.torch_gelur.   relu_khard_sigmoid	hard_tanhr@   swish
hard_swishhard_swish_mobilenetdishgelu_approxrT   rG   rQ   )r5   r8   r?   rA   rC   rD   rF   rH   rJ   rS   rU   r3   )rR   r2   r=   r4   create_pytorch_funcs;   s6   r^   opc                 C   sZ  dd t | D }|D ]}tt|sJ t| |}t|drt|}dd |jD dd }ttt|}t| dd |jD dd }||ksOJ |dd |j D dd }d	d |j D dd }	||	kssJ |d
d |j D dd }
dd |j D dd }t	t
|
|D ]\}\}}|tjjkr|tjjkr	 qqdS )zTest that specific ops don't define any methods that are not on the
    Ops base class and that all ops methods define the exact same arguments.c                 S   s   g | ]	}| d s|qS )_)
startswith).0mr3   r3   r4   
<listcomp>   s    z(test_ops_consistency.<locals>.<listcomp>__call__c                 S      g | ]}|qS r3   r3   rb   pr3   r3   r4   rd          r;   Nc                 S   rf   r3   r3   rg   r3   r3   r4   rd      ri   c                 S      g | ]}|j qS r3   defaultrg   r3   r3   r4   rd          c                 S   rj   r3   rk   rg   r3   r3   r4   rd      rm   c                 S   rj   r3   
annotationrg   r3   r3   r4   rd      rm   c                 S   rj   r3   rn   rg   r3   r3   r4   rd      rm   )dirhasattrr   getattrinspect	signature
parametersprintvalues	enumeratezip	Parameterempty)r_   attrsattrmethodsigparamsbase_sigbase_paramsdefaultsbase_defaultsannotsbase_annotsip1p2r3   r3   r4   test_ops_consistency   s*   


r   opszignore::RuntimeWarningc                 C   sF  | j jddd}| j jddd}| ||||dddd tt | ||||dddd W d    n1 s8w   Y  tt | ||||dddd W d    n1 sYw   Y  tt | ||||dddd W d    n1 szw   Y  tt | ||||dddd W d    d S 1 sw   Y  d S )Nr;   fdtyper           )xpzerosadampytestraises
ValueError)r   onetwor3   r3   r4   test_adam_incorrect_inputs   s   "r   c                 C   s(  | j | j| j| jf}t|D ]5\}}d|d  }|| }|jtjks$J |jt	|ks-J | 
|}|jtjks:J |jt	|ksCJ q| j| j| j| jf}t|D ]5\}}d|d  }|| }|jtjkshJ |jt	|ksqJ | |}|jtjks~J |jt	|ksJ qR| djdksJ d S )N)r;   r;   )alloc1falloc2falloc3falloc4frx   r   numpyr!   ndimlenalloc_falloc1ialloc2ialloc3ialloc4ir#   alloc_ialloc)r   float_methodsr   r~   shapearrint_methodsr3   r3   r4   
test_alloc   s&   

r   c                 C   st   | j ddd}| |d}|jdksJ |jdksJ tt|D ]}t|jd D ]}|||f dks6J q*q!d S )N   uint64r   r   )r      uint32r;   )r   hashr   r   ranger   )r   idskeysr   jr3   r3   r4   test_hash_gives_distinct_keys   s   r   c                 C   sL   d}d}|  ||}|dkr||dk  sJ d S ||dk  s$J d S )Nr   r   r   rK   )get_dropout_maskallr   r   dropmaskr3   r3   r4   test_get_dropout_empty   s   r   c                 C   sF   d}d}|  ||}|dk sJ |dk sJ |j|ks!J d S )N)   r   r:   rK   r   )r   anyr   r   r3   r3   r4   test_get_dropout_not_empty   s   r   r   index_dtyper   c                 C   sl   | j jd|ddd}| j jddgddgddgg|d}| ||}| j j|g dg d	g d
g d S )N   r   r      r   r   r;   )      @       @      $@)      (@g      ,@      0@)rM         @      @)r   arangereshapearray
gather_addtestingr   )r   r   r   tableindicesgatheredr3   r3   r4   test_gather_add   s   "r   )r   c                 C   sV   |  |}| jjddddd|jd  }| jjj| |||| ddd	 d S )
Nd   r   r      r   r   r;   h㈵>atol)	asarrayr   r   r   r   r   r   r   sumr   r   r   r3   r3   r4   test_gather_add_against_numpy   s   
"

r   c                 C   sv   | j jddddd}| j jddgddgd	dggd
d}tt | || W d    d S 1 s4w   Y  d S )Nr   r   r   r   r   r   r   r;   r   r   )r   r   r   r   r   r   
IndexErrorr   r   r3   r3   r4   test_gather_add_oob_raises  s
   ""r   c                 C   s   | j dgdgdgdggdd}| |d}t|dr| }t|d	 g d
 t|d g d t|d g d t|d g d d S )NrK   rM         @r   r!   r   r;   getr   )r   rK   rM   )rK   rM   r   r   )rM   r   r   r   )r   r   r   )r   seq2colrq   r   r   )r   seqcolsr3   r3   r4   test_seq2col_window_one_small
  s   
r   )max_examplesdeadlineXc                 C   s   | j ||d}|jdd|}| |\}}|j|ksJ | jjj||ddd | jjj| jj|| j	|ddd| j	|ddd d S )Nr   axisMbP?rtolr   g|=r   )
r   maxastypemaxoutr   r   r   r   take_along_axisexpand_dims)r   r   r   expected_bestpredicted_bestwhichr3   r3   r4   test_maxout  s   
r   c              	   C   s   |  | jddgddgg|d| ddgddggd	}|j|ks"J | jj|g d
g dgg dg dgg tt	" |  | ddgddgg| ddgd	dggd	 W d    d S 1 sbw   Y  d S )NrK   rL   rM   r   r   r;   r   r   r   )r   rK   r   rL   r   r   r   r   rM   )r   r   r   )
backprop_maxout	asarray2f	asarray2ir   r   r   r   r   r   r   r   r   dXr3   r3   r4   test_backprop_maxout-  s   *"r   c                 C   sd   |  |}t }| j|_||j| }|j| |dd}| j|dd}| jjj||ddd d S )Nr;   nWr   r   r   )r   r   r   r   r   r   r   r   )r   r   base_opsbaseXtarget	predictedr3   r3   r4   test_seq2col_window_oneA  s   
r  c              
   C   s`  | j j| jd|d| j| jd|dd| j jdddd | j j| jd|d| j| jd|dd| j jdddd | j j| jd|d| j| jd|dd| dgd | j j| jd|d| j| jd|dd| dgd | j j| jd|d| j| jd|dd| ddgd | j j| jd|d| j| jd|dd| ddgd d S )N)r   r   r   r;   )r   r#   lengthsr   )r   r   r   r   r   r   backprop_seq2col	asarray1ir   r   r3   r3   r4   test_seq2col_lengths_all_zeroN  s@    "r  c              
   C   s   | j g dg dg dg dg dg|d}| j g dg dg d	g d
g dg|d}| jj|| j| jjdd|dddd| ddgd | jj|| j|d| ddgd | jj|| j| jjdd|dddd| ddgd d S )N	r   r   r   r;   r   r   r   r      	r;   r   r   r   r   r        	   	r   r   r  r  r  r  r      r   	r  r  r  r   r  r            	r   r  r   r  r  r  r   r   r   r   r   r   r  r   r                 !   $         r"  rK   r   r   r   r;   r   r  	r   r   r   r   r   r   r   r
  r	  r   r   
cols_check
grad_checkr3   r3   r4   $test_seq2col_lengths_zero_first_last{  sH   "	
r,  c              
   C   s8  | j g dg dg dg dg dg dg dg|d}| j g d	g d
g dg dg dg dg dg|d}| jj|| j| jjdd|dddd| g dd | jj|| j|d| g dd | jj|| j| jjdd|dddd| g dd | jj|| j|d| g dd d S )Nr  r  r  r  r  )	r   r   r         r        r  )	r-  r.  r  r/  r0  r  r   r   r   r   r  r  r  r!  r%  )    "   r$  )&   (   *   rK   g      6@r  r   r;   )r   r   r   r  )r   r   r   r   r(  r)  r3   r3   r4   !test_seq2col_lengths_zero_between  sl   			
	r6  c              	   C   t   | j jdd|ddd}| g d}| j|d|d}| j j| jg d	g d
g dg dg dg|d| d S )NrK   r   r   r   r   r;   r   r;   r;   r  )	r   r   r   r;   r   r   r   r   r   )	r   r   r   r   r   r  r  r  r  r  )	r  r  r  r   r  r   r   r   r   )	r   r   r   r  r  r  r   r   r   r   r   r   r
  r   r   r   r   r   r   r   r  r   r3   r3   r4   test_seq2col_window_one_lengths     
r;  c              	   C   r7  )NrK   r   r   r   r   r8  r   r  )r   r   r   r   r   r   r;   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r  r  r  r  r   r  r   )r   r   r   r   r   r  r  r  r  r   r  r   r   r   r   )r   r   r  r  r  r  r   r  r   r   r   r   r   r   r   )r   r   r   r   r   r   r  r  r  r   r   r   r   r   r   r9  r:  r3   r3   r4   test_seq2col_window_two_lengths  r<  r=  c                 C   sd   | j g dg dg dg|d}dgdgdgg}| |d}t|tjs(| }t||d	d	d
 d S )N)r   r   r   )      r   rK   r   r   r>  rL   rK   r;   r   r  )r   r	  
isinstancer   ndarrayr   r   r   r   r   expectedr   r3   r3   r4   &test_backprop_seq2col_window_one_small  s   rC  c                 C   s   |j d d r	d S | j||d}| j| dkrd S t }| j|_|j|dd}| j|dd}t|j d D ]'}||  ||   }|dk sN|dkr_t	|| t	||  t	||  q8| jj
j||d	d	d
 d S )Nr;   r   r   r"  r   r   g皙?r   r  )r   r   r   absr   r   r	  r   r   rv   r   r   )r   r   r   r  r  r  rowdiffr3   r3   r4    test_backprop_seq2col_window_one&  s"   
rH  c                 C   sz   | j jddd|ddd}| g d}| j|d|d}| j jj| jg d	g d
g dg dg dg|d|dd d S )NrD  gffffff@stepr   r   r  r8  r;   r  )g?r:   g333333?)g	@g333333@g@)gffffff@皙@g@)gffffff@g333333@r   )r   gffffff@g@r   ư>r   r   r   r   r
  r	  r   r   r   r   r   d_yr  d_seqsr3   r3   r4   (test_backprop_seq2col_window_one_lengths=  s    

rQ  c                 C   s   | j dgdgdgdgg|d}| |d}t|tjs| }t|d g d t|d	 g d
 t|d g d t|d g d d S )NrK   rL   rM   r   r   r   r   r   r   rK   rL   rM   r;   r   rK   rL   rM   r   rK   rL   rM   r   r   r   rL   rM   r   r   r   )r   r   r?  r   r@  r   r   )r   r   r   r   r3   r3   r4   test_seq2col_window_twoT  s   rV  c              	   C   sv   | j jddd|ddd}| g d}| j|d|d}| j j| jg d	g d
g dg dg dg|d| d S )NrD  gffffff@rI  r   r  r8  r   r  )gffffff?g?g?)gffffff$@g      %@g%@)g333333&@g&@gffffff'@)r   g(@g333333)@)g@g333333@rK  r   rM  rN  r3   r3   r4   (test_backprop_seq2col_window_two_lengthsa  s   
rW  c                 C   sj   | j g dg dg dg dg|d}| j dgdgdgdgg|d}| |d	}| jjj||d
d
d d S )NrR  rS  rT  rU  r   rM   r   r   r   r   r  )r   r	  r   r   r   rA  r3   r3   r4    test_backprop_seq2col_window_twow  s$   	rX  zneeds GPU/CuPy)reasonr   r;   c           
      C   s   t  }t }d}|jj|d ddd}||}|g d|d  }||}|j	|| |d}|j	|| |d}	t
||	  d S )N   r   r!   r   r;   r   r   r;   r  r   r  )r   r   r   randomrandnr   r   r   r
  r   r   r   )
r   cupy_ops	numpy_ops
batch_sizer   X_gpur  lengths_gpur   cols_gpur3   r3   r4   "test_large_seq2col_gpu_against_cpu  s    

re  c                 C   s   t  }t }d}d|  d }|jj|| dd|}||}|g d|d  }||}|j	|| |d}	|j	|| |d}
t
|	|
  d S )	NrZ  r   r;   r!   r   r[  r  r\  )r   r   r   r]  r^  r   r   r   r
  r	  r   r   )r   r_  r`  ra  nFd_cols
d_cols_gpur  rc  rP  
d_seqs_gpur3   r3   r4   +test_large_backprop_seq2col_gpu_against_cpu  s   

rj  c                 C   s   | j ||d}| j| dkrd S | j dgt| dd}| ||}|j|ks,J |jt||jd fks:J d}t	|D ]!\}}| jj
j||||  jdd|| | ddd	 ||7 }q@d S )
Nr   r   r   r   r;   r   r   g{Gz?r   )r   r   rE  r   r   backprop_reduce_sumr   r   r   rx   r   r   )r   r   r   r  outstartr   lengthr3   r3   r4   test_backprop_reduce_sum  s   $
ro  c                 C   s>   |  | |}|D ]}d|   krdksJ  J q
d S )NwJ?gqh ?)softmaxr   r   )r   r   yrF  r3   r3   r4   test_softmax_sums_to_one  s   "rs  c                 C   sF   |  |}| j|dd}|D ]}d|   krdks J  J qd S )NTinplacerp  grZ|
 ?)r   rq  r   )r   r   rF  r3   r3   r4   test_softmax_works_inplace  s
   
"rv  r   dYtemperaturereturnc                 C   sf   dd l }t| dd}t|}|| }|jjj|dd}|| ttt|tttt|j	|j
fS )Nr   Trequires_gradr   )dim)r2   r   r,   r-   rq  backwardr   r   r   Tensorgrad)r   rw  rx  r2   XtdYtXt_tempYtr3   r3   r4   torch_softmax_with_temperature  s   
r  zneeds PyTorch)r:   rK   rL   c                 C   s   | j jddddddd}| j jddd}| j||d}| j|||d}t|||\}}| j jj||dd | j jj||dd d S )	Nr   r9   r   r   )rx  rL  r   )	r   r   r   eyerq  backprop_softmaxr  r   r   )r   rx  r   rw  Yr   r  dXtr3   r3   r4   test_softmax_temperature  s   r  cpu_opsc                 C   s  t jddd}t jddd}|t jj|jd|j7 }|t jj|jd|j7 }| j||dd}t ||j	}t
||ddd	 t jd
dd}t jddd}|t jj|jd|j7 }|t jj|jd|j7 }| j||dd}t |j	|}t
||ddd	 | j||d|d d S )N)r   r   r   r   )r   r   )sizeT)trans2-C6?r  )r   r   )r   r   )trans1)r  rl  )r   r   r]  uniformr  r   r   gemmdotTr   )r  Wr   r  rB  r3   r3   r4   test_gemm_computes_correctly  s   r  c                 C   sJ   t jddd }}t jddd}| j|||d t |t ds#J d S )Nr   r   r   )rl  )r   r   onesr  array_equal)r  abcr3   r3   r4   test_gemm_out_used  s   r  c                 C   s   |  dd |D }|jdksJ | |dd |D }t|| | j dd |D ddd}t|t|ks8J | j|dd |D dd	}t|| d S )
Nc                 S   rf   r3   r3   rb   r0   r3   r3   r4   rd   +  ri   z4test_flatten_unflatten_roundtrip.<locals>.<listcomp>r;   c                 S      g | ]}t |qS r3   r   r  r3   r3   r4   rd   -      c                 S   rf   r3   r3   r  r3   r3   r4   rd   /  ri   r   )padr   c                 S   r  r3   r  r  r3   r3   r4   rd   1  r  )r  )flattenr   	unflattenr   r   )r  r   flatunflatflat2unflat2r3   r3   r4    test_flatten_unflatten_roundtrip'  s   
r  c              
   C   s  | j jdd|d| j jdd|dg}| j j| |g dg dg | j j| j|ddg d	g d
g | j jdd|ddd| j jdd|dddg}| j j| |ddgddgddgddggddgddgddgddggg | j j| j|ddddgddgddgddgddggddgddgddgddgddggg tjtdd | j|dd W d    d S 1 sw   Y  d S )Nr;   r   r   r   )r;   r   r   r   r;   r   r   r   r  )round_to)r;   r   r   r   r   r   r   r   )r;   r   r   r   r   r   r   r   r   r  r   r   r  r  z'Rounding for padding must at least be 1match)	r   r   r   r   r  r   r   r   r   )r   r   r   r3   r3   r4   test_pad5  s.   $"  "r  c                 C   s(  | j ddgddgddgddgddgg|d}| dd	g}| jj| ||d
dgddgg | g d}| jj| ||d
dgddgddgg tt | || jj	g ddd W d    n1 siw   Y  tt
 | || jj	g ddd W d    d S 1 sw   Y  d S )NrK   rL   rM   r   r   r   r   r   r         "@r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r
  r   r   r   r)   r   r   r   r   r   r   r   r   r  r3   r3   r4   test_reduce_sumV  s"   ""r  c                 C   sf   t jtdd" | | jjdd|ddd| jjddgd	d W d    d S 1 s,w   Y  d S )
Nlengths must ber  r;   r  r   r   r   r   r#   )r   r   r   rk  r   r   r   r   r  r3   r3   r4   )test_backprop_fails_with_incorrect_lengthn  s   "r  c                 C   s  | j ddgddgddgddgd	d
gg|d}| ddg}| ||\}}| jj|| g d | jj|ddgddgg | g d}tjt	dd | 
|| W d    n1 s_w   Y  | g d}tjtdd | 
|| W d    d S 1 sw   Y  d S )NrK   r   rL   r   rM   r   r   r  r   r   r   r   r   r   r   r   r   all sequence lengths must be > 0r  r   r   r;   )lengths must sum up to the number of rows)r   r
  r%   r   r   assert_array_equalr   r   r   r   r&   r   )r   r   r   r  r  starts_endsr3   r3   r4   test_reduce_firstx     ""r  c              	   C   sf   | j ddgddgg|d}| g d}| ||}| jj|ddgddgddgddgddgg d S )NrK   rM   rL   r   r   r  r   )r   r
  backprop_reduce_firstr   r   r   )r   r   rw  r  r   r3   r3   r4   test_backprop_reduce_first     "r  c                 C   s  | j ddgddgddgddgd	d
gg|d}| ddg}| ||\}}| jj|| ddg | jj|ddgd	d
gg | g d}tjt	dd | || W d    n1 s_w   Y  | g d}tjt
dd | || W d    d S 1 sw   Y  d S )NrK   r   rL   r   rM   r   r   r  r   r   r   r   r   r   r  r  r  r  r  )r   r
  r&   r   r   r  r   r   r   r   r   )r   r   r   r  r  lastsr3   r3   r4   test_reduce_last  r  r  c              	   C   sf   | j ddgddgg|d}| ddg}| ||}| jj|ddgddgddgddgddgg d S )	NrK   rM   rL   r   r   r   r   r   )r   r
  backprop_reduce_lastr   r   r   )r   r   rw  r  r   r3   r3   r4   test_backprop_reduce_last  r  r  c           
      C   s   | j jd|d}|| j jdd|j7 }| j jg ddd}| ||\}}|j|ks-J | j |dks7J | j ||jd k sDJ d}t	|D ]\}}||||  j
dd}	| j j|| |	 ||7 }qJd S )	N)r  r   r   r   r;   )r   r   r   r   r   r   )r   r   r]  r  r   r   r'   r   r   rx   r   r   r   )
r   r   r   r  maxesr   rm  r   rn  truthr3   r3   r4   test_reduce_max_sm  s   
r  c           
      C   s  | j jd|d}|| j jdd|j7 }| j jg ddd}| ||\}}|j|ks-J | j |dks7J | j ||jd k sDJ d}t	|D ]\}}||||  j
dd}	| j j|| |	 ||7 }qJtt | || j jg d	dd W d    n1 sw   Y  tt | || j jg d
dd W d    n1 sw   Y  tt | || j jg ddd W d    d S 1 sw   Y  d S )N)r/  r   r   r   r;   )r   r   r   r  r   r   r   r  r  )r   r   r   r   r  )r   r   r]  r  r   r   r'   r   r   rx   r   r   r   r   r   r   r   )
r   r   rc   r  r  r   rm  r   rn  r  r3   r3   r4   test_reduce_max  s*   
"r  c              
   C   s  |  | jjdd|ddd| jg dg dgd| jjddgdd}|j|ks.J | jj|g d	g d
g dg dg dg t	
t/ |  | jjdddddd| jg dg dgd| jjddgdd W d    n1 s|w   Y  t	
t/ |  | jjdd|ddd| jg dg dgd| jjddgdd W d    n1 sw   Y  t	
t3 |  | jjdd|ddd| jg dg dg dgd| jjg ddd W d    d S 1 sw   Y  d S )Nr;   r  r   r   r   )r   r;   r   )r;   r   r;   r#   r   )r   rL   r   )rK   r   r   )r   r   r   )r   r   r   r   )r   r   r   r  )backprop_reduce_maxr   r   r   r   r   r   r   r   r   r   r   r   r   r3   r3   r4   test_backprop_reduce_max  sH   ""r  c                 C   sz  | j ddgddgddgddgddgg|d}| dd	g}| jj| ||ddgddgg | g d
}| jj| ||ddgddgddgg | j ddgddgddgg|d}| ddg}| jj| ||ddgddgg tt | || jj	ddgdd W d    n1 sw   Y  tt
 | || jj	ddgdd W d    d S 1 sw   Y  d S )NrK   rL   rM   r   r   r   r   r   r   r  r   r   r   r   r   )r   r
  r   r   r   r(   r   r   r   r   r   r  r3   r3   r4   test_reduce_mean  s,   " "r  c              
   C   s   |  | jjdd|ddd| jjddgdd}|j|ks J | jj|g dg dg dg dg d	g d	g tj	t
d
d" |  | jjdd|ddd| jjddgdd W d    d S 1 sfw   Y  d S )Nr;   r  r   r   r   r   r#   )g      ?r:   g      ?)rL   g      @rM   r  r  r   )backprop_reduce_meanr   r   r   r   r   r   r   r   r   r   r   r3   r3   r4   test_backprop_reduce_mean/  s*   "r  	reductionc           
      C   s   t | |}t | d| }| g }|| jd|d|}|dkr+|\}}||||}nt|tr:|\}}	|||	}n|||}|jdksFJ |jdksMJ d S )N	backprop_)r   r   r   r'   rr   r
  r   r?  tupler   
r   r   r  funcbackprop_funcr  r  r   r   extrar3   r3   r4   test_reduce_empty_batchJ  s   



r  c           
      C   s   t | |}t | d| }| ddg}|| jd|d|}|dkr-|\}}||||}nt|tr<|\}}	|||	}n|||}|jdksHJ |jdksOJ d S )Nr  r   r   )r   r   r   r'   )r   r   r  r  r3   r3   r4   test_reduce_empty_hiddena  s   


r  reduction_raisesc                 C   s   |\}}t | |}| jddgddgddgddgddgg|d}| g d}|rEtt ||| W d    d S 1 s>w   Y  d S | jj|||d	 d
d
g d S )NrK   rL   rM   r   r   r   r   r  r;   r   )	rr   r   r
  r   r   r   r   r   r   )r   r   r  reduction_strr   r  r   r  r3   r3   r4   test_reduce_zero_seq_lengthx  s   
"""r  c                 C   <   |  |}| |}|j|jksJ | j| rJ d S r+   )r   r@   r   r   isnanr   r   r   r  r3   r3   r4   	test_mish     

r  )backprop_clipped_linearbackprop_dishbackprop_gelubackprop_gelu_approxbackprop_hard_sigmoidbackprop_hard_swishbackprop_hard_swish_mobilenetbackprop_hard_tanhbackprop_mishbackprop_relubackprop_relu_kr  backprop_swishc              	   C   sB  t | |}dd t|j D }t|dkrutt  || j	j
d|d| j	j
d|d| j	j
d|d W d    n1 s@w   Y  tt! || j	j
d|d| j	j
d|d| j	j
d|d W d    d S 1 snw   Y  d S tt || j	jdd|d| j	jddd	|d W d    d S 1 sw   Y  d S )
Nc                 S   s   g | ]}|j tjjkr|qS r3   )rl   rs   rz   r{   rg   r3   r3   r4   rd     s
    zBtest_eltwise_backprop_rejects_incorrect_shapes.<locals>.<listcomp>r   r   r   r   r  r   )rr   rs   rt   ru   rw   r   r   r   r   r   r   r   )r   r   r_   backproppositional_argsr3   r3   r4   .test_eltwise_backprop_rejects_incorrect_shapes  s2   
""r  c                 C   h   |  |}| |}|j|jksJ | j| rJ |dk |jks'J |dk |jks2J d S )Nr   r   )r   rV   r   r   r  r   r   r  r  r3   r3   r4   test_relu_k     

r  c                 C   r  r+   )r   rY   r   r   r  r   r  r3   r3   r4   
test_swish  r  r  c                 C   r  )Nr   rK   )r   rW   r   r   r  r   r   r  r  r3   r3   r4   test_hard_sigmoid  r  r  c                 C   r  )Nr>  rK   )r   rX   r   r   r  r   r   r  r  r3   r3   r4   test_hard_tanh  r  r  c                 C   r  r+   )r   rZ   r   r   r  r   r  r3   r3   r4   test_hard_swish  r  r  c                 C   r  r+   )r   r[   r   r   r  r   r  r3   r3   r4   test_hard_swish_mobilenet  r  r  c                 C   r  r+   )r   r]   r   r   r  r   r  r3   r3   r4   test_gelu_approx  r  r  c                 C   r  r+   )r   r\   r   r   r  r   r  r3   r3   r4   	test_dish  r  r  c                 C   r  r+   )r   rT   r   r   r  r   r  r3   r3   r4   	test_gelu  r  r  c                 C   sF   |  |}| |j}| ||}|j|jksJ |dk s!J d S )Nr   )r   r   r   r  r   )r   r   r   r   r3   r3   r4   test_backprop_mish%  s
   
r  c                 C   s  |dkr7|d | |d  |d |  |d  }t d| D ]}||d | |d  |d |  |d  7 }qn:|d | |d  |d |d   |d  }t d| D ]}||d | |d  |d |d   |d  7 }qR|d9 }t||| |dkd }|dj|ksJ |r|t|}	tj|fdd}
|tt|
	 |f}ntj|fdd	}	tj|fdd}
t
t|
	 |f}t
| ||| f}t
| ||| f}|	||||
fS )
Nr;   r   r   )depthbir
   r#   )r   r   r   r   )r   r
   
initialize	get_paramr  r   r   r  intr   r   )r  dirsnOra  nIdrawn_paramsr`   lstmr   	size_at_tr   H0C0r3   r3   r4   get_lstm_args1  s*   (.,2r
  c                 C   s^   | t dd}| t dd}| t dd| }| t dd}| t dd}t|||||| dS )Nr;   r   r   r-  r  )r  )r   r
  )r  r  r  r  ra  r  r3   r3   r4   draw_lstm_argsM  s   r  zdepth,dirs,nO,batch_size,nI))r;   r;   r;   r;   r;   )r;   r;   r   r;   r;   )r;   r;   r   r;   r   )r   r;   r;   r;   r;   )r   r;   r   r   r   )r;   r   r   r;   r;   )r   r   r   r   r   c                 C   s   t  }t|||||\}}}	}
}||||	|
|}| |||	|
|\}}t|d |d d ddd t|d |d d ddd t||d ddd d S Nr   r;   r  r   r  r   )r   r
  lstm_forward_trainingr   )r   r  r  r  ra  r  reference_opsr   r  r	  r   r  	referencer  	fwd_stater3   r3   r4   test_lstm_forward_trainingW  s   r  zFlaky, skip temporarily)argsc                 C   s   |\}}}}}t  }||||||}| |||||\}	}
t|
d |d d ddd t|
d |d d ddd t|	|d ddd d S r  )r   r  r   )r   r  r   r  r	  r   r  r  r  r  r  r3   r3   r4   test_lstm_forward_training_fuzzn  s   r  c                  C   s   t tdts	J t tdtsJ zddlm}  t td| s"J W n ty5   t tdts3J Y nw zddlm} t td|sFJ W n tyY   t tdtsWJ Y nw t	
t td W d    n1 snw   Y  tt}|jtks~J d S )Nr   cupyr   )AppleOpscpu)BigEndianOpsblah)r?  r   r   r   thinc_apple_opsr  ImportErrorthinc_bigendian_opsr  r   r   r   r   r   r   )r  r  r   r3   r3   r4   test_get_ops|  s&   
r  c                  C   s   t  } td t  }|jdksJ W d    n1 sw   Y  td t  }|jdks0J W d    n1 s:w   Y  t  }| j|jksJJ d S )Nr   r  )r   r   name)	class_opsnew_opsr3   r3   r4   test_use_ops  s   

r   c                  C   s  t d t } g d}| d|}t|g dg dgksJ | dd dD |}t|g dd	d
gdggks:J t| dt|}t|d tjsOJ t|d tg ds^J t|d tg dsmJ t| jdd dD |dd}|g dd	d
gdggksJ t	|d dksJ t	|d dksJ t	|d dksJ t
t | ddd tdD  W d    n1 sw   Y  t
t | dd W d    d S 1 sw   Y  d S )Nr   )r;   r   r   r   r   r  r   )r;   r   r   )r   r   r  c                 s       | ]}|V  qd S r+   r3   rb   r   r3   r3   r4   	<genexpr>      z!test_minibatch.<locals>.<genexpr>r  r   r   r  r;   c                 s   r!  r+   r3   r"  r3   r3   r4   r#    r$  Tshuffler   r   c                 s   r!  r+   r3   r"  r3   r3   r4   r#    r$  r   )r   r   	minibatchlistr   r   r?  r@  r  r   r   r   r   r   )r   itemsbatchesr3   r3   r4   test_minibatch  s,    "r+  c                  C   s  t d t } tg d}tg d}t| d||}t| ddgddgdd	gd
dggks5J t| jd||dd}t|dksHJ t|d dksRJ t|d dks\J t| dg dg d}|ddgddggdd	gd
dgggks}J t	
t | ddd tdD dd tdD  W d    n1 sw   Y  t	
t | d|dd tdD | W d    d S 1 sw   Y  d S )Nr   r  )r   r  r  r  r   r;   r   r  r   r   r  r  Tr%  r   c                 s   r!  r+   r3   r"  r3   r3   r4   r#    r$  z"test_multibatch.<locals>.<genexpr>r   c                 s   r!  r+   r3   r"  r3   r3   r4   r#    r$  )r   r   r   r   r(  
multibatchconcatenatetolistr   r   r   r   r   )r   arr1arr2r*  r3   r3   r4   test_multibatch  s$   .(, "r1  c                  C   s   t  } tjg dtjd}tddD ]}t| ||td|jd |d  ks*J qt| d|dks7J t| |jd d |dksHJ d S )N)r;   r   r   r   r   r   r;   r   r   r   )	r   r   r   r   r   r   ngramsr   r   )r   r/  nr3   r3   r4   test_ngrams  s   .&r4  z1.9.0zneeds PyTorch 1.9.0
torch_funcir"  )	min_value	max_valuer   )r0   rw  c                 C   s  dd l }|\}}t| |}t| d| }	| j|g|d}
t|
dd}||}||
}|  |
j|jks6J ||
us<J ||
dd}||
u sHJ | jj||ddsSJ | jj|| d	ds`J | j|g|d}
| j|g|d}|	 }t
|	}d
d |jD }|h dkr|	|||
d}|j|
jksJ ||usJ |	|||
dd}||u sJ | j||sJ | jj|j | t|ddsJ d S |ddhkr|	||d}|j|
jksJ | j||	||ddsJ | jj|j | t|ddsJ d S |ddhkr;|	||
d}|j|
jksJ | j||	||
dds"J | jj|j | t|	||
ddds9J d S td| )Nr   r  r   Trz  rt  rL  r   r   c                 S   s   h | ]}|d v r|qS ))rw  r   r  r3   rg   r3   r3   r4   	<setcomp>  s    z4test_compare_activations_to_torch.<locals>.<setcomp>>   r   r  rw  )r  r   )rw  r  r   ru  r  rw  )r  )rw  r  ru  r   r   )rw  r   ru  z5No PyTorch comparison implemented for parameter set: )r2   rr   r   r   r}  r   r   isclosedetachcopyrs   rt   ru   r  itemfloatNotImplementedError)r   r   r0   rw  r5  r2   	func_namepytorch_funcforwardr}  x_thincx_torchrr  y_thincy_think_inplacedY_thincdY_thinc_inplacesr   dx_thincdx_thinc_inplacer3   r3   r4   !test_compare_activations_to_torch  sd   

((
rK  r  r/   c                 C   s   | j |g}| j | j|dd| |sJ | j | j| dg|dd| | dg|s3J | j | j|ddd| |sEJ | j | j| dg|ddd| 	| dg|saJ d S )Nr   )max_valrK   r9   r:   )slopeoffset)
r   r   r9  clipped_linearrV   r  	asarray1fr  rW   r  )r   r0   rB  r3   r3   r4   test_clipped_linear  s   "rQ  
byte_order)><=|c                 C   sx   | j |g}| j||d}t| || |sJ |dv r0|jddj|ks.J d S |jj|jjks:J d S )N)rR  )rS  rT  S)r   r   to_numpyr   r  r   newbyteorder	byteorder)r   rR  r0   rr  r3   r3   r4   test_to_numpy_byteorder.  s    r[  c                  C   s2   t D ]} t| }|d usJ qt d usJ d S r+   )r   r   get_functionr   )kernel_namecompiled_kernelr3   r3   r4   test_custom_kernel_compilation=  s   
r_  c                 C   s&   ddg}|t | j|ddksJ d S )Nr-  l   4dU5
 r   r   )r(  r   )r   uint64_listr3   r3   r4   test_asarray_from_list_uint64F  s   ra  r+   )rs   platformtypingr   r   r   r   
hypothesisr   r   hypothesis.strategiesr   r   numpy.testingr   packaging.versionr	   	thinc.apir
   r   r   r   r   r   r   r   thinc.backends._custom_kernelsr   r   r   thinc.compatr   r   r   thinc.typesr   
thinc.utilr   r    r   r   r   MAX_EXAMPLESVANILLA_OPS	NUMPY_OPSBLIS_OPSCPU_OPSXP_OPSappendALL_OPSFLOAT_TYPES	INT_TYPES
REDUCTIONSREDUCE_ZERO_LENGTH_RAISESr^   TORCH_FUNCSmarkparametrizer   filterwarningsr   r   r   r   r   r   r   r   r   
arrays_BOPr   r   r  r  r,  r6  r;  r=  rC  rH  rQ  rV  rW  rX  skipifre  rj  ro  rs  rv  r=  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  skipr  r  r   r+  r1  r4  floatsrK  rQ  r[  r_  ra  r3   r3   r3   r4   <module>   s<   (


C







		





+.B








		(

	

	
	






	
	



	
;


