o
    i                     @   sP  d dl mZmZmZmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ eZeZed		d!ddddddee dee dee dee dee dedeeef fddZdeeef dededeeef fddZ		d!dededeeef dee dee ddfdd ZdS )"    )CallableOptionalTuplecast   )registry)glorot_uniform_init	zero_init)Model)Floats1dFloats2d)	get_widthpartial   )chain)Dropout)	LayerNormzMish.v1NF)init_Winit_bdropout	normalizenOnIr   r   r   r   returnc                C   s   |du rt }|du rt}tdttt||| |ddddd}|r0t|tttt	f t
| d}|durBt|tttt	f t|}|S )zODense layer with mish activation.
    https://arxiv.org/pdf/1908.08681.pdf
    Nmish)r   r   )Wb)initdimsparams)r   )r   r	   r
   forwardr   r   r   r   InTOutTr   r   )r   r   r   r   r   r   model r$   E/home/ubuntu/.local/lib/python3.10/site-packages/thinc/layers/mish.pyMish   s    
r&   r#   Xis_trainc                    sj   t td t td}jj dd|7 j}dtdtf fdd}||fS )	Nr   r   T)trans2dYr   c                    sN   j | }dj j|dd d|jdd j | }|S )Nr   T)trans1r   r   )axis)opsbackprop_mishinc_gradgemmsum)r*   dY_pre_mishdXr   r'   
Y_pre_mishr#   r$   r%   backprop6   s
   zforward.<locals>.backprop)	r   r   	get_paramr   r-   r0   r   r"   r!   )r#   r'   r(   r   Yr6   r$   r4   r%   r    /   s   r    r8   c              	   C   st   |d ur| dt| |d ur| dt| |d| |j|d|df |d||j|df d S )Nr   r   r   r   )set_dimr   	set_paramr-   get_dim)r   r   r#   r'   r8   r$   r$   r%   r   @   s   $ r   )NN) typingr   r   r   r   configr   initializersr   r	   r#   r
   typesr   r   utilr   r   r   r   r   	layernormr   r!   r"   layersintfloatboolr&   r    r   r$   r$   r$   r%   <module>   sd    
*
