o
    ߥiXD                     @   sj  d Z ddlmZ ddlZddlZddlmZ ddlm  m	Z
 dd Zdd ZG dd	 d	ejZG d
d dejZG dd dejZG dd dejZ		d)dededededef
ddZG dd dejZedkredddddd d!dddd"Zee ed#d$ e D Zed%e edd&dZee\ZZed'ej  ed(ej  ee!  dS dS )*z8
FSMN implementation.

Copyright: 2022-03-09 yueyue.nyy
    )TupleNc                 C   s:   t jt jt jd t| }|dd}|dd}d| S )N)	threshold	linewidth[ ]z[ %s ]
)npset_printoptionsinfnanstrreplace)np_matout_str r   ^/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/audio/kws/nearfield/fsmn.pytoKaldiMatrix   s
   r   c                 C   s,   d}|     }|t|7 }t| d S )Nr   )detachsqueezenumpyr   print)torch_tensorre_strxr   r   r   printTensor   s   r   c                       4   e Zd Z fddZdd Zdd Zdd Z  ZS )	LinearTransformc                    sH   t t|   || _|| _tj||dd| _tj	
 | _tj	 | _d S )NF)bias)superr   __init__	input_dim
output_dimnnLinearlineartorchquantization	QuantStubquantDeQuantStubdequantselfr    r!   	__class__r   r   r   !   s   zLinearTransform.__init__c                 C   "   |  |}| |}| |}|S Nr(   r$   r*   r,   inputoutputr   r   r   forward)      


zLinearTransform.forwardc                 C   sH   d}|d| j | jf 7 }|d7 }|  d }|  }|t|7 }|S )Nr   z<LinearTransform> %d %d
z<LearnRateCoef> 1
linear.weightr!   r    
state_dictr   r   r   )r,   r   linear_weightsr   r   r   r   to_kaldi_net0   s   zLinearTransform.to_kaldi_netc           
      C   s  |  }|  }t|dksJ |d dksJ t|d | _t|d | _|  }|ddks5J | j	  t
j| j| jft
jd}t| jD ]/}|  }| d	  }t|| jkseJ t
jd
d |D t
jd}	|	||d d f< qK|| jj_d S )N   r   z<LinearTransform>      LearnRateCoefdtype[]c                 S      g | ]}t |qS r   float.0itemr   r   r   
<listcomp>R       z2LinearTransform.to_pytorch_net.<locals>.<listcomp>)readlinestripsplitlenintr!   r    findr$   reset_parametersr%   zerosfloat32rangetensorweightdata)
r,   freadlinear_linelinear_splitlearn_rate_linenew_weightsilinesplitscolsr   r   r   to_pytorch_net=   s*   
zLinearTransform.to_pytorch_net__name__
__module____qualname__r   r5   r;   rb   __classcell__r   r   r-   r   r      s
    r   c                       r   )	AffineTransformc                    sD   t t|   || _|| _t||| _tj	
 | _tj	 | _d S r0   )r   rh   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r-   r   r   r   [   s   zAffineTransform.__init__c                 C   r/   r0   r1   r2   r   r   r   r5   d   r6   zAffineTransform.forwardc                 C   sl   d}|d| j | jf 7 }|d7 }|  d }|  }|t|7 }|  d }|  }|t|7 }|S )Nr   z<AffineTransform> %d %d
z4<LearnRateCoef> 1 <BiasLearnRateCoef> 1 <MaxNorm> 0
r7   zlinear.biasr8   )r,   r   r:   r   linear_biasr   r   r   r;   k   s   zAffineTransform.to_kaldi_netc                 C   sl  |  }|  }t|dksJ |d dksJ t|d | _t|d | _td| j| jf  |  }|ddks?J | j	
  tj| j| jftjd	}t| jD ]/}|  }| d
  }t|| jksoJ tjdd |D tjd	}	|	||d d f< qU|| j	j_|  }
|
 d
  }t|| jksJ tjdd |D tjd	}|| j	j_d S )Nr<   r   z<AffineTransform>r=   r>   z'AffineTransform output/input dim: %d %dr?   r@   rA   rC   c                 S   rD   r   rE   rG   r   r   r   rJ      rK   z2AffineTransform.to_pytorch_net.<locals>.<listcomp>c                 S   rD   r   rE   rG   r   r   r   rJ      rK   )rL   rM   rN   rO   rP   r!   r    r   rQ   r$   rR   r%   rS   rT   rU   rV   rW   rX   r   )r,   rY   affine_lineaffine_splitr\   r]   r^   r_   r`   ra   	bias_linenew_biasr   r   r   rb   |   s>   


zAffineTransform.to_pytorch_netrc   r   r   r-   r   rh   Y   s
    	rh   c                       sH   e Zd Z				ddedef fddZdd Zd	d
 Zdd Z  ZS )	FSMNBlockNr=   r    r!   c                    s   t t|   || _|d u rd S || _|| _|| _|| _tj	| j| j|dg|dg| jdd| _
|dkrGtj	| j| j|dg|dg| jdd| _nd | _tj | _tj | _d S )Nr=   F)dilationgroupsr   r   )r   rn   r   dimlorderrorderlstriderstrider"   Conv2d	conv_left
conv_rightr%   r&   r'   r(   r)   r*   )r,   r    r!   rr   rs   rt   ru   r-   r   r   r      s4   	


zFSMNBlock.__init__c           	      C   s   t |d}|dddd}t|dd| jd | j dg}| |}| |}| 	|}|| }| j
d urht|ddd| j| j g}|d d d d | jd d d f }| |}| 
|}| 	|}||7 }|dddd}|d}|S )Nr=   r   r<   r>   )r%   	unsqueezepermuteFpadrr   rt   r(   rw   r*   rx   rs   ru   r   )	r,   r3   r   x_pery_leftouty_rightout_perr4   r   r   r   r5      s"    



"



zFSMNBlock.forwardc                 C   s   d}|d| j | j f 7 }|dd| j| j| j| jf 7 }|  d }t| 	 j
}|t|7 }| jd urI|  d }| 	 j
}|t|7 }|S )Nr   z<Fsmn> %d %d
zQ<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0
r=   zconv_left.weightzconv_right.weight)rq   rr   rs   rt   ru   r9   r   flipudr   r   Tr   rx   )r,   r   lfitersr   rfitersr   r   r   r;      s   
zFSMNBlock.to_kaldi_netc                 C   s  |  }|  }t|dksJ |d dksJ t|d | _|  }| d  }t|dks8J |d dks@J |d d	ksHJ t|d | _|d
 dksWJ t|d | _|d dksfJ t|d | _|d dksuJ t|d | _	|d dksJ t
d tj| jd| jdftjd}t| jD ]<}t
d|  |  }| d  }	t|	| jksJ tjdd |	D tjd}
|
|| jd | dd d df< qt|dd}| j  || jj_| jdkrWt
d tj| jd| jdftjd}|  }t| jD ]9}t
d|  |  }| d  }	t|	| jks+J tjdd |	D tjd}
|
||dd d df< q
t|dd}| j  || jj_d S d S )Nr<   r   z<Fsmn>r=   rC      z<LearnRateCoef>r>   z<LOrder>   z<ROrder>      z	<LStride>      z	<RStride>	   
   z	<MaxNorm>zread conv_left weightrA   zread conv_left weight -- %dc                 S   rD   r   rE   rG   r   r   r   rJ     rK   z,FSMNBlock.to_pytorch_net.<locals>.<listcomp>zread conv_right weightzread conv_right weight -- %dc                 S   rD   r   rE   rG   r   r   r   rJ   /  rK   )rL   rM   rN   rO   rP   rq   rr   rs   rt   ru   r   r%   rS   rT   rU   rV   	transposerw   rR   rW   rX   rx   )r,   rY   	fsmn_line
fsmn_splitparams_lineparams_splitnew_lfiltersr^   r_   r`   ra   new_rfiltersr   r   r   rb      sj    


zFSMNBlock.to_pytorch_net)NNr=   r=   )	rd   re   rf   rP   r   r5   r;   rb   rg   r   r   r-   r   rn      s    )rn   c                       r   )	RectifiedLinearc                    s.   t t|   || _t | _td| _d S )Ng?)	r   r   r   rq   r"   ReLUreluDropoutdropoutr+   r-   r   r   r   <  s   
zRectifiedLinear.__init__c                 C   s   |  |}|S r0   )r   )r,   r3   r   r   r   r   r5   B  s   
zRectifiedLinear.forwardc                 C   s   d}|d| j | j f 7 }|S )Nr   z<RectifiedLinear> %d %d
)rq   )r,   r   r   r   r   r;   G  s   zRectifiedLinear.to_kaldi_netc                 C   sx   |  }|  }t|dksJ |d dksJ t|d t|d ks(J t|d | jks3J t|d | _d S )Nr<   r   z<RectifiedLinear>r=   r>   )rL   rM   rN   rO   rP   rq   )r,   rY   r_   r`   r   r   r   rb   U  s   zRectifiedLinear.to_pytorch_netrc   r   r   r-   r   r   :  s
    r   r=   fsmn_layers
linear_dimproj_dimrr   rs   c                    s&    fddt | D }tj| S )Nc                    s<   g | ]}t t td d t t  qS )r=   )r"   
Sequentialr   rn   rh   r   )rH   r^   r   rr   r   rs   r   r   rJ   h  s    z"_build_repeats.<locals>.<listcomp>)rU   r"   r   )r   r   r   rr   rs   rt   ru   repeatsr   r   r   _build_repeats_  s   	
	r   c                       s   e Zd Zdedededededededed	ed
edef fddZdd Zejdddejdfdej	dej	de
ej	ej	f fddZdd Zdd Z  ZS )FSMNr    input_affine_dimr   r   r   rr   rs   rt   ru   output_affine_dimr!   c                    s   t t|   || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _t||| _t||| _t||| _t|||||||	| _t||
| _t|
|| _dS )a  
            Args:
                input_dim:              input dimension
                input_affine_dim:       input affine layer dimension
                fsmn_layers:            no. of fsmn units
                linear_dim:             fsmn input dimension
                proj_dim:               fsmn projection dimension
                lorder:                 fsmn left order
                rorder:                 fsmn right order
                lstride:                fsmn left stride
                rstride:                fsmn right stride
                output_affine_dim:      output affine layer dimension
                output_dim:             output dimension
        N)r   r   r   r    r   r   r   r   rr   rs   rt   ru   r   r!   rh   
in_linear1
in_linear2r   r   r   fsmnout_linear1out_linear2)r,   r    r   r   r   r   rr   rs   rt   ru   r   r!   r-   r   r   r   v  s(   
zFSMN.__init__c                 C   s   d S r0   r   )r,   r   r   r   fuse_modules  s   zFSMN.fuse_modulesr   rA   r3   in_cachereturnc           	      C   sD   |  |}| |}| |}| |}| |}| |}||fS )z
        Args:
            input (torch.Tensor): Input tensor (B, T, D)
            in_cache(torch.Tensor): (B, D, C), C is the accumulated cache size
        )r   r   r   r   r   r   )	r,   r3   r   x1x2x3x4x5x6r   r   r   r5     s   





zFSMN.forwardc                 C   s   d}|d7 }|| j  7 }|| j 7 }|| j 7 }| jD ]"}||d  7 }||d  7 }||d  7 }||d  7 }q|| j 7 }|| j 7 }|d| j| jf 7 }|d7 }|S )	Nr   z<Nnet>
r   r=   r>   r<   z<Softmax> %d %d
z</Nnet>
)r   r;   r   r   r   r   r   r!   )r,   r   r   r   r   r   r;     s   
zFSMN.to_kaldi_netc                 C   sH  t |ddd}t |d}| }| dksJ | j| | j| | j| | jD ]}|d | |d | |d | |d | q.| j| | j	| | }| 
 }|d  d	ksmJ t|d | jksxJ t|d | jksJ | }| d
ksJ W d    n1 sw   Y  |  d S )Nrutf8)encodingz<Nnet>r   r=   r>   r<   z	<Softmax>z</Nnet>)openrL   rM   r   rb   r   r   r   r   r   rN   rP   r!   close)r,   
kaldi_filerY   nnet_start_liner   softmax_linesoftmax_splitnnet_end_liner   r   r   rb     s.   

zFSMN.to_pytorch_net)rd   re   rf   rP   r   r   r%   rS   rF   Tensorr   r5   r;   rb   rg   r   r   r-   r   r   t  sF    	
5
r   __main__i     r         r   r>   i'
  c                 c   s    | ]}|  V  qd S r0   )numel)rH   pr   r   r   	<genexpr>  s    r   zthe number of model params: {}   zinput shape: {}zoutput shape: {})r=   r=   )"__doc__typingr   r   r   r%   torch.nnr"   torch.nn.functional
functionalr{   r   r   Moduler   rh   rn   r   rP   r   r   rd   r   r   sum
parameters
num_paramsformatrS   r   y_shaper;   r   r   r   r   <module>   sN    :K +
 