o i[@sdZddlZddlmZmZmZmZddlZddlm Z ddl m Z ddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlm Z m!Z!m"Z"dd l#m$Z$e ej%e d kZ&d Z'Gd d d ej(j)Z*Gdddej(j)Z+dS)zDNN beamformer module.N)ListOptionalTupleUnion)parse) functional) ComplexTensor)apply_beamforming_vectorblind_analytic_normalizationget_gev_vectorget_lcmv_vector_with_rtfget_mvdr_vectorget_mvdr_vector_with_rtfget_mwf_vectorget_rank1_mwf_vectorget_rtf_matrixget_sdw_mwf_vectorget_WPD_filter_v2get_WPD_filter_with_rtfperform_WPD_filteringprepare_beamformer_stats)stack to_doubleto_float) MaskEstimatorz1.9.0)mvdr mvdr_soudenmpdr mpdr_soudenwmpdr wmpdr_soudenwpd wpd_soudenmwfwmwfsdw_mwfr1mwflcmvlcmpwlcmpgevgev_banmvdr_tfsmvdr_tfs_soudenc+sDeZdZdZ                d7dedededededededededededededed ed!ed"ed#ed$ed%ed&ef*fd'd( Z ) )d8d*e e j e fd+e j d,eee j d-eee j d.ee e j e fe j e j ff d/d0Z ) ) 1d9d2d3Zd*e e j e fd+e j d.eee j d4fe j ffd5d6ZZS):DNN_BeamformerzDNN mask based Beamformer. Citation: Multichannel End-to-end Speech Recognition; T. Ochiai et al., 2017; http://proceedings.mlr.press/v70/ochiai17a/ochiai17a.pdf blstmp,@Tsigmoidr?ư>Hz>Fbtypeblayersbunitsbprojsnum_spkuse_noise_mask nonlinear dropout_ratebadim ref_channelbeamformer_typertf_iterationsmwf_muepsdiagonal_loadingdiag_eps mask_flooringflooring_thresuse_torch_solverbtapsbdelayc s~t|r |dn|}t|||||| ||d|_| dkr%t|| |dnd|_| |_||_|dks6J|||_||_ | t vrFt d| | dksO| dsz|sz|dkrht d| t d | nt d | t d | | |_| ds| d ksJ| | |_||_|dkr|dksJ||f||_|jdkr|nd|_||_||_||_||_||_||_dS) Nr3)nmaskrBr)rIz!Not supporting beamformer_type=%sr_soudenzMInitializing %s beamformer without noise mask estimator (single-speaker case)zI(1 - speech_mask) will be used for estimating noise PSD in %s beamformer!zLInitializing %s beamformer without noise mask estimator (multi-speaker case)zQInterference speech masks will be used for estimating noise PSD in %s beamformer!r7)super__init__rmaskAttentionReferencerefrErAr@rQBEAMFORMER_TYPES ValueErrorendswithloggingwarningupperrFrGrHrOrPrIrJrKrLrMrN)selfbidimr<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPbnmask __class__U/home/ubuntu/.local/lib/python3.10/site-packages/espnet2/enh/layers/dnn_beamformer.pyrTKsv     zDNN_Beamformer.__init__Ndatailenspowers oracle_masksreturnc s|dddd}t|}|dur|}n||\}}jt|ks(Jt|jr4fdd|D}jdkrjrA|\}} n|d}d|} jdvrRt d t ||g| |jj j j d } jd vr{j||| d | d | dd\} } n7jdsjdksjdksjdksjdksjdr||| d | d \} } ntdj| dd} njrt|dd}|d} nt|}d} t ||| |jj j j d } jdvrt| d | djjjjjd} gg} } tjD]}jdvr$j||| d || d || d|d\}}njdvr=||| d || d |\}}nrjdkrZj||| d | d || d|d\}}nUjdvrq||| d | d |\}}n>jdkrj||| d | d || |d\}}n$jd vrj||| d || d || |d\}}ntdj|dd}| || |qd!d|D}| ||fS)"aDNN_Beamformer forward function. Notation: B: Batch C: Channel T: Time or Sequence length F: Freq Args: data (torch.complex64/ComplexTensor): (B, T, C, F) ilens (torch.Tensor): (B,) powers (List[torch.Tensor] or None): used for wMPDR or WPD (B, F, T) oracle_masks (List[torch.Tensor] or None): oracle masks (B, F, C, T) if not None, oracle_masks will be used instead of self.mask Returns: enhanced (torch.complex64/ComplexTensor): (B, T, F) ilens (torch.Tensor): (B,) masks (torch.Tensor): (B, T, C, F) rr0r7r3Ncsg|] }tj|jdqS))min)torchclamprM.0mr^rcrd sz*DNN_Beamformer.forward..)r'r(r)z"Single source is not supported yet)rgrFrPrOrI)rrrr!psd_n psd_speechpsd_distortion)rtrRr#r$r%r&r*!Not supporting beamformer_type={}r6)rJrErGrNrK)rr,rr!) rr-r r"r$r%r&r*r+r)rr#r()rtf_matspk)r'r)cSg|]}|ddqSr6 transposermrcrcrdrqg)permuterrUrQlenrLr@rArFNotImplementedErrorrrPrOrIapply_beamformingrZ startswithrYformatr}listrrJrErGrNrKrangeappend)r^rerfrgrhdata_dmasks_ mask_speech mask_noisebeamformer_statsenhancedwsrwienhwrcrprdforwards                                       zDNN_Beamformer.forwardrc sjdkrjjd|\}n+jdr?tj dd dfj tjddjf d njjd vrht t |t t jjjjjd } t| t } nˈjd krt|ttfsvJfd d|D} tfdd| D} t| jddd} Wdn1swY| d| d} t| dd} nxjdvrtt t |jjjd} t| t } nYjdkr.t|ttfsJfdd|D} tfdd| D} t| jddd} Wdn 1swY| d| d} t| dd} njdkrXtt |t t jjjjjd } t| t jj } nۈjdkryt!t t |jjd} t| t jj } njdvrt"t t |jjjd} t| t } njdkrt#t t |j$jjjd} t| t } nzjdkrt%t t |j$jjjd} t| t } nYjdvrt&t |t ||jjjd } t| t } n:j'd!r+t(t |t d"jjd#} t| t } jd$kr*t)| t |} | | *d%} nt+d&,j| jjd| jjdfS)'aBeamforming with the provided statistics. Args: data (torch.complex64/ComplexTensor): (B, F, C, T) ilens (torch.Tensor): (B,) psd_n (torch.complex64/ComplexTensor): Noise covariance matrix for MVDR (B, F, C, C) Observation covariance matrix for MPDR/wMPDR (B, F, C, C) Stacked observation covariance for WPD (B,F,(btaps+1)*C,(btaps+1)*C) psd_speech (torch.complex64/ComplexTensor): Speech covariance matrix (B, F, C, C) psd_distortion (torch.complex64/ComplexTensor): Noise covariance matrix (B, F, C, C) rtf_mat (torch.complex64/ComplexTensor): RTF matrix (B, F, C, num_spk) spk (int): speaker index Return: enhanced (torch.complex64/ComplexTensor): (B, F, T) ws (torch.complex64/ComplexTensor): (B, F) or (B, F, (btaps+1)*C) r)dtyperRNr{rv)devicer.r3)rrr iterationsreference_vectornormalize_ref_channelrNrJrKr,c s<g|]}tt|ttjjjjjd qS)r)rrrGrErNrJrKrnpsd_n_i)rtrsr^urcrdrqs z4DNN_Beamformer.apply_beamforming..cg|] }t|tqSrcr rrnrrercrdrqT)dimkeepdimsr)rrr rNrJrKr-c s.g|]}ttt|jjjdqS)r)r rrNrJrKr)rsr^rrcrdrqs crrcrrrrcrdrqrr!r")rJrK)r#r$r%)denoising_weightrNrJrKr&)r(r)r')rrNrJrKr*power)moderJrKr+r6ru)-rErWtordoublerFrZrkzerossizerfill_rrrGrNrJrKr isinstancertuplerno_gradabsargmingathersqueezer rrrPrOrrrrHrr rr r unsqueezerYr) r^rerfrrrsrtrwrxrrrindexgainrc)rertrsr^rrdrjs                    z DNN_Beamformer.apply_beamforming.c Cs6|t|dddd|\}}dd|D}||fS)aPredict masks for beamforming. Args: data (torch.complex64/ComplexTensor): (B, T, C, F), double precision ilens (torch.Tensor): (B,) Returns: masks (torch.Tensor): (B, T, C, F) ilens (torch.Tensor): (B,) rr0r7r3cSryrzr|rmrcrcrdrq@r~z/DNN_Beamformer.predict_mask..)rUrr)r^rerfrrrcrcrd predict_mask2s zDNN_Beamformer.predict_mask)r/r0r1r2r3Tr4r5r2r6rr7r8r9Tr:Fr9Tr;r0)NN)NNr)__name__ __module__ __qualname____doc__strintboolfloatrTrrkTensorr LongTensorrrrrrr __classcell__rcrcrardr.Bs      ^    L I r.c sTeZdZd fdd Z d deejefdejde de ejejffd d Z Z S) rVr9cs4ttj|||_tj|d|_||_dS)Nr3)rSrTrknnLinearmlp_psdgvecrI)r^r_att_dimrIrarcrdrTEs  zAttentionReference.__init__@psd_inrfscalingric Cs|dd\}}}|d|dksJ||tj|tj|jdtjd}|jdd|ddd }|j d|j d|j d }| |} | t| d} tj|| dd} | |fS) a%Attention-based reference forward function. Args: psd_in (torch.complex64/ComplexTensor): (B, F, C, C) ilens (torch.Tensor): (B,) scaling (float): Returns: u (torch.Tensor): (B, C) ilens (torch.Tensor): (B,) Nr0r7)rrrr6rr3rvg?)r masked_fillrkeyerrtypesumr}realimagrIrrtanhrFsoftmax) r^rrfrBrCpsdpsd_featrerrcrcrdrKs  zAttentionReference.forward)r9)r) rrrrTrrkrrrrrrrrcrcrardrVDs  rV),rr[typingrrrrrkpackaging.versionrVtorch.nnrrtorch_complex.tensorrespnet2.enh.layers.beamformerr r r r r rrrrrrrrr espnet2.enh.layers.complex_utilsrrr!espnet2.enh.layers.mask_estimatorr __version__is_torch_1_9_plusrXrModuler.rVrcrcrcrds$   @ #