o iR@sdZddlZddlZddlmmZddlm Z ddl Z ddl m Z ddl mZmZddlmZmZmZddlZGdddeZd d ZGd d d ejjZGd ddejjZGdddejjZGdddejjZGdddejjZGdddejjZGdddejjZdS)zSubsampling layer definition.N)PositionalEncoding) sequence_mask)sub_factor_to_params pad_to_len)OptionalTupleUnioncs eZdZdZfddZZS)TooShortUttErrorzRaised when the utt is too short for subsampling. Args: message (str): Message for error catch actual_size (int): the short size that cannot pass the subsampling limit (int): the limit size for subsampling cst|||_||_dS)z/Construct a TooShortUttError for error handler.N)super__init__ actual_sizelimit)selfmessager r  __class___/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/transformer/utils/subsampling.pyr s  zTooShortUttError.__init__)__name__ __module__ __qualname____doc__r __classcell__rrrrr s r cCs\t|tr |dkr dSt|tr|dkrdSt|tr!|dkr!dSt|tr,|dkr,dSd S) z4Check if the utterance is too short for subsampling.)Tr)Tr )Tr)Tr)F) isinstanceConv2dSubsampling2Conv2dSubsamplingConv2dSubsampling6Conv2dSubsampling8)inssizerrrcheck_short_utt$sr%c2eZdZdZd fdd ZddZddZZS) r Convolutional 2D subsampling (to 1/4 length). Args: idim (int): Input dimension. odim (int): Output dimension. dropout_rate (float): Dropout rate. pos_enc (torch.nn.Module): Custom position encoding layer. Nc stt|tjtjd|ddtjtj||ddtj|_tjtj ||dddd||dur>|nt |||_ dS)&Construct an Conv2dSubsampling object.rN) r r r torchnn SequentialConv2dReLUconvLinearroutridimodim dropout_ratepos_encrrrr <  zConv2dSubsampling.__init__cCs|d}||}|\}}}}||dd||||}|dur,|dfS||dddddddfdddddddffS)Subsample x. Args: x (torch.Tensor): Input tensor (#batch, time, idim). x_mask (torch.Tensor): Input mask (#batch, 1, time). Returns: torch.Tensor: Subsampled tensor (#batch, time', odim), where time' = time // 4. torch.Tensor: Subsampled mask (#batch, 1, time'), where time' = time // 4. r)r*N unsqueezer0r$r2 transpose contiguousviewrxx_maskbctfrrrforwardJ  $8zConv2dSubsampling.forwardcC|dkrtd|j|SzGet item. When reset_parameters() is called, if use_scaled_pos_enc is used, return the positioning encoding. rz+Support only `-1` (for `reset_parameters`).NotImplementedErrorr2rkeyrrr __getitem__` zConv2dSubsampling.__getitem__Nrrrrr rGrOrrrrrr 1  r cr&) Conv2dSubsamplingPadr'Nc stt|tjtjjd|ddddtjtjj||ddddtj|_tjtj ||dddd||durB|nt |||_ tj dd|_ dS) r(r)rr*)rr)paddingN)r)r rTr r+r,r-r.r/r0r1rr2 ConstantPad1dpad_fnr3rrrr ws zConv2dSubsamplingPad.__init__c Cs|dd}||}|dd}|d}||}|\}}}}||dd||||}|dur=|dfStj |dddddfdd}|ddd}|ddd}t |d|j |dj }||dddddffS)r9r)r*Nrrdim) r=rYr<r0r$r2r>r?r+sumrdtypedevice) rrArBrCrDrErFx_lenmaskrrrrGs     $ zConv2dSubsamplingPad.forwardcCrIrJrKrMrrrrOrPz Conv2dSubsamplingPad.__getitem__rQrRrrrrrTls  rTcr&) rzConvolutional 2D subsampling (to 1/2 length). Args: idim (int): Input dimension. odim (int): Output dimension. dropout_rate (float): Dropout rate. pos_enc (torch.nn.Module): Custom position encoding layer. Nc stt|tjtjd|ddtjtj||ddtj|_tjtj ||ddd||dur<|nt |||_ dS)z'Construct an Conv2dSubsampling2 object.r)rr*N) r rr r+r,r-r.r/r0r1rr2r3rrrr s zConv2dSubsampling2.__init__cCs|d}||}|\}}}}||dd||||}|dur,|dfS||dddddddfdddddddffS)aSubsample x. Args: x (torch.Tensor): Input tensor (#batch, time, idim). x_mask (torch.Tensor): Input mask (#batch, 1, time). Returns: torch.Tensor: Subsampled tensor (#batch, time', odim), where time' = time // 2. torch.Tensor: Subsampled mask (#batch, 1, time'), where time' = time // 2. r)r*Nr:r;r@rrrrGrHzConv2dSubsampling2.forwardcCrIrJrKrMrrrrOrPzConv2dSubsampling2.__getitem__rQrRrrrrrrSrc*eZdZdZdfdd ZddZZS)r!zConvolutional 2D subsampling (to 1/6 length). Args: idim (int): Input dimension. odim (int): Output dimension. dropout_rate (float): Dropout rate. pos_enc (torch.nn.Module): Custom position encoding layer. Nc stt|tjtjd|ddtjtj||ddtj|_tjtj ||dddd||dur>|nt |||_ dS)z'Construct an Conv2dSubsampling6 object.r)rr*N) r r!r r+r,r-r.r/r0r1rr2r3rrrr r8zConv2dSubsampling6.__init__cCs|d}||}|\}}}}||dd||||}|dur,|dfS||dddddddfdddddddffS)aSubsample x. Args: x (torch.Tensor): Input tensor (#batch, time, idim). x_mask (torch.Tensor): Input mask (#batch, 1, time). Returns: torch.Tensor: Subsampled tensor (#batch, time', odim), where time' = time // 6. torch.Tensor: Subsampled mask (#batch, 1, time'), where time' = time // 6. r)r*Nr:rr;r@rrrrGrHzConv2dSubsampling6.forwardrQrrrrr rGrrrrrr!s r!cra)r"zConvolutional 2D subsampling (to 1/8 length). Args: idim (int): Input dimension. odim (int): Output dimension. dropout_rate (float): Dropout rate. pos_enc (torch.nn.Module): Custom position encoding layer. Nc stt|tjtjd|ddtjtj||ddtjtj||ddtj|_tjtj ||dddddd||durN|nt |||_ dS)z'Construct an Conv2dSubsampling8 object.r)rr*N) r r"r r+r,r-r.r/r0r1rr2r3rrrr %s( zConv2dSubsampling8.__init__cCs|d}||}|\}}}}||dd||||}|dur,|dfS||dddddddfdddddddfdddddddffS)aSubsample x. Args: x (torch.Tensor): Input tensor (#batch, time, idim). x_mask (torch.Tensor): Input mask (#batch, 1, time). Returns: torch.Tensor: Subsampled tensor (#batch, time', odim), where time' = time // 8. torch.Tensor: Subsampled mask (#batch, 1, time'), where time' = time // 8. r)r*Nr:r;r@rrrrG5s  $PzConv2dSubsampling8.forwardrQrdrrrrr"s r"csFeZdZdZ  d dedeffdd Zdefd d Zd d ZZ S)Conv1dSubsamplingzConvolutional 1D subsampling (to 1/2 length). Args: idim (int): Input dimension. odim (int): Output dimension. dropout_rate (float): Dropout rate. pos_enc (torch.nn.Module): Custom position encoding layer. stride_conv!seq2seq/proj_encoder/downsampling!tf2torch_tensor_name_prefix_torchtf2torch_tensor_name_prefix_tfcsNtt|tj|||||_tj|d|_||_ ||_ ||_ ||_ dS)NrW) r rer r+r,Conv1dr0rXrYstrider5rhri)rr4r5 kernel_sizerkpadrhrirrrr Ws  zConv1dSubsampling.__init__returncCs|jSrQ)r5)rrrr output_sizeiszConv1dSubsampling.output_sizecCs`|dd}||}tj||dd}|dd}|dur#|dfS|d|jd}||fS)z Subsample x.r)r*rW)negative_slopeN)r=rYF leaky_relur0rk)rrAr_rrrrGls   zConv1dSubsampling.forward)rfrg) rrrrstrr introrGrrrrrreLsrecseZdZdZ    ddedeeefded ed ed eed dffd d Z de j dee j dee j d ee j e j ffddZ de j d e j fddZ de j d e j fddZded efddZZS)StreamingConvInputa Streaming ConvInput module definition. Args: input_size: Input size. conv_size: Convolution size. subsampling_factor: Subsampling factor. vgg_like: Whether to use a VGG-like network. output_size: Block output dimension. rVTrN input_size conv_sizesubsampling_factorvgg_likeconv_kernel_sizerorncs2t|r|dkrz|\}}tjtjjd||d|dddtjtjj|||d|dddtjtjdtjj|||d|dddtjtjj|||d|dddtjtjd |_||dd} d|_ d|_ |j |_ n|\}}t |d} tjtjjd||d|dddtjtjj|||d|dddtjtj| dftjj|||d|dddtjtjj|||d|dddtjtjd |_||dd} ||_ |j |_ | |_ n|dkr8tjtjd|dddgddgtjtj|||ddgddgtj|_||dddd} ||_ ||_d|_|j|_ nAt||\} } } tjtjd|ddddgtjtj||| | | dddgtj|_|| } ||_ | |_| |_|j|_ ||_d|_|d urtj| ||_||_d Sd |_| |_d S) zConstruct a ConvInput object.r)r*)rkrU)r)r*)r*r*rrrN)r r r+r,r-r.r/ MaxPool2dr0rxstride_1create_new_vgg_maskcreate_new_maskrtkernel_2stride_2create_new_conv2d_maskrrymin_frame_lengthr1outputro)rrvrwrxryrzro conv_size1 conv_size2 output_projkernel_1rrconv_2_output_sizerrrr s       %       %       zStreamingConvInput.__init__rAr` chunk_sizec st|dur||}t|dd}|\}}}|d}|dur^t||jt t |||jt fdd|}t |}t j|dd}||j}|||d||j|}||}|\} } } }|dur|dd|d| |ddd|ddf}n|dd|d| |}|jdur||}||ddd|fddd|dffS) a'Encode input sequences. Args: x: ConvInput input sequences. (B, T, D_feats) mask: Mask of input sequences. (B, 1, T) Returns: x: ConvInput output sequences. (B, sub(T), D_out) mask: Mask of output sequences. (B, 1, sub(T)) Nrr)cs t|dS)Nr))r)inputsmax_input_lengthrr=s z,StreamingConvInput.forward..rZr*r)r~maxeqr\r$r<rtrxmathceilfloatmaplistr+stackr?r0r=r>r) rrAr`rolensrCrErFN_chunks_rDrrrrG%s4   6  .zStreamingConvInput.forwardcCs|jdkrK|d|d|jd}|ddd|fdddd|jdf}|d|dd}|ddd|fdddddf}|S|}|S)zCreate a new mask for VGG output sequences. Args: mask: Mask of input sequences. (B, T) Returns: mask: Mask of output sequences. (B, sub(T)) r)r*N)rxr$)rr` vgg1_t_len vgg2_t_lenrrrr}Ps ,&z&StreamingConvInput.create_new_vgg_maskcCs8|jdkr|dddddfdddd|jfS|S)zCreate new conformer mask for Conv2d output sequences. Args: mask: Mask of input sequences. (B, T) Returns: mask: Mask of output sequences. (B, sub(T)) r)Nr*)rxr)rr`rrrrbs *z)StreamingConvInput.create_new_conv2d_maskr$cCs ||jS)zReturn the original size before subsampling for a given size. Args: size: Number of frames after subsampling. Returns: : Number of frames before subsampling. )rx)rr$rrrget_size_before_subsamplingns z.StreamingConvInput.get_size_before_subsampling)rVTrN)rrrrrtrrboolrr r+TensorrGr}rrrrrrrru{sD  ! + ru) rnumpynpr+torch.nn.functionalr, functionalrq#funasr.models.transformer.embeddingrloggingfunasr.models.scama.utilsr*funasr.models.transformer.utils.nets_utilsrrtypingrrrr Exceptionr r%Moduler rTrr!r"rerurrrrs&   ;C;02/