o iq @sddlZddlZddlmZmZddlZddlZddlm Z ddl m m Z ddlm Z ddlmZddlmZddlmZddlmZdd lmZmZmZmZmZmZdd lmZdd lm Z m!Z!m"Z"d d l#m$Z$e"%e&Z'Gddde j(Z)Gddde j(Z*Gddde j(Z+Gddde j(Z,Gddde j(Z-GdddeZ.GdddeZ/Gddde j(Z0Gddde j(Z1Gd d!d!e j(Z2e Gd"d#d#eZ3Gd$d%d%eZ4Gd&d'd'eZ5Gd(d)d)eZ6Gd*d+d+e j(Z7Gd,d-d-e j(Z8Gd.d/d/e j(Z9  dLd0e:e;e;fd1ef d6d7Z?eZ@e Gd8d9d9e3ZAd:ZBe d;d<Gd=d>d>e3ZCe d?d<Gd@dAdAe3ZDe GdBdCdCe3ZEGdDdEdEe j(ZFGdFdGdGe j(ZGe dHd<GdIdJdJe3ZHgdKZIdS)MN)OptionalUnion)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)GradientCheckpointingLayer)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel)auto_docstringis_peft_availablelogging) WavLMConfigc$eZdZfddZddZZS)WavLMSamePadLayercs*t|ddkrd|_dSd|_dSNrr)super__init__num_pad_remove)selfnum_conv_pos_embeddings __class__e/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/wavlm/modeling_wavlm.pyr&s  zWavLMSamePadLayer.__init__cCs,|jdkr|ddddd|j f}|SNr)rr hidden_statesr!r!r"forward*s zWavLMSamePadLayer.forward__name__ __module__ __qualname__rr& __classcell__r!r!rr"r%s rcr)WavLMPositionalConvEmbeddingcs$ttj|j|j|j|jd|jd|_tjj }t tjj dr'tjj j }t r{ddl }|jj|jjdd||jddd|_Wdn1sLwYt |jdrd|jj jj}|jj jj}n|jj}|jj}|j|||j||n ||jddd|_t|j|_t|j|_dS) Nr) kernel_sizepaddinggroups weight_normr) modifier_rankweight)namedimparametrizations)rrnnConv1d hidden_sizernum_conv_pos_embedding_groupsconvutilsr0hasattrr5r deepspeedzeroGatheredParametersr2 original0 original1weight_gweight_vregister_external_parameterrr.rfeat_extract_activation activation)rconfigr0r=rBrCrr!r"r1s4     z%WavLMPositionalConvEmbedding.__init__cCs:|dd}||}||}||}|dd}|SNrr) transposer:r.rFr$r!r!r"r&Rs     z$WavLMPositionalConvEmbedding.forwardr'r!r!rr"r,0s !r,cr)WavLMFeatureProjectioncsJttj|jd|jd|_t|jd|j|_ t |j |_ dS)Neps) rrr6 LayerNormconv_dimlayer_norm_eps layer_normLinearr8 projectionDropoutfeat_proj_dropoutdropoutrrGrr!r"r^s zWavLMFeatureProjection.__init__cCs&||}||}||}||fSN)rQrSrV)rr%norm_hidden_statesr!r!r"r&ds   zWavLMFeatureProjection.forwardr'r!r!rr"rJ]s rJcseZdZdZ    d"dededed ed ed ef fd d Z    d#dej de ej de ej dede ej e ej e e ej ff ddZ dej deejejfdej dede ej ej ff ddZdededej fddZdej dej fd d!ZZS)$WavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper@ T embed_dim num_headsrV num_buckets max_distancehas_relative_position_biascst||_||_||_|||_|j||jkr'td|jd|d|jd|_t |||_ t |||_ t |||_ t |||_ ||_||_ttd|jdd|_t |jd|_|rqt|j|j|_dSdS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rrr^r_rVhead_dim ValueErrorscalingr6rRk_projv_projq_projout_projr`ra Parametertorchonesgru_rel_pos_constgru_rel_pos_linear Embeddingrel_attn_embed)rr^r_rVr`rarbrr!r"ros.    zWavLMAttention.__init__NFrr%attention_mask position_biasoutput_attentionsreturncCs|\}}}|dur$|||}|d|ddd||j||}||jdd|jdf} | dddd} || } | | jddd d} t | j ddd\} } | | |j d d } | ||jdd|}|d||f}|||||\}}|||fS) z'Attention layer with relative attentionNrrrKrr)rr4?g@)size compute_bias unsqueezerepeatviewr_shapepermuterosumrlsigmoidchunkrntorch_multi_head_self_attention)rr%rrrsrtindexbsztgt_len_gated_hidden_statesrelative_position_projgate_agate_b gate_outputgated_position_bias attn_output attn_weightsr!r!r"r&s"  $  zWavLMAttention.forwardrcCs|dd}}}|dur|dnd}d} } d} tj||||j|jtdgt|j j |j j |j j f| | | |j |jj|jj |j|||d|j j|j j|j jd\} } | dd} | durz| dddf| jdd|jf| jdd} | | fS)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)use_separate_proj_weight q_proj_weight k_proj_weight v_proj_weight)rIneFmulti_head_attention_forwardr^r_rlemptycatribiasrgrhrVrjr2training broadcast_tor~)rr%rrrrtquerykeyvaluekey_padding_maskbias_kbias_v add_zero_attnrrr!r!r"rsB   "z.WavLMAttention.torch_multi_head_self_attention query_length key_lengthcCsvtj|tjddddf}tj|tjddddf}||}||}||jjj}||}|gd}|S)Ndtype)rrr) rlarangelong_relative_positions_buckettorqr2devicer)rrrcontext_positionmemory_positionrelative_positionrelative_position_bucketvaluesr!r!r"rzs  zWavLMAttention.compute_biasrelative_positionscCs|jd}|dktj|}t|}|d}||k}t||}|t|j|}|||}||tj}t |t ||d}|t |||7}|Sr) r`rrlrabslogfloatmathramin full_likewhere)rrr`relative_buckets max_exactis_smallrelative_positions_if_largerelative_position_if_larger!r!r"rs   z)WavLMAttention._relative_positions_bucket)r[r\r]TNNFr)r(r)r*__doc__intrboolrrlTensorrtupler& FloatTensorr LongTensor BoolTensorrrzrr+r!r!rr"rZls^' ) 7 rZcr)WavLMFeedForwardcsptt|j|_t|j|j|_ t |j t r"t |j |_n|j |_t|j|j|_t|j|_dSrX)rrr6rTactivation_dropoutintermediate_dropoutrRr8intermediate_sizeintermediate_dense isinstance hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutrWrr!r"rs  zWavLMFeedForward.__init__cCs6||}||}||}||}||}|SrX)rrrrrr$r!r!r"r& s     zWavLMFeedForward.forwardr'r!r!rr"r  rcs2eZdZd dedeffdd Zd d d ZZS) WavLMEncoderLayerTrGrbcntt|j|j|j|j|j|d|_t |j |_ t j |j|jd|_t||_t j |j|jd|_dSN)r^r_rVr`rarbrLrrrZr8num_attention_headsattention_dropoutr`max_bucket_distance attentionr6rTrrVrNrPrQr feed_forwardfinal_layer_normrrGrbrr!r"r+  zWavLMEncoderLayer.__init__NFrc Csl|}|j|||||d\}}}||}||}||}|||}||}||f}|r4||f7}|S)Nrrrsrtr)rrVrQrr) rr%rrrsrtr attn_residualroutputsr!r!r"r&:s"    zWavLMEncoderLayer.forwardTrr(r)r*rrrr&r+r!r!rr"r*rcs2eZdZd dedeffdd Zd dd ZZS) WavLMEncoderLayerStableLayerNormTrGrbcrrrrrr!r"rTrz)WavLMEncoderLayerStableLayerNorm.__init__NFcCsf|}||}|j||||d\}}}||}||}||||}||f}|r1||f7}|S)N)rrrsrt)rQrrVrr)rr%rrrsrtrrrr!r!r"r&cs   z(WavLMEncoderLayerStableLayerNorm.forwardr)NNFrr!r!rr"rSrrc.eZdZfddZ    dddZZS) WavLMEncodercft|_t|_tjjjd|_ t j |_ t fddtjD|_d|_dS)NrLcg|] }t|dkdqSr)rb)r.0irGr!r" sz)WavLMEncoder.__init__..FrrrGr,pos_conv_embedr6rNr8rPrQrTrrV ModuleListrangenum_hidden_layerslayersgradient_checkpointingrWrrr"rys   zWavLMEncoder.__init__NFTcCs>|rdnd}|r dnd}|dur"|ddd|jd}d||<||} || }||}||}tp;t|} d} t|j D]?\} } |rN||f}t g}|j o_| dko_||j jk}|rd| ru| ||| || d}|dd\}} |ryd}|r||df}qC|r||f}|stdd |||fDSt|||d S) Nr!rKrrrrNNNcs|] }|dur|VqdSrXr!rvr!r!r" z'WavLMEncoder.forward..last_hidden_stater% attentions)r{r|r~rrQrVrr enumeraterrlrandrrG layerdroprr rr%rrrtoutput_hidden_states return_dictall_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddings synced_gpusrsrlayerdropout_probabilityskip_the_layer layer_outputsr!r!r"r&sN         zWavLMEncoder.forwardNFFTr'r!r!rr"rxs rcr) WavLMEncoderStableLayerNormcr)NrLcrr)rrrr!r"rsz8WavLMEncoderStableLayerNorm.__init__..FrrWrrr"rs    z$WavLMEncoderStableLayerNorm.__init__NFTcCs<|rdnd}|r dnd}|dur"|ddd|jd}d||<||} || }||}tp6t|} d} t|jD]>\} } |rI||f}t g}|j oZ| dkoZ||j j k}|r_| ro| |||| d}|dd\}} |rsd}|r|||df}q>||}|r||f}|stdd |||fDSt|||d S) Nr!rKrrr)rrrtrsrcsrrXr!rr!r!r"rrz6WavLMEncoderStableLayerNorm.forward..r)r{r|r~rrVrrrrrlrrrGrrQrr rr!r!r"r&sH         z#WavLMEncoderStableLayerNorm.forwardrr'r!r!rr"rs rcs4eZdZdZfddZeddZddZZS)WavLMGumbelVectorQuantizerz Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH GUMBEL-SOFTMAX](https://huggingface.co/papers/1611.01144) for more information. cst|j|_|j|_|j|jdkr"td|jd|jdt t d|j|j|j|j|_ t |jd|j|j|_d|_dS)Nrz`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.rrKr)rrnum_codevector_groups num_groupsnum_codevectors_per_groupnum_varscodevector_dimrer6rkrlr codevectorsrRrO weight_proj temperaturerWrr!r"rs   z#WavLMGumbelVectorQuantizer.__init__cCs8|jdd}ttj|t|ddd }|S)NrrwgHz>rK)meanrlexprr)probsmarginal_probs perplexityr!r!r"_compute_perplexity(s (z.WavLMGumbelVectorQuantizer._compute_perplexityc Cs|j\}}}||}||||jd}|jrAtjj||j dd}| |}t j ||||jddd}| |}n$|jdd}|j|jd|ddd}||||jd}| |}|||d}|d|j} | |||j|jd} | d||d} | |fS)NrKT)tauhardrwrrx)r~rr}rrr6 functionalgumbel_softmaxrrtype_asrlsoftmaxr argmax new_zerosscatter_r{rrr) rr% batch_sizesequence_lengthr8codevector_probscodevector_soft_distrcodevector_idxcodevectors_per_grouprr!r!r"r&.s*       z"WavLMGumbelVectorQuantizer.forward) r(r)r*rr staticmethodr r&r+r!r!rr"r s   rc@sreZdZUeed<dZdZdZdZdZ dZ ddZ dd e e jefd eefd d Z dd ede jfddZdS)WavLMPreTrainedModelrGwavlm input_valuesTFc Cst|tr|jjjjddd|jjjtj |j dSt|t rItj j|j jddtd|j jd|j jdtj |j jddSt|trqtd|jj}tj j |jj| |dtj j |jj| |ddSt|tjr|jjjd|jjd|jdur|jjdSdSt|tjtjfr|jj|jjddSt|tjrtj |j|jdurt|j|j|jd}tj j |j| |ddSdSdS) zInitialize the weightsr[r)rstdrr)abNrx)rrrr2datanormal_rzero_r6inituniform_rr,r:rsqrtr- in_channels constant_rJrS in_featuresrRrGinitializer_rangerN GroupNormfill_r7kaiming_normal_r/)rmodulekr!r!r" _init_weights]s<         z"WavLMPreTrainedModel._init_weightsN input_lengths add_adaptercCsn|dur|jjn|}dd}t|jj|jjD] \}}||||}q|r5t|jjD] }||d|jj}q*|S)zH Computes the output length of the convolutional layers NcSstj|||dddS)Nfloor) rounding_moder)rldiv input_lengthr-strider!r!r"_conv_out_lengthszOWavLMPreTrainedModel._get_feat_extract_output_lengths.._conv_out_lengthr)rGrIzip conv_kernel conv_stridernum_adapter_layersadapter_stride)rrHrIrPr-rOrr!r!r" _get_feat_extract_output_lengths~sz5WavLMPreTrainedModel._get_feat_extract_output_lengthsfeature_vector_lengthrrcCs|jdddddf}|j||d}|tj}|jd}tj||f|j|jd}d|tj |jd|jd|df<| dgd dg }|S)NrKrwrIr)rrr)r) cumsumrVrrlrr~zerosrrrflipr)rrWrrrInon_padded_lengthsoutput_lengthsr+r!r!r""_get_feature_vector_attention_masks  "z7WavLMPreTrainedModel._get_feature_vector_attention_maskrX)r(r)r*r__annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attnrGrrlrrrrrVr^r!r!r!r"r2Ss* "  r2c&eZdZdfdd ZddZZS)WavLMNoLayerNormConvLayerrcsjt|dkr|j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ dS)Nrrr-rOr)rrrO in_conv_dim out_conv_dimr6r7rRrS conv_biasr:rrErFrrGlayer_idrr!r"rs  z"WavLMNoLayerNormConvLayer.__init__cCs||}||}|SrX)r:rFr$r!r!r"r&s  z!WavLMNoLayerNormConvLayer.forwardrr'r!r!rr"rgsrgcrf)WavLMLayerNormConvLayerrcs|t|dkr|j|dnd|_|j||_tj|j|j|j||j||j d|_ tj |jdd|_ t |j|_dS)NrrrhT)elementwise_affine)rrrOrirjr6r7rRrSrkr:rNrQrrErFrlrr!r"rs  z WavLMLayerNormConvLayer.__init__cCs:||}|dd}||}|dd}||}|S)Nr#rK)r:rIrQrFr$r!r!r"r&s     zWavLMLayerNormConvLayer.forwardrnr'r!r!rr"rosrocrf)WavLMGroupNormConvLayerrcst|dkr|j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ tj|j|jdd|_dS)NrrrhT)r num_channelsaffine)rrrOrirjr6r7rRrSrkr:rrErFrBrQrlrr!r"rs   z WavLMGroupNormConvLayer.__init__cCs"||}||}||}|SrX)r:rQrFr$r!r!r"r&s   zWavLMGroupNormConvLayer.forwardrnr'r!r!rr"rqsrqcs0eZdZdZfddZddZddZZS)WavLMFeatureEncoderz.Construct the features from raw audio waveformcstjdkr tddgfddtjdD}njdkr2fddtjD}n td jd t||_ d |_ d |_ dS) Ngrouprrmcsg|] }t|ddqS)rrv)rgrrr!r"rsz0WavLMFeatureEncoder.__init__..rr csg|]}t|dqS)rv)rorrr!r"rsz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT) rrfeat_extract_normrqrnum_feat_extract_layersrer6r conv_layersr_requires_grad)rrGryrrr"rs       zWavLMFeatureEncoder.__init__cCs|D]}d|_qd|_dS)NF) parameters requires_gradrzrparamr!r!r"_freeze_parameterss  z&WavLMFeatureEncoder._freeze_parameterscCs:|dddf}|jr|jrd|_|jD]}||}q|S)NT)rzrr|ry)rr4r% conv_layerr!r!r"r& s    zWavLMFeatureEncoder.forward)r(r)r*rrrr&r+r!r!rr"rts  rtcr)WavLMAdapterLayercs0ttj|jd|j|j|jdd|_dS)Nrr)rOr.)rrr6r7output_hidden_sizeadapter_kernel_sizerUr:rWrr!r"rs  zWavLMAdapterLayer.__init__cCs||}tjj|dd}|S)Nrrw)r:r6r$glur$r!r!r"r&#s zWavLMAdapterLayer.forwardr'r!r!rr"rs  rcr) WavLMAdaptercsptjjkrtjj|_tj|_nd|_|_t fddt j D|_ j |_ dS)Nc3s|]}tVqdSrX)rrrrr!r"r5sz(WavLMAdapter.__init__..)rrrr8r6rRprojrNproj_layer_normrrrTrrrWrrr"r+s     zWavLMAdapter.__init__cCsr|jdur|jdur||}||}|dd}|jD]}tj}|jr,||jkr0||}q|dd}|SrH)rrrIrnprandomrr)rr%r layerdrop_probr!r!r"r&8s      zWavLMAdapter.forwardr'r!r!rr"r*rrr~ mask_prob mask_lengthrr min_masksrucs|\}dkr tdkrtdddtjdfdd}|dur:|d n fd d t|D}tj |ft d }g} |} | d krZ|S|D];} || } tjj t | d| dd} t | d kr}d}n| d }t| tj| | tjd |g} | | q\t| } t| dddddf|| f} | || } t ddddf}t||| f|| }| |} | dkrd| | dk<t|| dd |S)an Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `csXt|}t|}|kr}|d|kr*t|dd}|S)z;Given input length, compute how many spans should be maskedrr)rmax)rNnum_masked_spanepsilonrrrr,r!r"compute_num_masked_spanos  z6_compute_mask_indices..compute_num_masked_spanNrKcsg|]}qSr!r!r)r,r!r"rsz)_compute_mask_indices..rrF)replace)rerrritemdetachrtolistrrZrchoicerlen concatenatermint32appendarrayrreshaperput_along_axis)r~rrrrrr+rrH spec_aug_maskspec_aug_mask_idxsmax_num_masked_spanrNrspec_aug_mask_idxdummy_mask_idxoffsetsr!rr"_compute_mask_indicesIs\    rcseZdZdeffdd ZddZddZ  dd ejd e ejd e ej fd d Z e     dde ej d e ej d e ejde ede ede edeeeffddZZS) WavLMModelrGcst|||_t||_t||_|jdks|jdkr)t t |j |_|jr2t||_nt||_|jr>t|nd|_|dS)Nr[)rrrGrtfeature_extractorrJfeature_projectionmask_time_probmask_feature_probr6rkrlrr8r<masked_spec_embeddo_stable_layer_normrencoderrrIradapter post_initrWrr!r"rs      zWavLMModel.__init__cCtdt|dSz Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.Nwarningswarn FutureWarningfreeze_feature_encoderrr!r!r"freeze_feature_extractor  z#WavLMModel.freeze_feature_extractorcCs|jdS Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rrrr!r!r"rsz!WavLMModel.freeze_feature_encoderNr%mask_time_indicesrrcCst|jdds |S|\}}}|dur|j|j||<n-|jjdkrK|jrKt||f|jj|jj ||jj d}t j ||j t jd}|j|j||<|jjdkr|jrt||f|jj|jj|jjd}t j ||j t jd}|dddfd|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://huggingface.co/papers/1904.08779). apply_spec_augmentTNr)rrrrr)rr)rrrrK)getattrrGryrrrrrrmask_time_lengthmask_time_min_masksrltensorrrrmask_feature_lengthmask_feature_min_masksexpand)rr%rrrr+r,r8mask_feature_indicesr!r!r"_mask_hidden_statess4 zWavLMModel._mask_hidden_statesr4rtrrruc Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}||}|dd}|dur8|j|jd|dd}||\}}|j |||d}|j |||||d} | d}|j dur_| |}|sk||f| ddSt ||| j | jd S) a/ mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict masked extracted features in *config.proj_codevector_dim* space. NrrFrX)rrrrrrtrrr)rextract_featuresr%r)rGrtruse_return_dictrrIr^r~rrrrWavLMBaseModelOutputr%r) rr4rrrrtrrrr%encoder_outputsr!r!r"r&s@     zWavLMModel.forward)NNNNNNN)r(r)r*rrrrrlrrrrrrrrrrr&r+r!r!rr"rsD   . rrzm WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). ) custom_introcseZdZddeeffdd ZddZddZd d Zd d Z e     dd ee j dee j dee dee dee dee j deeeffddZZS) WavLMForCTCN target_langcs~t|t||_t|j|_||_|j dur#t d|j dt |dr.|j r.|jn|j}t||j |_|dS)a/ target_lang (`str`, *optional*): Language id of adapter weights. Adapter weights are stored in the format adapter..safetensors or adapter..bin. Only relevant when using an instance of [`WavLMForCTC`] with adapters. Uses 'eng' by default. NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.rI)rrrr3r6rT final_dropoutrVr vocab_sizerer r<rIrr8rRlm_headr)rrGrrrr!r"r^s     zWavLMForCTC.__init__cCsv|j}|durt|jdddurtd|d|dur,t|jdddur,tddS|dur9|j|dddSdS)a' This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. Nadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T) force_load)rrrGreloggerinfo load_adapter)rrr!r!r" tie_weights{s zWavLMForCTC.tie_weightscCrrrNrrr!r!r"rrz$WavLMForCTC.freeze_feature_extractorcC|jjdSrr3rrrr!r!r"rz"WavLMForCTC.freeze_feature_encodercC|jD]}d|_qdSz Calling this function will disable the gradient computation for the base model so that its parameters will not be updated during training. Only the classification head will be updated. FNr3r{r|r}r!r!r"freeze_base_modelzWavLMForCTC.freeze_base_modelr4rrrtrrlabelsruc Cs||dur|n|jj}|dur||jjkrtd|jj|j|||||d}|d}||}||} d} |dur|durC|ntj |tj d}| | d tj } |dk} | d} || }tjj| dtjddd}tjjjd d tjj||| | |jj|jj|jjd } Wdn1swY|s| f|td}| dur| f|S|St| | |j|jd S) a labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`. Nz$Label values must be <= vocab_size: rrrrK)r4rrF)enabled)blank reduction zero_infinitylosslogitsr%r)rGrrrrer3rVrrl ones_likerrVrr masked_selectr6r$ log_softmaxfloat32rIbackendscudnnflagsctc_loss pad_token_idctc_loss_reductionctc_zero_infinity_HIDDEN_STATES_START_POSITIONr r%r)rr4rrrtrrrrr%rrrH labels_masktarget_lengthsflattened_targets log_probsoutputr!r!r"r&sN      zWavLMForCTC.forwardrXr)r(r)r*rrrrrrrrrlrrrrr r&r+r!r!rr"rXs6  rz WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. cseZdZfddZddZddZddZe dd ee j d ee j d ee d ee dee dee j de e effddZZS)WavLMForSequenceClassificationcst|t|dr|jrtdt||_|jd}|jr*t t |||_ t |j|j|_t |j|j|_|dS)NrIz\Sequence classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrr<rIrerr3ruse_weighted_layer_sumr6rkrlrm layer_weightsrRr8classifier_proj_size projector num_labels classifierrrrG num_layersrr!r"rs    z'WavLMForSequenceClassification.__init__cCrrrrr!r!r"r rz7WavLMForSequenceClassification.freeze_feature_extractorcCrrrrr!r!r"rrz5WavLMForSequenceClassification.freeze_feature_encodercCrrrr}r!r!r"rrz0WavLMForSequenceClassification.freeze_base_modelNr4rrrtrrrrucCsz|dur|n|jj}|jjrdn|}|j|||||d}|jjrB|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |}|durV|jdd} n+||jd|} | ddd|jd} d || <|j dd| j dd dd} || } d}|durt}|| d|jj| d}|s| f|td}|dur|f|S|St|| |j|jd S)  input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`WavLMProcessor.__call__`] for details. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTrrrwrKrrr[r)rGrrr3rrlstackr6r$r'rr}rrrr^r~r{r|rrrr r%r)rr4rrrtrrrrr% norm_weights pooled_output padding_maskexpand_padding_maskrrloss_fctrr!r!r"r&&sH    z&WavLMForSequenceClassification.forwardr)r(r)r*rrrrrrrlrrrrr r&r+r!r!rr"rs4   rcseZdZfddZddZddZddZe dd ee j d ee j d ee j d ee dee dee de e effddZZS) WavLMForAudioFrameClassificationcszt|t|dr|jrtdt||_|jd}|jr*t t |||_ t |j|j|_|j|_|dS)NrIz_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rrr<rIrerr3rrr6rkrlrmrrRr8rr init_weightsrrr!r"rns    z)WavLMForAudioFrameClassification.__init__cCrrrrr!r!r"r~rz9WavLMForAudioFrameClassification.freeze_feature_extractorcCrrrrr!r!r"rrz7WavLMForAudioFrameClassification.freeze_feature_encodercCrrrr}r!r!r"rrz2WavLMForAudioFrameClassification.freeze_base_modelNr4rrrrtrrrucCs|dur|n|jj}|jjrdn|}|j|||||d}|jjrB|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |} d} |durht} | | d|jtj| d|jdd} |su| f|td} | St| | |j|jd S) rNTrrrwrKr)axisr)rGrrr3rrlrr6r$r'rr}rrrrr(r r%r)rr4rrrrtrrrr%r rrr rr!r!r"r&s: (z(WavLMForAudioFrameClassification.forwardr)r(r)r*rrrrrrrlrrrrr r&r+r!r!rr"rls4   rcs&eZdZdfdd ZddZZS) AMSoftmaxLoss>@皙?csBt||_||_||_tjt||dd|_ t |_ dS)NT)r|) rrscalemarginrr6rkrlrandnr2rr)r input_dimrrrrr!r"rs zAMSoftmaxLoss.__init__c Csx|}tjj|jdd}tjj|dd}t||}||j}tj||j }|j t | ||}| ||}|S)Nrrwr)flattenr6r$ normalizer2rlmmrone_hotrrrrr) rr%rr2 cos_thetapsionehotrrr!r!r"r&s   zAMSoftmaxLoss.forward)rrr'r!r!rr"rsrcs4eZdZdfdd ZdejdejfddZZS) TDNNLayerrcsvt|dkr|j|dn|j||_|j||_|j||_|j||_t |j|j|j|_ t |_ dS)Nrr)rrtdnn_dimrirj tdnn_kernelr- tdnn_dilationdilationr6rRkernelReLUrFrlrr!r"rs "   zTDNNLayer.__init__r%rucCstr ddlm}trt|j|rtd|dd}|jj |j |j |j dd}t jj|||jj|jd}|dd}||}|S)Nr) LoraLayerzDetected LoRA on TDNNLayer. LoRA weights won't be applied due to optimization. You should exclude TDNNLayer from LoRA's target modules.rr)r#)rpeft.tuners.lorar&rr$rrrIr2r}rjr-rir6r$conv1drr#rF)rr%r&r2r!r!r"r&s      zTDNNLayer.forwardrn)r(r)r*rrlrr&r+r!r!rr"rs rzi WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification. cseZdZfddZddZddZddZd eej e ffd d Z e dd e ejde ejde ede ede ede ejdeeeffddZZS)WavLMForXVectorcstt|_jd}jrtt |||_ t j j d|_fddttj D}t||_t j ddj|_t jj|_tjj|_|dS)Nrrcsg|]}t|qSr!)rrrr!r"rsz,WavLMForXVector.__init__..rKr)rrrr3rrr6rkrlrmrrRr8r rrrrtdnnxvector_output_dimrrrr objectiver)rrGr tdnn_layersrrr"rs     zWavLMForXVector.__init__cCrrrrr!r!r"r&rz(WavLMForXVector.freeze_feature_extractorcCrrrrr!r!r"r2rz&WavLMForXVector.freeze_feature_encodercCrrrr}r!r!r"r9rz!WavLMForXVector.freeze_base_modelrHcCs&dd}|jjD]}|||d}q|S)z? Computes the output length of the TDNN layers cSs|||dS)Nrr!rMr!r!r"rPFszBWavLMForXVector._get_tdnn_output_lengths.._conv_out_lengthr)rGr!)rrHrPr-r!r!r"_get_tdnn_output_lengthsAs z(WavLMForXVector._get_tdnn_output_lengthsNr4rrrtrrrrucCs|dur|n|jj}|jjrdn|}|j|||||d}|jjrB|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |}|jD]} | |}qN|durf|jdd} |jdd} nC||j dd} || }g} g} t|D]"\}}| ||d|fjdd| ||d|fjddq|t| } t| } tj| | gdd}||}||}d}|dur|||}|s||f|td}|dur|f|S|St||||j|jdS) rNTrrrwrKr)rr embeddingsr%r)rGrrr3rrlrr6r$r'rr}rrr*rr5rVr.rrrrrr,rr%r)rr4rrrtrrrrr%r  tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrlengthstatistic_poolingoutput_embeddingsrrrr!r!r"r&Ps\           zWavLMForXVector.forwardr)r(r)r*rrrrrrlrrr.rrrrrrr&r+r!r!rr"r) s6   r))rrrr)rr2r#)Jrrtypingrrnumpyrrltorch.nnr6torch.nn.functionalr$rr activationsrintegrations.deepspeedrintegrations.fsdprmodeling_layersr modeling_outputsr r r r rrmodeling_utilsrr;rrrconfiguration_wavlmr get_loggerr(rModulerr,rJrZrrrrrrr2rgrorqrtrrrrrrndarrayrrrrrrrrrr)__all__r!r!r!r"s           -')%JKFV&#  wsi