o iN@sdZddlZddlmZddlmZmZddlmZm Z ddl m Z ddl m Z mZmZmZdd lmZmZer=ddlZerDddlZGd d d e d d ZGddded d ZGdddeZdgZdS)zProcessor class for DiaN)Path)OptionalUnion) AudioInputmake_list_of_audio) BatchFeature) AudioKwargsProcessingKwargsProcessorMixinUnpack)is_soundfile_availableis_torch_availablec@s:eZdZUeed<eed<eed<eeed<eed<dS)DiaAudioKwargs bos_token_id eos_token_id pad_token_id delay_pattern generationN)__name__ __module__ __qualname__int__annotations__listboolrrc/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/transformers/models/dia/processing_dia.pyr"s   rF)totalc@s@eZdZUeed<dddddddgd dd d d d idZdS)DiaProcessorKwargs audio_kwargsTrightF)padding padding_sideadd_special_tokensiii) r iD)rrrrr sampling_ratereturn_tensorspt) text_kwargsr common_kwargsN)rrrrr _defaultsrrrrr*s  rcsVeZdZdZdZdZdZfddZ  d*d ee e e fd e e d e e d eefd dZ d+ddde ed eede dfddZ d+ddde ed eeddfddZddd eedefddZd e dee ee ee effd eefddZe d,dededed e ed!e ded"f d#d$Zed dd%ed&ed'ed"ddf d(d)ZZS)- DiaProcessora Constructs a Dia processor which wraps a [`DiaFeatureExtractor`], [`DiaTokenizer`], and a [`DacModel`] into a single processor. It inherits, the audio feature extraction, tokenizer, and audio encode/decode functio- nalities. See [`~DiaProcessor.__call__`], [`~DiaProcessor.encode`], and [`~DiaProcessor.decode`] for more information. Args: feature_extractor (`DiaFeatureExtractor`): An instance of [`DiaFeatureExtractor`]. The feature extractor is a required input. tokenizer (`DiaTokenizer`): An instance of [`DiaTokenizer`]. The tokenizer is a required input. audio_tokenizer (`DacModel`): An instance of [`DacModel`] used to encode/decode audio into/from codebooks. It is is a required input. DiaFeatureExtractor DiaTokenizerDacModelcstj|||ddS)N)audio_tokenizer)super__init__)selffeature_extractor tokenizerr7 __class__rrr9RszDiaProcessor.__init__NFtextaudio output_labelskwargsc' Kststd|durtd|jtfi|}|d}|d}|d}|dd} | dkr7t|jjd i} t|trB|g}nt|t t frRt d d |DsVtd |j |fi|} | | |d d} |dd} |dd}|dd}|dd}| dus|dus|dus| durtd|r|rtd|d|d| djd}t| }t| }|durt|}|j|fi|}t|jjj}|ddjd|}g}g}t|d|dD]\}}|jj}t|jdd||}||}||}t!|ddd|f|jj}|j |j!"dd}Wdn 1s,wY|s?tj#j$j%|d d!|d"}tj#j$j%|dd|ddddfd!| d"}|d|}||r^dnd7}tj&dg|dg|tj'd#dddf} |(||(| qtj)|dd}tj)|dd}n |rtj*|d|f| tj'd#}tj+|d|ftj'd$}ntd%||jdkrtd&|d'|jdd(|jd}!|!|}"|j,||!|| d)d*}#tj*||!|f|tj-d+}$||$ddd|"f<|j.|$|| |#d,}%| |%|d-|rL| d./ddddf}&d/|&|&|k<d/|&|&| k<|&"dd0||d1'| d0<| d.ddddf| d.<| d1ddddf| d1<t2| | d2S)3a Main method to prepare text(s) and audio to be fed as input to the model. The `audio` argument is forwarded to the DiaFeatureExtractor's [`~DiaFeatureExtractor.__call__`] and subsequently to the DacModel's [`~DacModel.encode`]. The `text` argument to [`~DiaTokenizer.__call__`]. Please refer to the docstring of the above methods for more information. zThe `DiaProcessor` relies on the `audio_tokenizer` which requires `torch` but we couldn't find it in your environment. You can install torch via `pip install torch`.Nz0You need to specify the `text` input to process.r0r r1r.r/z% only supports `return_tensors='pt'`.css|]}t|tVqdSN) isinstancestr).0trrr }sz(DiaProcessor.__call__..zAInvalid input text. Please provide a string, or a list of stringsrrrrrTzTo enable processing for Dia, we need the `bos_token_id`, `eos_token_id`, `pad_token_id`, and `delay_pattern`. You may have accidentally overwritten one of those.z9Labels with `generation` is incompatible, got generation=z, output_labels=. input_idsr padding_mask input_valuesdim.)rrrrPrrconstant)padmodevaluedtype)sizerWz;If you try to train, you should provide audio data as well.zNNeed the same amount of samples for both text and audio, but got text samples=z and audio samples = z instead.Fbszseq_len num_channelsrrevert) fill_valuerWr@rrprecomputed_idx)decoder_input_idsdecoder_attention_maskrailabelsrb)data tensor_type)3r ValueError _merge_kwargsrpopr>rrDrErtupleallr<updateshapelenmaxrr;mathprodr7configdownsampling_ratioszip hop_lengthceilsumtorchno_gradtodeviceencode audio_codes transposenn functionalrStensorlongappendcatfullones build_indicesrapply_audio_delayclonereshape contiguousr)'r:r?r@rArB output_kwargsr0r r1r.rd encodingsraudio_bos_token_idaudio_eos_token_idaudio_pad_token_idr batch_sizer\ max_delay input_audioscompression_ratemax_encoded_sequence_lenrarbrK base_pad_lencurrent_audio_lenencoded_sequence_len padding_lenrJnum_valid_inputsattention_mask max_seq_len max_audio_lenr`prefilldelayed_decoder_input_idsrcrrr__call__Us             ,      $ zDiaProcessor.__call__ra torch.Tensoraudio_prompt_lenreturncKs|jtfi|}|d}|dd}|dd}|dd}|dus+|dus+|dur/td|durHtj||jtjd}|d|j d} n|dddddf|kj d d } |j d |dddddf|kj d d d } |j \} } } |j | | | |d d }|j |d d |d d d}g}t<t| j dD]+}||dd| || |fd}||jj}|jj|dj}||qWd|S1swY|S)a Decodes a batch of audio codebook sequences into their respective audio waveforms via the `audio_tokenizer`. See [`~DacModel.decode`] for more information. Args: decoder_input_ids (`torch.Tensor`): The complete output sequence of the decoder. audio_prompt_len (`int`): The audio prefix length (e.g. when using voice cloning). r rNrrzTo enable decoding for Dia, we need the `bos_token_id`, `pad_token_id`, and `delay_pattern`. You may have accidentally overwritten one of those.)rzrWrrLrNrPTrYr_rQ)N.)r|)rgrrhrfrwrrzrexpandrlrvrrr}rxrangeryr7decode audio_valuescpusqueezer)r:rarrBrr rrrstart_of_generation_idxend_of_generation_idxrZr[r\r`output_sequencesaudiosioutput_iaudio_irrr batch_decodes^   ".  "  zDiaProcessor.batch_decodecKs<|jddkrtd|jdd|j||fi|dS)z Decodes a single sequence of audio codebooks into the respective audio waveform via the `audio_tokenizer`. See [`~DacModel.decode`] and [`~DiaProcessor.batch_decode`] for more information. rrPz5Expecting a single output to be decoded but received z samples instead.)rlrfr)r:rarrBrrrrIs  zDiaProcessor.decoderbcKsH|jtfi|}|d}|dd}|durtd|jdt|S)z0Utility function to get the audio prompt length.r rNzTo enable the utility of retrieving the prompt length for Dia, we need the `delay_pattern`. You may have accidentally overwritten this.rP)rgrrhrfrlrn)r:rbrBrr rrrrget_audio_prompt_lenZs z!DiaProcessor.get_audio_prompt_len saving_pathc Kststdt|}t|ttfr|g}nt|ttfr&tdd|Ds*t dt |t |kr6t d|j t fi|}|d}|d}t ||D]\}}t|tjr^|}t|||qLdS)Nz/Please install `soundfile` to save audio files.css|] }t|ttfVqdSrC)rDrEr)rFprrrrHsz*DiaProcessor.save_audio..zAInvalid input path. Please provide a string, or a list of stringsz5The number of audio and saving paths must be the samer r-)r ImportErrorrrDrErrrirjrfrmrgrrsrwTensorrfloatnumpysfwrite) r:r@rrBrr r- audio_valuerrrr save_audioos*  zDiaProcessor.save_audiorZr[r\rr])rrc Cstj|tjd}tj|tjddddf||d}|s*||ddddf}n ||ddddf}t|d|d}tj|tjdddddf|||}tj|tjdddddf|||} tj|d|d| dgdd} || fS)a Precompute (sequence_idx, all_idx) so that out[seq, channel] = in[seq - delay[channel], channel] or in[seq, channel] = out[seq + delay[channel], channel] if `revert`. Negative sequence_idx => BOS; sequence_idx >= seq_len => PAD. rVN).NrrPrLrN) rwrint32arangerclampstackrr) rZr[r\rr] delay_array sequence_idxvalid_sequence_idx batch_idx channel_idxall_idxrrrrs (((zDiaProcessor.build_indicesrrr`c Cs|j}|\}}||}||}tj|dd\}}} |||| f|} |dk} ||jdk} t| |t| || } | S)a Applies or reverts the delay pattern to batched audio tokens using precomputed indices, inserting BOS where sequence_idx < 0 and PAD where sequence_idx >= seq_len. Args: audio: audio tokens of shape [bsz, seq_len, num_channels] pad_token_id: the PAD token bos_token_id: the BOS token precomputed_idx: from `build_indices` Returns: final_audio: delayed or reverted audio tokens of shape [bsz, seq_len, num_channels] rLrNrrP)rzryrwunbindviewrXrlwhere)r@rrr`rzrrrrrgathered_audiomask_bosmask_pad final_audiorrrrs  zDiaProcessor.apply_audio_delay)NFrC)F)rrr__doc__feature_extractor_classtokenizer_classaudio_tokenizer_classr9rrErrrrr rrrrrrrr staticmethodrirr __classcell__rrr=rr3>s  1 J   " "r3)rropathlibrtypingrr audio_utilsrrfeature_extraction_utilsrprocessing_utilsr r r r utilsr rrw soundfilerrrr3__all__rrrrs&