o isN @sTddlZddlZddlmZdejd<ejejeje dddl Z ddl Z ddl Z ddl mZddlZedddlmZddlZddlZddlZddlZddlmZdd lmZmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%m&Z&ia'ia(ej)*rdnej+*rdn ej,j-*rdndZ.ej/dkrddinddiZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;d ZdBd#d$Z?ddd%e.dfd&d'Z@daAe.dfd(eBfd)d*ZCdCd+d,ZDdDd(eBfd.d/ZEe6d%e9d-e.fd0d1ZFdEd3d4ZGeHfd5d6ZIe6eHee7e8e:e;ee.f d7d8ZJdeddddd9d:dddd;f dd?ZLd@dAZMdS)FN)ThreadPoolExecutor1PYTORCH_ENABLE_MPS_FALLBACKz/../../third_party/BigVGAN/)filesAgg)hf_hub_download) AudioSegmentsilence)pipeline)Vocos)CFM)convert_char_to_pinyin get_tokenizercudaxpumpscpu) delete_on_closeFdeletei]divocosg?g333333?euler g@gg?cCsg}d}td|}|D]D}t|dt|d|kr4||r0t|dddkr0|dn|7}q |r=|||rNt|dddkrN|dn|}q |rZ|||S)a Splits the input text into chunks, each with a maximum number of characters. Args: text (str): The text to be split. max_chars (int): The maximum number of characters per chunk. Returns: List[str]: A list of text chunks. u*(?<=[;:,.!?])\s+|(?<=[；：，。！？])utf-8 )resplitlenencodeappendstrip)text max_charschunks current_chunk sentencessentencer.L/home/ubuntu/.local/lib/python3.10/site-packages/f5_tts/infer/utils_infer.py chunk_textIs   ,(r0rc Cs@|dkre|rtd||d}|d}ntdd}t||dd}t||d d}t|}tj|d d d } d dlm} t|j | rWdd|j j D} | | || ||}|S|dkrzd dlm} Wn ty~tdYnw|r| jj|dd}n | jjdd|d}|||}|S)NrzLoad vocos from local path z /config.yamlz/pytorch_model.binz8Download Vocos from huggingface charactr/vocos-mel-24khzzcharactr/vocos-mel-24khzz config.yaml)repo_id cache_dirfilenamezpytorch_model.binrT map_location weights_onlyr)EncodecFeaturescSsi|] \}}d||qS)zfeature_extractor.encodec.r.).0keyvaluer.r.r/ ysz load_vocoder..bigvgan)r<zSYou need to follow the README to init submodule and change the BigVGAN source code.F)use_cuda_kernelz$nvidia/bigvgan_v2_24khz_100band_256x)r=r2)printrr from_hparamstorchloadvocos.feature_extractorsr7 isinstancefeature_extractorencodec state_dictitemsupdateload_state_dictevaltothird_party.BigVGANr< ImportErrorBigVGANfrom_pretrainedremove_weight_norm) vocoder_nameis_local local_pathdevice hf_cache_dir config_path model_pathr1vocoderrFr7encodec_parametersr<r.r.r/ load_vocoderhsD         rZrTcCsR|durd|vrtj|jdkrtjdstjntj}tdd||da dS)Nr[ZLUDA]zautomatic-speech-recognitionzopenai/whisper-large-v3-turbo)model torch_dtyperT) r@rget_device_propertiesmajorget_device_nameendswithfloat16float32r asr_pipe)rTdtyper.r.r/initialize_asr_pipelines rgcCs>tdur ttdt|dd|rd|dnddiddd S) NrT transcribe)tasklanguagerlF)chunk_length_s batch_sizegenerate_kwargsreturn_timestampsr()rergrTr') ref_audiormr.r.r/rks rkTc Cs|durd|vrtj|jdkrtjdstjntj}||}| dd}|dkr.model_state_dict)zmel_spec.mel_stft.mel_scale.fbz$mel_spec.mel_stft.spectrogram.window)r@rr_r`rarbrcrdrKr#safetensors.torchrurArGrI empty_cache) r] ckpt_pathrTrfuse_ema ckpt_typeru checkpointr9r.r.r/load_checkpoints>        rc Cs|dkr ttdd}d}td|td|td|dt||\} } t|di|| td ttt t tt |d t|d | d  |} |d krMt jnd} t| ||| |d} | S)Nrf5_ttszinfer/examples/vocab.txtcustomz vocab : ztoken : zmodel :  )text_num_embedsmel_dim)n_fft hop_length win_lengthn_mel_channelstarget_sample_rate mel_spec_type)method) transformermel_spec_kwargs odeint_kwargsvocab_char_mapr<)rfrr.)strrjoinpathr>rr rdictrrrrrKr@rdr) model_cls model_cfgrr vocab_file ode_methodrrT tokenizerr vocab_sizer]rfr.r.r/ load_models6    rcCs\tj||d}||d}|j}t|D] }|j|krn|d8}q|dt|d}|S)N)silence_thresholdgMbP?)r detect_leading_silenceduration_secondsreverseddBFSint)audiornon_silent_start_idxnon_silent_end_durationms trimmed_audior.r.r/remove_silence_edgess    rc CsD|dt|d}|}t|}Wdn1swY|tvr1|dt|}ntjdddit }|j }Wdn1sIwYt |} t j | ddddd } t jd d } | D]} t| d kr{t| | d kr{|dn| | 7} qet| d krt j | ddddd } t jd d } | D]} t| d krt| | d kr|dn| | 7} q| } t| d kr| dd } |dt| t jdd } | j|dd|}|t|<|s|tvr|dt|}n|dt|}|t|<n|d|ds|ds|dr|d7}n|d7}td|||fS)NzConverting audio...rbz,Using cached preprocessed reference audio...suffixz.wavr min_silence_lensilence_thresh keep_silence seek_steprdurationipi.z&Audio is over 12s, clipping short. (1)riz&Audio is over 12s, clipping short. (2)z&Audio is over 12s, clipping short. (3)2wavformatzUsing cached reference text...z;No reference text provided, transcribing reference audio...zUsing custom reference text...z. u。rsr!z ref_text r.)openreadhashlibmd5 hexdigest_ref_audio_cachetempfileNamedTemporaryFiletempfile_kwargsnamer from_filer split_on_silencesilentr$rexportr'_ref_text_cacherkrbr>) ref_audio_origref_text show_info audio_file audio_data audio_hashrrf temp_pathasegnon_silent_segsnon_silent_wavenon_silent_segr.r.r/preprocess_ref_audio_text(sj                 rcCst|\}}tt|d|jd|d|jd|| }t||d}t|D] \}}td||q,td|dt|dt t ||f|||||||| | | | | ||d S) Nrr)r)z gen_text rzGenerating audio in z batches...) rprogress target_rmscross_fade_durationnfe_step cfg_strengthsway_sampling_coefspeed fix_durationrT) torchaudiorArr$r%shaper0 enumerater>nextinfer_batch_process)rrrgen_text model_objrXrrrrrrrrrrrTrsrr)gen_text_batchesir.r.r/ infer_process~s46 rrr ic(#s|\}jddkrtjdddttt kr) |tkr8tj|t}| |g}g}t d ddkrP d  fdd  r|durn| |n|D] }|D]}|VqvqpdSt 6fd d |D}|dur| |n|D]}|}|rt|\}}||||qWdn1swY|rB|dkrt|}nf|d}tdt |D]Z}|}||}t|t} t| t |t |} | dkrt||g}q|| d}!|d| }"tdd| }#tdd| }$|!|#|"|$}%t|d| |%|| dg}&|&}qtj|dd }'|t|'fVdSdtdfVdS) Nrr T)dimkeepdimrrr!c 3s }t|ddkrd}|g}t|}jdt}dur+ttt}ntd}t|d}|t||||}tj || d\}} ~ | tj }|dd|dddf}| ddd}d kr| |} nd kr |}  kr|  } | } rtdt| D] } | | | tfVqn|d} ~| | fVWddSWddS1swYdS) Nrrg333333?r)condr(rstepsrrrr rr<)r$r%r rrrrr@inference_modesamplerKrdpermutedecodesqueezernumpyrange) r local_speed text_listfinal_text_list ref_audio_lenr ref_text_len gen_text_len generated_generated_wavej generated_cpu)rr chunk_sizerrrrrrmsr streamingrrrXr.r/ process_batchsT       "z*infer_batch_process..process_batchcsg|]}|qSr.)submit)r8r)executorrr.r/ sz'infer_batch_process..)axis)rr@meansqrtsquarerr transformsResamplerKr$r%tqdmrresultrr&np concatenaterrminlinspace)(rrrrrrXrrrrrrrrrrTrrr resamplergenerated_waves spectrogramsrchunkfuturesfuturerrgenerated_mel_spec final_waver prev_wave next_wavecross_fade_samples prev_overlap next_overlapfade_outfade_incross_faded_overlapnew_wavecombined_spectrogramr.)rrrrrrrrrrrrrrrrXr/rsr  &3        rcCsRt|}tj|ddddd}tjdd}|D]}||7}q|}|j|dd dS) Nrrirrrrrr)rrr rrr)r3rrrrr.r.r/ remove_silence_for_generated_wavIs    r'cCs:tjddtj|dddtt|tdS)N)r)figsizelowerauto)originaspect)pltfigureimshowcolorbarsavefigclose) spectrogrampathr.r.r/save_spectrogramXs   r6)r)N)NT)r)Nossysconcurrent.futuresrenvironr5r&dirnameabspath__file__rr"rimportlib.resourcesr matplotlibusematplotlib.pylabpylabr.rrr@rrhuggingface_hubrpydubrr transformersr rr f5_tts.modelr f5_tts.model.utilsr rrrr is_availablerbackendsrrT version_inforrrrrrrrrrrrrrrr0rZrerrgrkrrrr>rrrr'r6r.r.r.r/s  "          , 4 ) \ 9