o ib@sdZddlZddlmZmZmZmZmZddlZddl m Z ddl m m Z ddlZddlZddlmZddlmZz ddlmZdZWn eyPdZYnwdd lmZdd lmZd d lmZmZd d lmZd dl m!Z!d dl"m#Z#m$Z$m%Z%d dl&m'Z'd dl(m)Z)m*Z*ddl+m,Z,m-Z-GdddeZ.GdddeZ/GdddeZ0GdddeZ1e02Gddde j3Z4dS)a VoxCPM: A Tokenizer-free speech generation model This module contains the main VoxCPM model implementation, including configuration classes and the core VoxCPMModel for text-to-speech generation. Copyright 2025 OpenBMB Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. N)TupleUnion GeneratorListOptional) rearrange) BaseModel) load_fileTF)tqdm)LlamaTokenizerFast)AudioVAEAudioVAEConfig)ScalarQuantizationLayer)"apply_lora_to_named_linear_modules) CfmConfig UnifiedCFM VoxCPMLocDiT) VoxCPMLocEnc)MiniCPM4Config MiniCPMModel) get_dtypemask_multichar_chinese_tokensc@sJeZdZUdZeed<dZeed<dZeed<dZeed<d Z eed <d S) VoxCPMEncoderConfig hidden_dimffn_dim num_heads num_layersN kv_channels) __name__ __module__ __qualname__rint__annotations__rr r"r#r)r)G/home/ubuntu/.local/lib/python3.10/site-packages/voxcpm/model/voxcpm.pyr1s    rc@sReZdZUdZeed<dZeed<dZeed<dZeed<d Z eed <e ed <d S) VoxCPMDitConfigrrrrrr r!r"Nr# cfm_config) r$r%r&rr'r(rr r"r#rr)r)r)r*r+9s      r+c@seZdZUeed<dZeed<dZeed<dZeed<dZ eed <d Z eed <e ed <e ed <dZ eeed<dZeed<dZeed<dZeed<dZeed<dS) VoxCPMConfig lm_configr patch_size@feat_dimresidual_lm_num_layersscalar_quantization_latent_dim scalar_quantization_scaleencoder_config dit_configNaudio_vae_configr max_lengthcudadevicebfloat16dtypeF dit_mean_mode)r$r%r&rr(r/r'r1r3r5r7rr+r:rrr;r=strr?r@boolr)r)r)r*r-Cs         r-c@seZdZUdZeed<dZeed<dZeed<dZe ed<dZ e ed<d Z e ed <gd Z eeed <gd Zeeed <gdZeeed<dS) LoRAConfigF enable_lm enable_dit enable_projrralphadropout)q_projv_projk_projo_projtarget_modules_lmtarget_modules_dit)enc_to_lm_projlm_to_dit_projres_to_dit_projtarget_proj_modulesN)r$r%r&rDrBr(rErFrHr'rIrKfloatrPlistrArQrUr)r)r)r*rCUs      rCcs2eZdZ dadedededeffdd Zdd Zdbd e fd d Z dd dde j de j de j de j de j de j de j de de fddZddZde j fddZdee j ddffd d!Ze  " " # $ % & ' ( dcd)ed*ed+ed,ed-ed.ed/e d0e d1ed2e d3e dee j ddffd4d5Ze d*ed+efd6d7Zd8ed9ed:e j fd;d<Zdee j e j e j ffd=d>Zdeee j e j ee j fddffd?d@Ze  # $ % & ' ( ddd)edAed,ed-ed.ed/e d0e d1ed2e d3e deee j e j ee j ee j ffddffdBdCZdee j e j ffdDdEZ deee j ee j fddffdFdGZ!e  # $ % & 'dedHe j de j dIe j dJe j d,ed-ed.ed/e d3e dKedeee j ee j ee j ffddffdLdMZ"e#dfdOedPe dQe defdRdSZ$dTdUZ%dadVedWefdXdYZ&dZe fd[d\Z'd]d^Z(defd_d`Z)Z*S)g VoxCPMModelNconfig tokenizer audio_vae lora_configcst||_||_|j|_|j|_|j|_tj s)tj j r&d|_nd|_t d|jd|jj t|j|_|jd|j|jt|jj t||_d|_d|_|jjdd }|j|_d |_t||_|jd|j|jt|jj |jjdd }|jj|_|jj |_!|jj"|_#|jj$|_|jj%|_%d |_t&||jd |_'|jjdd }|j(j|_|j(j |_!|j(j"|_#|j(j$|_|j(j%|_%d |_t)|j|j(j*t+||jd |j,d |_-t.|jj|jj|j/|j0|_1t23|jj|jj|_4t23|jj|j(j|_5t23|jj|j(j|_6t23|jj|jj|_7t28|_9t2j3|jjddd|_:t2j;dd|_<||_=|j>|_>|j?|_?|jdur@|@dSdS)NmpscpuzRunning on device: z , dtype: refT)deepr) input_dim) in_channels)rc cfm_params estimator mean_moder F)biasnone) reduction)Asuper__init__rYr\r1r/r=torchr< is_availablebackendsr]printr?rr.base_lm setup_cacher;rrtext_tokenizeraudio_start_tokenaudio_end_token model_copyr3num_hidden_layers vocab_size residual_lmr8r hidden_sizerintermediate_sizer num_attention_headsr"r#r feat_encoderr9rr,rr@ feat_decoderrr5r7 fsq_layernnLinearrRrSrT stop_projSiLU stop_actn stop_headCrossEntropyLoss stop_lossr[ chunk_size sample_rate _apply_lora)selfrYrZr[r\residual_lm_configr8decoder_config __class__r)r*rkisz                    zVoxCPMModel.__init__cCs|j}t|j|j|jd}|jr$|j|jfD] }t|fd|j i|q|j r4t|j j fd|j i||jr\ddlm}|jD]}t||d}t|tjr[t|||dd|i|q@dSdS)u$注入 LoRA 到 LM / DiT / 投影层)rHrIrKtarget_submodule_namesr  LoRALinearNbaser))r\dictrHrIrKrDrprxrrPrEr}rerQrFmodules.layers.lorarrUgetattr isinstancerrsetattr)rcfg lora_kwargslmr attr_namemoduler)r)r*rs8    zVoxCPMModel._apply_loraFdisablec Cs|r|SzH|jdkrtdzddl}Wntdtj|jjddd|j_tj|jjddd|j_tj|jddd|_tj|j j ddd|j _ W|St yf}zt d|WYd}~|Sd}~ww) Nr<z0VoxCPMModel can only be optimized on CUDA devicerztriton is not installedzreduce-overheadT)mode fullgraphz"Warning: torch.compile disabled - ) r= ValueErrortritonrlcompilerp forward_steprxr|r}re Exceptionro)rrrer)r)r*optimizes&  zVoxCPMModel.optimizerJ)progresssample_generate text_tokens text_mask audio_feats audio_mask loss_mask position_idslabelsrrc&Cs~|j|jtjd}|j|j|d}|j|j|d}|j|j|d}|j|j|d}|j|jtjd}|j\} } } } ||}||}t|j j dd}t|j j dds^d}|j ||}| d|| d|}|j |dd\}}||}||| d|| d}tjt|ddd d ddf|ddddddffd d }|| d|}|j|dd\}}||}tjt|ddd d ddf|ddddddffd d }||||}t|d }|}t||d }tjt|ddd d df|dddddffd d }t||d }| dd d |j}t|d|}|jj|d d||d d|d d|d}||||}||d d|}tj| dd} || | }!d}"| r|d d}#|j||j|#t!|j j"j#dr|j j"j#j$ndd}$t|$d dd| |jd}"t|d| |jd}%||!|%|"dS)Nr? scale_emb?use_mupFT inputs_embeds is_causalrrdimzb t c -> (b t) czb t p d -> (b t) p d.zb t p -> (b t) p 1r )condtgt_maskr)mininference_cfg_rate )mur/r n_timestepsz(b t) d p -> b d (t p)bpz(b t) p d -> b d (t p))z loss/diffz loss/stopfeat_gt feat_pred)%tor=rllong_dtypeshaper|rRrrYr.rp embed_tokens unsqueezer~cat zeros_likerxrSrTrrepeatr/r} compute_loss transpose contiguousrrrrclampsumhasattrr9r,r)&rrrrrrrrrrBTPD feat_embedr text_embedcombined_embed enc_outputs_ lm_hiddenresidual_inputsresidual_outputsresidual_hidden dit_hidden target_dtyper feat_cond loss_seq_mask diff_loss stop_logits stop_lossesdenomrrfeat_cond_for_sample feat_pred_seqfeat_gt_tensorr)r)r*forwards   "D8 0zVoxCPMModel.forwardcCs t|jjSN)rrYr?rr)r)r*rJs zVoxCPMModel._dtypereturncOt|j|ddi|SN streamingF)next _generaterargskwargsr)r)r*generateNzVoxCPMModel.generatecO|j|ddi|SNrT)rrr)r)r*generate_streamingQzVoxCPMModel.generate_streamingr r@@ target_text prompt_textprompt_wav_pathmin_lenmax_leninference_timesteps cfg_value retry_badcaseretry_badcase_max_timesretry_badcase_ratio_thresholdrc cs|r | r tdd}t|dkr_|} t|| } tj| tj|jgtj | j dgdd} | j d}tj ||j |jjftj| j d}t|tj | j }t |tj | j }n||} t|| } tj| tj|jgtj | j dgdd} | j d}t|\}}|ddkr|jddd }||jkrtj|||j}|j |j}|d|dkr||d|}tjj||df}|j||j |j}| |jjd|j !dd d}|d}tj |tj | j d}t| |g} tj ||j |jjftj| j d}tj||gdd}tt|t |gtj | j }tt |t|gtj | j }| "d|j } |"d|j }|"d|j t#|j$j%}|"d|j }t||}d}|| kr|j&| ||||t't(|| d |||| d }| r|j |j}|D] \}}|j)|tj}|d | df*d}|Vqn(t+|\}}|r|j d|| krt,d|j d|d|d7}qrn | s|j)|tj*d}|VdSdS)NSRetry on bad cases is not supported in streaming mode, setting retry_badcase=False.Frr?r=rrrTrkeepdimr rrrr r r.% Badcase detected, audio_text_ratio= , retrying...)-warningswarnlenrl LongTensorrrrtensorrsint32r=rzerosr/r[ latent_dimfloat32onestyper torchaudioloadsizemeanr functionalresamplerrpadencoder^viewpermuterrrYr? _inferencerr'decodesqueezerro) rrrrrrr r r r r rtext text_token text_length audio_featrraudiosr patch_len padding_size audio_lengthtext_pad_tokenaudio_pad_feattarget_text_lengthretry_badcase_timesinference_result latent_predr decode_audiopred_audio_featr)r)r*rTs        ((       zVoxCPMModel._generatec Cs|r|stdt|\}}|ddkr|jddd}||jkr+tj|||j}|j|j }|d|dkrM||d|}t j j ||df}|j ||j|j}||j jd|jddd}||d}|S) a Build prompt cache for subsequent fast generation. Args: prompt_text: prompt text (required) prompt_wav_path: prompt audio path (required) Returns: prompt_cache: dict with prompt_text (raw text) and audio features. Text tokenization will be done during generation for consistency. z,prompt_text and prompt_wav_path are requiredrrTrrr rr0)rr r!r"r#rr$r%r/rrlrr&r[r'rr=r^r(rr)) rrrr1r2r3r4r0 prompt_cacher)r)r*build_prompt_caches.   zVoxCPMModel.build_prompt_cacheoriginal_cachenew_textnew_audio_featc CsJ|dur ||dS|d}|d}||}tj||gdd}||d}|S)as Merge original prompt cache with newly generated content to stabilize voice. Args: original_cache: original prompt cache new_text: newly generated text new_audio_feat: newly generated audio features Returns: merged_cache: merged cache with prompt_text and audio_feat Nr>rr0rr)rlr) rrArBrCoriginal_prompt_textoriginal_audio_featmerged_prompt_textmerged_audio_feat merged_cacher)r)r*merge_prompt_cacheszVoxCPMModel.merge_prompt_cachecOrr)r_generate_with_prompt_cacherr)r)r*generate_with_prompt_cache3rz&VoxCPMModel.generate_with_prompt_cachecOrr)rJrr)r)r*$generate_with_prompt_cache_streaming7sz0VoxCPMModel.generate_with_prompt_cache_streamingr?c  cs|r | r tdd}|dur!tjd|j|jjftjd} |} n |d} |d} | |} t| | }tj |tj |j gtj |jdgd d }t| |}| d}|jd}tj|tj |jd}tj|jd|j|jjftj|jd}t ||g}tj || gdd }t t|t|gtj |j}t t|t|gtj |j}|d|j}|d|j}|d|jt|jj}|d|j}t| |}d}||krX|j|||||tt|| d |||| d }| r1|j|j}|D]#\}}|j|tj}|d | dfd }|||fVq n't!|\}}|rW|jd|| krVt"d|jd|d|d7}qn | sr|j|tjd }|||fVdSdS)a$ Generate audio using pre-built prompt cache. Args: target_text: Text to convert to speech prompt_cache: Cache built by build_prompt_cache (can be None) min_len: Minimum audio length to avoid very short audio max_len: Maximum audio length inference_timesteps: Number of diffusion sampling steps cfg_value: Classifier-free guidance value retry_badcase: Whether to retry on bad cases retry_badcase_max_times: Maximum retry attempts retry_badcase_ratio_threshold: Threshold for audio-to-text ratio streaming: Whether to return a generator of audio chunks Returns: Generator of Tuple containing: - Decoded audio tensor for the current step if ``streaming=True``, else final decoded audio tensor - Tensor of new text tokens - New audio features up to the current step as a List if ``streaming=True``, else as a concatenated Tensor rFNrrr0rrrrrr.rrr)#rrrlemptyr/r[rrrrrrrrsrr=r"rrrrrrrrYr?rr*rr'rr+r,r^rro)rrr?rrr r r r r rprompt_audio_featr-rr.target_text_tokenr5r/r6r7r0rrr8r9r:r3r;r=r<r)r)r*rJ=s#   **       z'VoxCPMModel._generate_with_prompt_cachecOrr)rr*rr)r)r* inferencerzVoxCPMModel.inferencecOrr)r*rr)r)r*inference_streamingrzVoxCPMModel.inference_streamingr-feat feat_maskstreaming_prefix_lenc $ cs|j\} } } }||}||}|jjjr|jjj}nd}|j||}| d|| d|}|ddddf}g}d}|j|dd\}}|jj || || d|| d}|dddddf}|j || d|dd\}}|j j ||dddddf}tt|D]}||}||}||}|j||j|dd||d dd} || d}||}|| d| }| rtj|| ddd }!t|!d | |jd }"|"|fV||||jdd d }#||kr|#dkrnB|j |ddd ddftj!|jj "g|j#d$}| |}|j ||ddd ddftj!|j j "g|j#d$}q| sptj|dd }t|d | |jd }"|"|%d fVdSdS)aCore inference method for audio generation. This is the main inference loop that generates audio features using the language model and diffusion transformer. Args: text: Input text tokens text_mask: Mask for text tokens feat: Input audio features feat_mask: Mask for audio features min_len: Minimum generation length max_len: Maximum generation length inference_timesteps: Number of diffusion steps cfg_value: Classifier-free guidance value streaming: Whether to yield each step latent feature or just the final result Returns: Generator of Tuple containing: - Predicted latent feature at the current step if ``streaming=True``, else final latent features - Predicted audio feature sequence so far as a List if ``streaming=True``, else as a concatenated Tensor rrN.Trrr )rr/rrr rzb t p d -> b d (t p)rrr=)&rr|rRrYr.rrrprrkv_cache fill_cachesr~rxr rangerSrTr}r/rrappendrlrrrrrargmaxr^itemrrstepr=cloner,)$rr-rrRrSrrr r rrTrrrrrrrrprefix_feat_cond pred_feat_seq curr_embedrkv_cache_tuplerresidual_enc_outputsresidual_kv_cache_tupleri dit_hidden_1 dit_hidden_2r pred_featpred_feat_chunkr stop_flagr)r)r*r*s#     "     *, 0zVoxCPMModel._inferenceTpathrtrainingcCstttj|d}t|}t |dd}|r t |dnt }t j tj|ddddd} |||||} |sGt | jj} | | } n| D]\} } d | vrWd | _qK|durbd | vrbd | _qK| jt j| _tj|d }tj|d }tj|rtrtd|t|}n&tj|rtd|t j |ddd}|d|}n td|d|| D] \}}||d|<q| j|d d|r| S| | jj| dS)Nz config.jsonr:)rYz audiovae.pthr^T map_location weights_only state_dictr[Florazmodel.safetensorszpytorch_model.binz Loading model from safetensors: z&Loading model from pytorch_model.bin: z&Model file not found. Expected either  or z audio_vae.)strict)r) r-model_validate_jsonopenosrjjoinreadr from_pretrainedrr rlr!rrYr?rnamed_parameters requires_gradr[rexistsSAFETENSORS_AVAILABLEror getFileNotFoundErroritemsload_state_dictr=evalr)clsrjrrkr\rYrZr:r[vae_state_dictmodellm_dtypenameparamsafetensors_pathpytorch_model_pathmodel_state_dict checkpointkwvalr)r)r* from_local7s\       zVoxCPMModel.from_localccs0ddlm}|D] }t||r|Vq dS)zIterate over all LoRA modules.r rN)rrmodulesr)rrrr)r)r*_iter_lora_modulesqs   zVoxCPMModel._iter_lora_modules lora_pathr=cCs@ddlm}|p |j}||}|r|d}|d}n|jdkr#|nd}|jdvr,|nd}|r?|r?tr?tt||d}n|rT|rTt j ||d d }| d |}n t d |d |t |}dd|D} gg} } |D]'\} } | |vr~| n| | }|r||j| || | qt| | qt| | fS)a Load LoRA weights from file, supports calling after torch.compile. Uses named_parameters() to handle compile's _orig_mod wrapper. Supports both safetensors and pytorch formats. Args: lora_path: Checkpoint path (directory or .safetensors/.ckpt file) device: Target device, defaults to model's current device Returns: tuple: (loaded_keys, skipped_keys) r)Pathzlora_weights.safetensorszlora_weights.ckptz .safetensorsN)z.ckptz.pthrUFrlroz+LoRA checkpoint not found. Expected either rqcSs"i|] }d|vr|dd|qS)z ._orig_mod..)replace).0kr)r)r* s"z1VoxCPMModel.load_lora_weights..)pathlibrr=is_dirsuffixr{r|r rArlr!r}r~rryrdatacopy_rrY)rrr=rsafetensors_file ckpt_filerockpt model_params key_mapping loaded_keys skipped_keyskeyvalue target_keyr)r)r*load_lora_weightsxs4       zVoxCPMModel.load_lora_weightsenabledcCs|D]}||qdS)zEnable/disable all LoRA layers.N)r set_enabled)rrrr)r)r*set_lora_enableds  zVoxCPMModel.set_lora_enabledcCs|D]}|qdS)zJReset all LoRA weights (A: kaiming, B: zeros), effectively unloading LoRA.N)rreset_lora_parameters)rrr)r)r*reset_lora_weightss  zVoxCPMModel.reset_lora_weightscCsdd|DS)z(Get all LoRA parameters (lora_A/lora_B).cSs$i|]\}}d|vr||jqS)lora_)rr])rrrr)r)r*rs  z3VoxCPMModel.get_lora_state_dict..)ryrr)r)r*get_lora_state_dictszVoxCPMModel.get_lora_state_dictr)F) rrr rrrFrrF)r rrrFrrF)r rrrFr)TFN)+r$r%r&r-r r rCrkrrBrrlTensorrVrrrrrinference_moderAr'rr@rrIrrKrrLrrJrPrQr* classmethodrrrrrr __classcell__r)r)rr*rXhsDU  `      3 %     * }(   & u93rX)5__doc__rutypingrrrrrrltorch.nnrtorch.nn.functionalr$Fr reinopsrpydanticrsafetensors.torchr r| ImportErrorr transformersr modules.audiovaer rmodules.layersrrrmodules.locditrrrmodules.locencrmodules.minicpm4rrutilsrrrr+r-rC model_rebuildModulerXr)r)r)r*s>