o }oiT@sddlZddlmZmZddlmZddlZddlmZddl m Z m Z ddl m Z mZmZddlmZGdd d e eZGd d d eZGd d d eZGddde ZdS)N)ABCabstractmethod)Optional)mask_sequence_tensor) NeuralModule typecheck) LengthsType NeuralTypeSpectrogramType)loggingc seZdZdZ    d0dededed effd d Zed efd dZed efddZ dede j d e j fddZ eddZeddZede j d e j fddZede j d e j fddZede j d e j fddZd e j d e j e j ffd!d"Zde j d e j e j e j ffd#d$Zede j d e j fd%d&Zd'e j d e j e j ffd(d)Zde j d e j e j e j ffd*d+Zed,d-Zd.d/ZZS)1SBNoiseScheduleu^Noise schedule for the Schrödinger Bridge Args: time_min: minimum time for the process time_max: maximum time for the process num_steps: number of steps for the process eps: small regularization References: Schrödinger Bridge for Generative Speech Enhancement, https://arxiv.org/abs/2407.16074 ?d:0yE>time_mintime_max num_stepsepscst|dkrtd|||krtd|d|||_||_|dkr/td|||_|dkr=td|||_td|j j td|jtd |jtd |jtd |jdS) Nrz/time_min should be non-negative, current value z5time_max should be larger than time_min, current max z and min zExpected num_steps > 0, got Expected eps > 0, got Initialized %s with time_min: %s time_max: %s num_steps: %s eps: %s) super__init__ ValueErrorrrrrr debug __class____name__)selfrrrrro/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/audio/parts/submodules/schroedinger_bridge.pyr's$ zSBNoiseSchedule.__init__returncCs |j|jS)zTime step for the process.)rrr!r#r#r$dtJ zSBNoiseSchedule.dtcCs |j|jS)zTime range for the process.)rrr&r#r#r$ time_deltaOr(zSBNoiseSchedule.time_deltasizedevicecCstj||d|j|j}|S)z.Generate random time steps in the valid range.r+)torchrandr)r)r!r*r+timer#r#r$ generate_timeTszSBNoiseSchedule.generate_timecCstj|jgtjd}||S)zReturn alpha_t at t_max.r,)r-tensorralphar+r!t_maxr#r#r$ alpha_t_maxY zSBNoiseSchedule.alpha_t_maxcCstj|jgtjd}||S)zReturn sigma_t at t_max.r,)r-r1rr2r+sigmar3r#r#r$ sigma_t_max_r6zSBNoiseSchedule.sigma_t_maxr/cCdS)zDrift scaling f(t). Args: time: tensor with time steps Returns: Tensor the same size as time, representing drift scaling. Nr#r!r/r#r#r$fe zSBNoiseSchedule.fcCr9)zDiffusion scaling g(t). Args: time: tensor with time steps Returns: Tensor the same size as time, representing diffusion scaling. Nr#r:r#r#r$gqr<zSBNoiseSchedule.gcCr9)zReturn alpha for SB noise schedule. alpha_t = exp( int_0^s f(s) ds ) Args: time: tensor with time steps Returns: Tensor the same size as time, representing alpha for each time. Nr#r:r#r#r$r2} zSBNoiseSchedule.alphar2cCs0|tj|jg|jd}|||j}||fS)zReturn alpha_bar for SB. alpha_bar = alpha_t / alpha_t_max Args: alpha: tensor with alpha values Returns: Tensors the same size as alpha, representing alpha_bar and alpha_t_max. r,)r2r-r1rr+r)r!r2r5 alpha_barr#r#r$alpha_bar_from_alphas z$SBNoiseSchedule.alpha_bar_from_alphacC"||}||\}}|||fS)zReturn alpha, alpha_bar and alpha_t_max for SB. Args: time: tensor with time steps Returns: Tuple of tensors with alpha, alpha_bar and alpha_t_max. )r2r@)r!r/r2r?r5r#r#r$ get_alphas  zSBNoiseSchedule.get_alphascCr9)zReturn sigma_t for SB. sigma_t^2 = int_0^s g^2(s) / alpha_s^2 ds Args: time: tensor with time steps Returns: Tensor the same size as time, representing sigma for each time. Nr#r:r#r#r$r7r>zSBNoiseSchedule.sigmar7cCs>|tj|jg|jd}|d|d}t||j|fS)zReturn sigma_bar_t for SB. sigma_bar_t^2 = sigma_t_max^2 - sigma_t^2 Args: sigma: tensor with sigma values Returns: Tensors the same size as sigma, representing sigma_bar and sigma_t_max. r,)r7r-r1rr+sqrtr)r!r7r8 sigma_bar_sqr#r#r$sigma_bar_from_sigmas z$SBNoiseSchedule.sigma_bar_from_sigmacCrA)zReturn sigma, sigma_bar and sigma_t_max for SB. Args: time: tensor with time steps Returns: Tuple of tensors with sigma, sigma_bar and sigma_t_max. )r7rG)r!r/r7 sigma_barr8r#r#r$ get_sigmasrCzSBNoiseSchedule.get_sigmascCr9)z$Return a copy of the noise schedule.Nr#r&r#r#r$copyszSBNoiseSchedule.copycCsJ|jjd|jd|jd|jd}|d|j7}|d|j7}|S)Nz (time_min=z , time_max=z , num_steps=)z dt: z time_delta: )rr rrrr'r)r!descr#r#r$__repr__s&zSBNoiseSchedule.__repr__r rrr)r __module__ __qualname____doc__floatintrpropertyr'r)r-r+Tensorr0r5r8rr;r=r2r@rBr7rGrIrJrN __classcell__r#r#r"r$r sN#        r c seZdZdZ    ddededed ed ed ef fd d ZdejdejfddZ dejdejfddZ dejdejfddZ dejdejfddZ ddZ fddZZS)SBNoiseScheduleVEuVariance exploding noise schedule for the Schrödinger Bridge. Args: k: defines the base for the exponential diffusion coefficient c: scaling for the diffusion coefficient time_min: minimum time for the process time_max: maximum time for the process num_steps: number of steps for the process eps: small regularization References: Schrödinger Bridge for Generative Speech Enhancement, https://arxiv.org/abs/2407.16074 r rrrkcrrrrcstj||||d|dkrtd||dkr td|||_||_td|jjtd|jtd|jtd |j td |j td |j td |j dS) NrrrrzExpected k > 1, got rExpected c > 0, got rz k: %s c: %srrrr) rrrrZrYr rrr rrrr)r!rYrZrrrrr"r#r$rs zSBNoiseScheduleVE.__init__r/r%cC t|SN)r- zeros_liker:r#r#r$r;  zSBNoiseScheduleVE.fcCst|j|j|jSr`)r-rErZrYr/r:r#r#r$r= szSBNoiseScheduleVE.gcCr_r`)r- ones_liker:r#r#r$r2rbzSBNoiseScheduleVE.alphacCs8|j|jd|ddt|j|j}t|SNrDr\)rZrYmathlogrr-rEr!r/sigma_sqr#r#r$r7s. zSBNoiseScheduleVE.sigmacCs t|j|j|j|j|j|jdS)N)rYrZrrrr)rXrYrZrrrrr&r#r#r$rJszSBNoiseScheduleVE.copycs.t}|d|j7}|d|j7}|S)Nz k: z c: )rrNrYrZrLr"r#r$rN s zSBNoiseScheduleVE.__repr__rOr rPrQrRrSrTrr-rVr;r=r2r7rJrNrWr#r#r"r$rXs2 rXcseZdZdZ     ddededed ed ed ed effd d ZdejdejfddZ dejdejfddZ dejdejfddZ dejdejfddZ ddZ fddZZS)SBNoiseScheduleVPuVariance preserving noise schedule for the Schrödinger Bridge. Args: beta_0: defines the lower bound for diffusion coefficient beta_1: defines upper bound for diffusion coefficient c: scaling for the diffusion coefficient time_min: minimum time for the process time_max: maximum time for the process num_steps: number of steps for the process eps: small regularization rr rrbeta_0beta_1rZrrrrcstj||||d|dkrtd||dkr td|||kr.td|d||dkr9td|||_||_||_td|jj td |jtd |jtd |jtd |j td |j td|j td|j dS)Nr[rzExpected beta_0 >= 0, got zExpected beta_1 >= 0, got z%Expected beta_0 < beta_1, got beta_0=z and beta_1=r]rz beta_0: %sz beta_1: %sr^rrrr)rrrrkrlrZr rrr rrrr)r!rkrlrZrrrrr"r#r$r4s( zSBNoiseScheduleVP.__init__r/r%cCsd|j||j|jS)N)rkrlr:r#r#r$r;ZszSBNoiseScheduleVP.fcCs&|j|j||j|j}t|Sr`)rZrkrlr-rE)r!r/g_sqr#r#r$r=]s zSBNoiseScheduleVP.gcCs0|j||j|jd|d}td|S)NrDrm)rkrlr-exp)r!r/tmpr#r#r$r2as"zSBNoiseScheduleVP.alphacCsD|j||j|jd|d}t|d}|j|}t|Srd)rkrlr-rorZrErgr#r#r$r7es"  zSBNoiseScheduleVP.sigmac Cs$t|j|j|j|j|j|j|jdS)N)rkrlrZrrrr)rjrkrlrZrrrrr&r#r#r$rJkszSBNoiseScheduleVP.copycs>t}|d|j7}|d|j7}|d|j7}|S)Nz beta_0: z beta_1: z c: )rrNrkrlrZrLr"r#r$rNvs zSBNoiseScheduleVP.__repr__)rr rrrrir#r#r"r$rj's8& rjcseZdZdZ      d/deded ed ed ed eed eede deffdd Z e ddZ e j defddZ e ddZej defddZe ddZej de fddZe ddZej defddZe dd Zej defd!d Zeed"eed"ed#d$eed%ed#d$d&ed"eeed%ed#d$d'd(e d0d)ejd*ejd+eejd,ejfd-d.ZZS)1 SBSampleruMSchrödinger Bridge sampler. Args: noise_schedule: noise schedule for the bridge estimator: neural estimator estimator_output: defines the output of the estimator, e.g., data_prediction estimator_time: time for conditioning the estimator, e.g., 'current' or 'previous'. Default is 'previous'. process: defines the process, e.g., sde or ode time_max: maximum time for the process time_min: minimum time for the process num_steps: number of steps for the process eps: small regularization to prevent division by zero References: Schrödinger Bridge for Generative Speech Enhancement, https://arxiv.org/abs/2407.16074 Schrodinger Bridges Beat Diffusion Models on Text-to-Speech Synthesis, https://arxiv.org/abs/2312.03491 previoussdeN2rnoise_schedule estimatorestimator_outputestimator_timeprocessrrrrc st||_|dur||j_td|jj|dur*||j_td|jj||j_td|jj||_ ||_ ||_ ||_ | dkrMt d| | |_td|jjtd|j td|j td |j td |jtd |jtd |jtd |jdS)N"noise_schedule.time_max set to: %s"noise_schedule.time_min set to: %s#noise_schedule.num_steps set to: %srrrz estimator_output: %sz estimator_time: %sz process: %sz time_min: %sz time_max: %sz num_steps: %sz eps: %s)rrrJrurr inforrrvrwrxryrrrrr ) r!rurvrwrxryrrrrr"r#r$rs2 zSBSampler.__init__cC|jjSr`)rurr&r#r#r$rzSBSampler.time_maxvaluecC||j_td|jjdS)Nrz)rurr rr!rr#r#r$rcCr~r`)rurr&r#r#r$rrzSBSampler.time_mincCr)Nr{)rurr rrr#r#r$rrcCr~r`)rurr&r#r#r$rrzSBSampler.num_stepscCr)Nr|)rurr rrr#r#r$rrcC|jSr`)_processr&r#r#r$ryzSBSampler.processcC.|dvr td|||_td|jdS)N)rsodeUnexpected process: zprocess set to: %s)rrr r}rr#r#r$rycCrr`)_estimator_timer&r#r#r$rxrzSBSampler.estimator_timecCr)N)currentrrzUnexpected estimator time: zestimator time set to: %s)rrr r}rr#r#r$rxr)BCDTT)optionalr) prior_meanestimator_condition state_length)sampler) input_types output_typesrrrr%cCs|}|dur t||}tj|j|j|jd|jd}|dtj|jd|jd}|j |\}}} |j |\} } } |ddD]%} | tj|jd|jd}|durW|ntj ||gdd}|j dkrg|n|}|jdkry|j|||d\}}ntd |j|j |\}}}|j |\}}}|jd kr||d || d |j}d|d | d |j}||}||t|}|d ddd}|d ddd}|d ddd}t|}||||||}nt|jd krW||||| | |j}|| d |j|d | ||| |j}|| | d |j|d | ||| |j}|d ddd}|d ddd}|d ddd}||||||}ntd|j|}|}|} |} qA|durrt||}||fS)z(Takes prior mean and generates a sample.Nr\r,r)dimrdata_prediction)input input_length conditionzUnexpected estimator output: rsrDrr)rr-linspacerrrr+onesshaperurBrIcatrxrwrvNotImplementedErrorryrrEview randn_like RuntimeError)r!rrrstate time_steps time_prev alpha_prev_r5 sigma_prevsigma_bar_prevr8tr/estimator_inputrxcurrent_estimatealpha_t alpha_bar_tsigma_t sigma_bar_t weight_prevrpweight_estimateweight_zz_normweight_prior_meanr#r#r$forwardsl         zSBSampler.forward)rrrsNNrtrr`)r rPrQrRr rstrrrSrTrrUrsetterrrryrxrr r tuplerr-inference_moderVrrWr#r#r"r$rq~s  2        rq)reabcrrtypingrr-#nemo.collections.common.parts.utilsrnemo.core.classesrrnemo.core.neural_typesrr r nemo.utilsr r rXrjrqr#r#r#r$s    DJW