o :i @sddlZddlZddlZddlZddlmZmZmZzddlZWne y;Z ze de dWYdZ [ ndZ [ wwddl Z ddl mZddl mZe jrTdZn e jjr]dZndZd d Zd d Zee jd rxdZddlmZmZn dZddlmZmZeeeZeeeZdededefddZdededefddZGddde Z!Gddde jj"Z#ee$e#fZ%GdddZ&Gddde j'j(Z)ded e*fd!d"Z+Gd#d$d$e j'j(Z,Gd%d&d&e jj"Z-d'd(d)e$dej.fd*d+Z/Gd,d-d-e j'j(Z0Gd.d/d/e jj"Z1 d{ded0e$d1e$d2e2def d3d4Z3defd5d6Z4ded7e*fd8d9Z5Gd:d;d;e j'j(Z6Gdd?d?e j'j(Z8d@dAZ9GdBdCdCe j'j(Z: d|dedEe$dFe$dGe$dHe;f dIdJZGdOdPdPej"Z?GdQdRdRe j'j(Z@GdSdTdTej"ZAGdUdVdVe j'j(ZBGdWdXdXe jj"ZCGdYdZdZe jj"ZDGd[d\d\e j'j(ZEGd]d^d^e jj"ZFGd_d`d`e jj"ZGdefdadbZHdefdcddZIGdedfdfe j'j(ZJGdgdhdhe jj"ZKdidjZLdkdlZMdmdnZNdodpZOdqdrZPdsdtZQdudvZRdwdxZSeTdykrDeUVejWe Xdze YdzeReQeLeMeNePeOeSdSdS)}N)OptionalTupleUnionzFailed import k2 with error zr. Swoosh functions will fallback to PyTorch implementation, leading to slower speed and higher memory consumption.)TensorcudampscpucCs(tdkr tjStdkrtjSdS)Nrrr) DEVICE_TYPEtorchrmemory_allocatedrcurrent_allocated_memoryr r 6/home/ubuntu/LuxTTS/zipvoice/models/modules/scaling.pyget_memory_allocated+s   rcsfdd}|S)Ncss|Std|S)N) device_type)r )funccuda_amp_deprecateddecr r decorator4sz'custom_amp_decorator..decoratorr )rrrr rrcustom_amp_decorator3sr custom_fwdT) custom_bwdrFxyreturncCs0t||}t||}|tt| SN)r maxabslog1pexp)rr max_valuediffr r rlogaddexp_onnxEs r#cCs6tjr t||Stjrt||St||Sr)r jit is_scripting logaddexponnxis_in_onnx_exportr#)rrr r rr&Qs     r&c@sdeZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ dddde fddZ dS)PiecewiseLinearz Piecewise linear function, from float to float, specified as nonempty list of (x,y) pairs with the x values in order. x values <[initial x] or >[final x] are map to [initial y], [final y] respectively. cGst|dks Jt|t|dkr"t|dtr"t|dj|_ndd|D|_|jD]\}}t|ttfs>Jt|t|ttfsKJt|q-tt|jdD]!}|j|dd|j|dksvJ||j||j|dfqUdS)NrcSs g|] \}}t|t|fqSr )float.0rrr r r p z,PiecewiseLinear.__init__..) len isinstancer)listpairsr+inttyperange)selfargsrrir r r__init__ks" zPiecewiseLinear.__init__cCsdt|jdddS)NzPiecewiseLinear(r*))strr3r7r r r__str__|szPiecewiseLinear.__str__cCs||jddkr|jddS||jddkr |jddS|jd\}}tdt|jD]&}|j|\}}||krP||krP|||||||S||}}q/J)Nrr*r;)r3r6r0)r7rcur_xcur_yr9next_xnext_yr r r__call__s  zPiecewiseLinear.__call__cstfdd|jDS)Ncsg|] \}}||fqSr r r,alphar rr.sz+PiecewiseLinear.__mul__..)r)r3)r7rFr rEr__mul__szPiecewiseLinear.__mul__csPtttfrtfdd|jDS|\}tddt|jjDS)Ncs g|] }|d|dfqSrr*r )r-prr rr.r/z+PiecewiseLinear.__add__..cSs(g|]\}}|d|d|dfqSrHr r-spxpr r rr.s()r1r+r4r)r3get_common_basiszipr7rsr rJr__add__s zPiecewiseLinear.__add__cCsHt|ttfr td|f}|j|dd\}}tddt|j|jDS)NrTinclude_crossingscS*g|]\}}|dt|d|dfqSrH)rrKr r rr.*z'PiecewiseLinear.max..r1r+r4r)rNrOr3rPr r rrs  zPiecewiseLinear.maxcCsNt|ts t|trtd|f}|j|dd\}}tddt|j|jDS)NrTrScSrUrHminrKr r rr.rVz'PiecewiseLinear.min..rWrPr r rrYs  zPiecewiseLinear.mincCs |j|jkSr)r3)r7otherr r r__eq__s zPiecewiseLinear.__eq__FrIrTc sjtts JtttddjDddjD}fdd|D}fdd|D}|rg}tt|dD]I}||||k||d||dkkrt||||}t||d||d} ||| } ||| ||d||} | | q=t|dkrtt||}fdd|D}fdd|D}tt ||tt ||fS) aI Returns (self_mod, p_mod) which are equivalent piecewise linear functions to self and p, but with the same x values. p: the other piecewise linear function include_crossings: if true, include in the x values positions where the functions indicate by this and p crosss. cSsg|]\}}|qSr r )r-r_r r rr.z4PiecewiseLinear.get_common_basis..cg|]}|qSr r r-rr>r rr.r]cr^r r r_rIr rr.r]r*rcr^r r r_r>r rr.r]cr^r r r_r`r rr.r]) r1r)r5sortedsetr3r6r0rappendrO) r7rIrTx_valsy_vals1y_vals2 extra_x_valsr9diff_cur diff_nextpos extra_x_valr )rIr7rrNs* ((      z PiecewiseLinear.get_common_basisN)F)__name__ __module__ __qualname____doc__r:r?rDrGrRrrYr[boolrNr r r rr)dsr)csReZdZdZdddeffddZdefdd Zd d Zd d Z ddZ Z S)ScheduledFloataK This object is a torch.nn.Module only because we want it to show up in [top_level module].modules(); it does not have a working forward() function. You are supposed to cast it to float, as in, float(parent_module.whatever), and use it as something like a dropout prob. It is a floating point value whose value changes depending on the batch count of the training loop. It is a piecewise linear function where you specify the (x,y) pairs in sorted order on x; x corresponds to the batch index. For batch-index values before the first x or after the last x, we just use the first or last y value. Example: self.dropout = ScheduledFloat((0.0, 0.2), (4000.0, 0.0), default=0.0) `default` is used when self.batch_count is not set or not in training mode or in torch.jit scripting mode. defaultrtcs*td|_d|_||_t||_dSr)superr: batch_countnamertr)schedule)r7rtr8 __class__r rr:s zScheduledFloat.__init__rcCs"d|jdt|jjddS)Nz batch_count=z , schedule=r*r;)rvr=rxr3r>r r r extra_reprs zScheduledFloat.extra_reprcCsn|j}|dus|jrtjstjrt|jS||j}t dkr5t d|j d|jd||S)Ng-C6*?zScheduledFloat: name=z, batch_count=z, ans=) rvtrainingr r$r% is_tracingr+rtrxrandomloggingdebugrw)r7rvansr r r __float__s&    zScheduledFloat.__float__cCsDt|ts t|trt|j||jdSt|j|j|j|jdSNrs)r1r+r4rqrxrtr7rr r rrRs zScheduledFloat.__add__cCsJt|ts t|trt|j||jdSt|j|jt|j|jdSr)r1r+r4rqrxrrtrr r rr s   zScheduledFloat.max) rlrmrnror+r:r=r{rrRr __classcell__r r ryrrqsrqc@s0eZdZdZdefddZdedefddZd S) CutoffEstimatorz Estimates cutoffs of an arbitrary numerical quantity such that a specified proportion of items will be above the cutoff on average. p is the proportion of items that should be above the cutoff. rIcCs||_d|_d|_d|_dSNr)rIcount count_abovecutoffr7rIr r rr:!s zCutoffEstimator.__init__rrcCsp||jk}|jd7_|r|jd7_|j|j}||j}|dk|kr6t|}|||jd||_|S)z8 Returns true if x is above the cutoff. r*r)rrrrIr)r7rrcur_pdelta_pqr r rrD*s    zCutoffEstimator.__call__N)rlrmrnror+r:rprDr r r rrs rc@s8eZdZdZededefddZedefddZd S) SoftmaxFunctionz Tries to handle half-precision derivatives in a randomized way that should be more accurate for training than the default behavior. rdimcCs<|j|d}tr|tj}|||j|_||_|SNr) softmaxr is_autocast_enabledtofloat16save_for_backwarddtypex_dtyper)ctxrrrr r rforward@s   zSoftmaxFunction.forwardans_gradcCs||j\}tjjtdd(|tj}|tj}||}|||j|jdd}|dfWdS1s7wYdS)NFenabledTrkeepdim) saved_tensorsr ampautocastr rfloat32sumrrrrx_gradr r rbackwardMs  $zSoftmaxFunction.backwardN) rlrmrnro staticmethodrr4rrr r r rr:s  rrcCs2|jr tjs tjr|j|dSt||Sr) requires_gradr r$r%r}rrapply)rrr r rrXs  rc @sHeZdZededededededef ddZed edefd d Zd S) BiasNormFunctionrbias log_scale channel_dimstore_output_for_backproprc Cs|jdksJ|dkr||j}||_||_t|d|jD]}|d}qtj||d|ddd|}||}||rD| n|| | | |S)Nr*rr;Tr) ndimrrr6 unsqueezer meanr rdetach) rrrrrrr\scalesrr r rrfs$   zBiasNormFunction.forwardrcCs|j\}}}}|jr||}n|}|}d|_d|_d|_t%tj||d|jddd|}||}|j |dWdn1sJwY|j |j |j ddfS)NTrrrgradient) rrrrr enable_gradrrr rgradflatten)rrans_or_xrrrrrr r rrs   zBiasNormFunction.backwardN) rlrmrnrrr4rprrr r r rr_s" rcs\eZdZdZ     ddeded ed ed ed ed dffdd Zded efddZ Z S)BiasNorma This is intended to be a simpler, and hopefully cheaper, replacement for LayerNorm. The observation this is based on, is that Transformer-type networks, especially with pre-norm, sometimes seem to set one of the feature dimensions to a large constant value (e.g. 50), which "defeats" the LayerNorm because the output magnitude is then not strongly dependent on the other (useful) features. Presumably the weight and bias of the LayerNorm are required to allow it to do this. Instead, we give the BiasNorm a trainable bias that it can use when computing the scale for normalization. We also give it a (scalar) trainable scale on the output. Args: num_channels: the number of channels, e.g. 512. channel_dim: the axis/dimension corresponding to the channel, interpreted as an offset from the input's ndim if negative. This is NOT the num_channels; it should typically be one of {-2, -1, 0, 1, 2, 3}. log_scale: the initial log-scale that we multiply the output by; this is learnable. log_scale_min: FloatLike, minimum allowed value of log_scale log_scale_max: FloatLike, maximum allowed value of log_scale store_output_for_backprop: only possibly affects memory use; recommend to set to True if you think the output of this module is more likely than the input of this module to be required to be stored for the backprop. r;??F num_channelsrr log_scale_min log_scale_maxrrNcsTtt|||_||_tt||_ tt ||_ ||_ ||_ ||_dSr)rurr:rrnn Parameterr tensorrzerosrrrr)r7rrrrrrryr rr:s  zBiasNorm.__init__rcCs|j|j|jks JtjstjrK|j}|dkr!||j7}|j}t |d|jD]}| d}q,tj ||d|ddd|j }||St|j t|jt|j|jd}t||j||j|jS) Nrr*r;rTrr)rYrr|)shaperrr r$r%r}rrr6rrrr limit_param_valuer+rrr|rrr)r7rrrr\rrr r rrs2  zBiasNorm.forward)r;rrrF) rlrmrnror4r+rpr:rrrr r ryrrs.!rr) initial_scalercOstj|i|}t/|jdd|9<|jdur3tjj|jd|d|Wd|SWd|S1s>wY|S)aT Behaves like a constructor of a modified version of nn.Linear that gives an easy way to set the default initial parameter scale. Args: Accepts the standard args and kwargs that nn.Linear accepts e.g. in_features, out_features, bias=False. initial_scale: you can override this if you want to increase or decrease the initial magnitude of the module's output (affects the initialization of weight_scale and bias_scale). Another option, if you want to do something like this, is to re-initialize the parameters. Ng皙?)rLinearr no_gradweightrinituniform_)rr8kwargsrr r r ScaledLinears    rc@s`eZdZededededededededefd d Zed edeed d d d d ffd dZ d S)BalancerFunctionrmin_meanmax_meanmin_rmsmax_rms grad_scalerrcCs8|dkr ||j7}||_||||||||f|_|Sr)rrrconfig)rrrrrrrrr r rrs  zBalancerFunction.forwardrNc s|j\}|j\}}}}}zttjjtdd|tj}| }d|_ fddt |j D}|dj |dd} |j |dd} | | | jdd } | jdd } | | } | | j||d }| j||d }|| }||}|jt|d |j}|dj |ddjdd }|||}|tj}|||}||j}Wdn1swYWdn1swYWn!ty}ztd |d t|jdWYd}~nd}~ww|ddddddfS)NFrTcsg|]}|kr|qSr r )r-r9rr rr.,sz-BalancerFunction.backward..rr#B ;rX)rYrrz'Caught exception in Balancer backward: , size=, will continue.)rrr rrrr rrrrr6rrclampsqrtrlogr ones_likerr Exceptionrinfor2r)rrrrrrrr mean_dimsuncentered_varrstddevrmsmm_loss rms_clampedr_lossloss loss_grad loss_grad_rms x_grad_float x_grad_moder rrrsf    (zBalancerFunction.backward) rlrmrnrrr+r4rrrr r r rrs* (rcsfeZdZdZ      dded ed ed ed ed ededeeffdd ZdedefddZ Z S)Balancera* Modifies the backpropped derivatives of a function to try to encourage, for each channel, that it is positive at least a proportion `threshold` of the time. It does this by multiplying negative derivative values by up to (1+max_factor), and positive derivative values by up to (1-max_factor), interpolated from 1 at the threshold to those extremal values when none of the inputs are positive. Args: num_channels: the number of channels channel_dim: the dimension/axis corresponding to the channel, e.g. -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative. min_positive: the minimum, per channel, of the proportion of the time that (x > 0), below which we start to modify the derivatives. max_positive: the maximum, per channel, of the proportion of the time that (x > 0), above which we start to modify the derivatives. scale_gain_factor: determines the 'gain' with which we increase the change in gradient once the constraints on min_abs and max_abs are violated. min_abs: the minimum average-absolute-value difference from the mean value per channel, which we allow, before we start to modify the derivatives to prevent this. max_abs: the maximum average-absolute-value difference from the mean value per channel, which we allow, before we start to modify the derivatives to prevent this. prob: determines the minimum probability with which we modify the gradients for the {min,max}_positive and {min,max}_abs constraints, on each forward(). This is done randomly to prevent all layers from doing it at the same time. 皙?ffffff?皙?Y@{Gz?Nrr min_positive max_positivemin_absmax_absrprobc s^t|durtdddd}||_td|_||_||_||_||_ ||_ ||_ ||_ dS)N)rr?)g@@g?g?rsr) rur:rqrr mem_cutoffrrrrrrr) r7rrrrrrrrryr rr:xs   zBalancer.__init__rrc Cstjs|jr|js|jjdkr|trt |St |j }t |krfdd}dd}|t |j }|t |j}|t |j}|t |j}t |j} |j|j|jksZJt|||||| |jSt |S)NrcSsd|S)Ng/v ?r rJr r r _abs_to_rmssz%Balancer.forward.._abs_to_rmscs(ddfdd}dd|}||S)NcSs,d}td||td||dS)Ng|=r*@)mathr)repsr r r_atanhs(zFBalancer.forward.._proportion_positive_to_mean.._atanhcs d|S)Ng?= ?r rJrr r_approx_inverse_erfs zSBalancer.forward.._proportion_positive_to_mean.._approx_inverse_erfr;rr )rrr rr_proportion_positive_to_means  z6Balancer.forward.._proportion_positive_to_mean)r r$r%ris_cudadevicer5rr_no_opr+rr~rrrrrrrrrr) r7rrrrrrrrrr r rrs8    zBalancer.forward)rrrrrN) rlrmrnror4 FloatLikerr:rrrr r ryrrXs4# rlimitpenaltyrwcCs@|}||dk}|||tj|}t|||}|S)aH Returns x unmodified, but in backprop will put a penalty for the excess of the absolute values of elements of x over the limit "limit". E.g. if limit == 10.0, then if x has any values over 10 it will get a penalty. Caution: the value of this penalty will be affected by grad scaling used in automatic mixed precision training. For this reasons we use this, it shouldn't really matter, or may even be helpful; we just use this to disallow really implausible values of scores to be given to softmax. The name is for randomly printed debug info. r)signrrr int8 with_loss)rr r rwx_sign over_limitaux_lossr r rpenalize_abs_values_gts  rcCs^|jdkr |S|j\}}}||||}|dddd|df}|j||fks-J|S)Nrr*)rdiagrreshape)rbatchrr r r_diags  r num_groupsc Cs|jtjksJ|d|jd}|j\}}||dksJ||}||||dd}||jddd}t|dd|}t|}|d ||}||dd}|S)a Computes the "whitening metric", a value which will be 1.0 if all the eigenvalues of of the centered feature covariance are the same within each group's covariance matrix and also between groups. Args: x: a Tensor of shape (*, num_channels) num_groups: the number of groups of channels, a number >=1 that divides num_channels Returns: Returns a scalar Tensor that will be 1.0 if the data is "perfectly white" and greater than 1.0 otherwise. r;rr*Trrr) rr rrr transposermatmulrr) rr num_framesrchannels_per_groupx_covarx_covar_mean_diagx_covarsq_mean_diagmetricr r r_whitening_metrics   r"c@s:eZdZededejdefddZedefddZd S) WhiteningPenaltyFunctionrmodulercCs||||_|Sr)rr$)rrr$r r rr&s z WhiteningPenaltyFunction.forwardrc Cs|j\}|j}zttjjtdd|tj }d|_ t ||j }t dks1tdkrPtd|jd|j d|jd d |d d t|j |t|jkrn|j|_|dfWdWdWS|j|_||j}|j|tj|d }||}|||jdfWdWdWS1swYWdn 1swYW|dfSW|dfSty}ztd|dt |jdWYd}~|dfSd}~ww)NFrT{Gzt?__main__zWhitening: name=z , num_groups=z,num_channels=r;z , metric=z.2fz vs. limit=rz%Caught exception in Whiten backward: rr)!rr$r rrrr rrrrr"rr~rlrrrwritemr+whitening_limitmin_probrmax_probrrrnormrrrr2) rrx_origw x_detachedr! penalty_gradscalerr r rr,sb   !!z!WhiteningPenaltyFunction.backwardN) rlrmrnrrrModulerrr r r rr#%s r#c sPeZdZdededeeeeeffdeffdd Zde de fd d Z Z S) Whitenrr(rrcstt||dks Jt|dksJ|dksJ||_||_||_t|tr-||f}|\|_|_ d|jkrE|j krEdksHJJ|j |_ d|_ dS)a Args: num_groups: the number of groups to divide the channel dim into before whitening. We will attempt to make the feature covariance within each group, after mean subtraction, as "white" as possible, while having the same trace across all groups. whitening_limit: a value greater than 1.0, that dictates how much freedom we have to violate the constraints. 1.0 would mean perfectly white, with exactly the same trace across groups; larger values give more freedom. E.g. 2.0. prob: the probability with which we apply the gradient modification (also affects the grad scale). May be supplied as a float, or as a pair (min_prob, max_prob) grad_scale: determines the scale on the gradient term from this object, relative to the rest of the gradient on the attention weights. E.g. 0.02 (you may want to use smaller values than this if prob is large) r*rN) rur2r:r+rr(rr1r)r*rrw)r7rr(rrryr rr:Ws    * zWhiten.__init__rrcCs:t|j}|jrt|jks|dkrt|St||S)a In the forward pass, this function just returns the input unmodified. In the backward pass, it will modify the gradients to ensure that the distribution in each group has close to (lambda times I) as the covariance after mean subtraction, with the same lambda across groups. For whitening_limit > 1, there will be more freedom to violate this constraint. Args: x: the input of shape (*, num_channels) Returns: x, unmodified. You should make sure you use the returned value, or the graph will be freed and nothing will happen in backprop. r)r+rrr~rr r#r)r7rrr r rrs  zWhiten.forward) rlrmrnr4r rr+rr:rrrr r ryrr2Vs(r2c@s8eZdZedededefddZedefddZd S) WithLossrrrwcCsD|j|_tdkr |dur |}td|d|d|S)NgMb`?zWithLoss: name=z , loss-sum=z.3e)ry_shaper~rr'rr)rrrrwloss_sumr r rrs  zWithLoss.forwardrcCs|tj|j|j|jddfS)Nrr )r onesr4rr )rrr r rrszWithLoss.backwardN)rlrmrnrrr=rrr r r rr3 r3cCst|||Sr)r3r)rrrwr r rrsrc@s8eZdZedededefddZedefddZd S) LimitParamValuerrYrcCs&||||ks J||_||_|Sr)rrYr)rrrYrr r rrs  zLimitParamValue.forwardrcCsZ|j\}|tt|dk||jkdd}|tt|dk||jkdd9}|ddfS)Nrr)rr where logical_andrYr)rrrr r rrs $ zLimitParamValue.backwardN)rlrmrnrrr+rrr r r rr9r8r9333333?rYrrr|cCs"|rt|krt|||S|Sr)r~r9r)rrYrrr|r r rrsrcCs*tjs tjr |S|jddddS)Nr*r;rr)r r$r%r}chunkrJr r rr sr cs$eZdZfddZddZZS)Identitycstt|dSr)rur?r:r>ryr rr:szIdentity.__init__cCst|Sr)r rr r rrszIdentity.forward)rlrmrnr:rrr r ryrr?s r?cs4eZdZdeffdd ZdedefddZZS)Dropout2rIcst||_dSr)rur:rIrryr rr:s  zDropout2.__init__rrcCstjjj|t|j|jdS)N)rIr|)r r functionaldropoutr+rIr|rr r rrszDropout2.forward)rlrmrnr r:rrrr r ryrr@sr@c@s,eZdZeeddZeeddZdS)MulForDropout3cCs*|jrJ|||}||||_|Sr)rrrF)rrrrFrr r rrs   zMulForDropout3.forwardcCs$|j\}|j||dk}|ddfSr)rrFrr r rrs zMulForDropout3.backwardN)rlrmrnrrrrrr r r rrCs rCcs8eZdZdedeffdd ZdedefddZZS) Dropout3rI shared_dimcst||_||_dSr)rur:rIrE)r7rIrEryr rr:s  zDropout3.__init__rrcCsht|j}|jr |dkrt|Sdd|}t|j}d||j<tj|d|j i|k}t |||}|S)Nrrr*r ) r+rIr|r r2rrEr randr rCr)r7rrIr0 rand_shapemaskrr r rr s    zDropout3.forward) rlrmrnr r4r:rrrr r ryrrDsrDc@<eZdZdZededefddZededefddZd S) SwooshLFunctionz; swoosh_l(x) = log(1 + exp(x-4)) - 0.08*x - 0.035 rrc Cs|j}|jtjkr|tj}tjd|j|jd}d}tjj t ddt | }d|_t ||d||d}|sP|WdWdS|jt|d |j}|}d |d }||d ||t|} td kr| dksJ| dksJ| tj} || |jtjkstr|tj}|WdWdS1swYWddS1swYdS)Nrrr6{GzFrT@Q?rrr%o@r&p@rrr rrrrr rrr rrr&rrr rand_likerlrYruint8rr) rrrzerocoeffrrfloorceild_scaledd_intr r rrsB        "zSwooshLFunction.forwardy_gradcCs8|j\}d}|}d|d}|||d|}||S)NrKrr%rNr)rrYdrTrUrVr r rrDs  zSwooshLFunction.backwardNrlrmrnrorrrrr r r rrJs &rJc@eZdZdedefddZdS)SwooshLrrcCptjs tjr!tjd|j|jd}t||dd|dSdtj vr+t |S|j s3t |St |S)Return Swoosh-L activation.rrr6rL{Gz?rMk2)r r$r%r}rrr r&sysmodulesrJrrrbswoosh_l_forwardswoosh_lr7rrSr r rrP    zSwooshL.forwardNrlrmrnrrr r r rr^Or^c@r]) SwooshLOnnxrrcC.tjd|j|jd}t||dd|dS)r`rrr6rLrarMr rrr r#rgr r rr_zSwooshLOnnx.forwardNrir r r rrk^rjrkc@rI) SwooshRFunctionzo swoosh_r(x) = log(1 + exp(x-1)) - 0.08*x - 0.313261687 derivatives are between -0.08 and 0.92. rrc Cs|j}|jtjkr|tj}tjd|j|jd}tjj t ddt | }d|_t ||dd|d}|sN|WdWdS|jt|d |j}d }d }||d ||t|}td kr|dksyJ|dksJ|tj} || |jtjkstr|tj}|WdWdS1swYWddS1swYdS)Nrrr6FrTrratN0z ?rrK皙?rNr&rOrP) rrrrSrrrUrVrWrXr r rrls@      "zSwooshRFunction.forwardrYcCs,|j\}d}d}|||d|}||S)NrKrqrNrZ)rrYr[rUrVr r rrs zSwooshRFunction.backwardNr\r r r rroes $roc@r])SwooshRrrcCr_)Return Swoosh-R activation.rrr6rrarprb)r r$r%r}rrr r&rcrdrorrrbswoosh_r_forwardswoosh_rrgr r rrrhzSwooshR.forwardNrir r r rrrrjrrc@r]) SwooshROnnxrrcCrl)rsrrr6rrarprmrgr r rrrnzSwooshROnnx.forwardNrir r r rrvrjrvcCstjjtdd1|tj}|d}d||j}t |t dk||}|d|dWdS1sz x1.grad = z x2.grad = cSs,||d|d|dkS)Nrr)rrrr r risclose.s z4_test_activation_dropout_and_linear..isclose)r Sequentialr^rrrDrrr rrrrrrrorr~rr manual_seedrrrr)rrrrrm1m2x1x2seedrrYrrr r r#_test_activation_dropout_and_linearsr               rr&r*r)r=T)Zrrr~rctypingrrrrbrrwarningr torch.nnrrr is_availabler backendsrrrhasattrr deprecated torch.amprrtorch.cuda.ampr#r&objectr)r1rqr+r rrFunctionrr4rrrrrrrr=rrr"r#r2r3rr9rprr r?r@rCrDrJr^rkrorrrvr|r~rrrrrrrrrrrl getLoggersetLevelDEBUGset_num_threadsset_num_interop_threadsr r r rs          n D!8RX~   %1A    76   JN  I