o Ni-@s<ddlmZddlZddlmZddlmZddlmZddlZddlm Z m Z ddl m Z ddl mZmZddlm mZddlZdd lmZmZmZmZdd lmZed gd Zd dZddZd+ddZd,ddZddZ d-ddZ!ddZ"dd Z#d.d!d"Z$d#d$Z%d%d&Z&Gd'd(d(e Z'Gd)d*d*e Z(dS)/) annotationsNrandom) nullcontext) namedtuple)nnpi)Module)grad_and_valuevmap) rearrangerepeatpackunpack)TransformerWrapperLosses)lossgenerator_loss critic_losscCs|duSN)valrr\/home/ubuntu/.local/lib/python3.10/site-packages/x_transformers/nonautoregressive_wrapper.pyexistsrcCst|r|S|Sr)r)rdrrrdefaultsr?cCsLtd||jd}|j|dd\}}t|td}|d|||S)Ndimz-inf)mathceilshapetopktorch full_likefloatscatter_)logitsthreskrindprobsrrrtop_k!s r0绽|=cCst||Sr)r'log)tepsrrrr2(sr2cCs"t|dd}tt|  S)Nrr)r' zeros_likeuniform_r2)r3noiserrr gumbel_noise+sr8?rcCs|t|dt|j|dS)Nr1r )maxr8argmax)r3 temperaturer!rrr gumbel_sample/sr=cCs t|kSrr)probrrr sample_prob4s r?cCstdS)Ng?)r?rrrr coin_flip7rr@c Csg|j|jR\}}}|jddd|j|d}tj||f|d}||d}|jddjdd}|jddd} || 8}||k} | |d| S)NrT)r!keepdimmindevicer F) r%rEsumclampr'rand masked_fillargsortr) masked_fill_) maskr>min_maskbatchseqrE num_to_maskr+randperm num_padding subset_maskrrrget_mask_subset_prob<srTcCsd|SNrrr3rrrlinear_scheduleMrrWcCst|tdS)z" https://arxiv.org/abs/2202.04200 r")r'cosrrVrrrcosine_schedulePsrYcs$eZdZfddZddZZS) SelfCriticcs*t||_|jj}t|d|_dSrU)super__init__net attn_layersr!rLinear to_logits)selfr]r! __class__rrr\Xs zSelfCritic.__init__cCs|j|dd}||S)NT)return_embeddings)r]r`)raxembedrrrforward_s zSelfCritic.forward)__name__ __module__ __qualname__r\rg __classcell__rrrbrrZWs rZc sfeZdZdZdddddddddd d d dfddZe   dddZ   dddZZ S)NonAutoregressiveWrapperzO https://arxiv.org/abs/1904.09324 https://arxiv.org/abs/2202.04200 Fg?g333333?g?linearNr9T) steps self_condself_cond_train_probno_replace_probrandom_token_probschedulecan_mask_prev_unmasked token_criticself_token_criticcritic_loss_weightuse_simple_mdlm_loss_weightrvTransformerWrapper | Nonec st| r t| r J||_|j}||_|j|_||_||_||_ |j |_ ||_ t |r1||_ |dkr9t|_ n|dkrAt|_ ntd|d|_| r]tt|j fdd}||_| |_||_|r~tt||_|rxtj||ddnd|_||_| |_| rt||_| |_ dS)Nrncosinezinvalid schedule cs|\}}|d|S)Nr9r)timesgradvaluegrad_and_value_schedule_fnrrloss_weight_fns  z9NonAutoregressiveWrapper.__init__..loss_weight_fnF)bias)!r[r\rr]emb_dimr! num_tokensmask_idrrrs max_seq_lenrocallable schedule_fnrWrY ValueErrorrr r rurpr Parameterr'randn null_embedr_ to_self_condrqrvrZrx)rar]rrorprqrrrsrtrurvrwrxryr!rrbrrr\isB    z!NonAutoregressiveWrapper.__init__ffffff?cKs4t| }t|d}t|jj}|j}|t dd|j d} ||j f} tj | |j |d} tj | d|d} || dd|j } |j}|rO|jnd}t| tt|j D]\}}|rh||nd}|j| f|dd|\}}|r||}t|rt||}||j }||}|t|djdd }t|t|dd }t| || } t|jr|| }t|d }||t||}nd|jdd }|d t|d }t|d }|dkr |j s|!| t"|j#j }|j$|dd j%}tj&|tj'd(d|d} | !| |j } q]|)||rt| d} | S)Nrr9rDT) sum_embedsreturn_logits_and_embeddingsgMbP?rr r<z b n 1 -> b nr"z b n -> b n 1r)dtypez1 n -> n)*rrnextr] parametersrEtrainingevalr'linspacerorfullrrlongrprziptolistreversedrangerr0r:softmaxr=wherervr r8gatherrurIfinforr&indicesr5boolscattertrain)ra batch_sizestart_temperature filter_thresnoise_level_scalekwargs sample_onerE was_trainingr|r%rOrLall_mask_num_tokens has_self_cond last_embedmask_num_tokenssteps_until_x0rpr+embedsannealing_scaler<r/ sampled_idsscores mask_indicesrrrgeneratesb            z!NonAutoregressiveWrapper.generatec!Ksg|j|jR\}}}||jksJ|} tj||ddd} tj||f|djdd } | | } | |j dd} | t | dk}|}d}|j d krctrc||j 8}t||j }||M}|jd krtrt||j|}tjd|j||f|d}t|||}||M}t||j|}|jr|j}t|jrt|j|fd d i|}Wdn1swY|j||d |rtjnt}||j|fi|}Wdn1swY|jjst j!nt j"}t#|j$r|t |d | dd}|$| }t%&d||}||'}n |||| |}t#|j(r*|r0t)||dSt*|t+|t,d}t||| }|(|}|| k }t -t |d|}|r]|} d}n|||j.} t)| ||S)NrDrrrr r9rBzb -> b 1rrdT)rzb n l -> b l nnone) reductionzb n, brz ... 1 -> ...)/r%rErcloner'emptyr6rHrJr)rrGr rrr@rTrsrandintrrrrprr?rqno_gradr]detachupdaterroutput_is_log_probF cross_entropynll_lossrreinxmultiplymeanrvrr=rr binary_cross_entropy_with_logitsrx)!rareonly_train_generatoronly_train_criticgenerator_sample_temperaturerbnrEorig_seq rand_timesbatched_randperm rand_probsnum_tokens_maskrLreplace_mask_id_mask frac_seq_leftno_replace_prob_maskrandom_token_prob_mask random_tokensmaskedrpcontextr+loss_fnr loss_weightsr generated critic_logits critic_labelsr total_lossrrrrgsx             z NonAutoregressiveWrapper.forward)rvrz)Nr9rr9)FFN) rhrirj__doc__r\r'rrrgrkrrrbrrlcs0 S Yrl)r)r1)r9r)r)) __future__rr#r contextlibr collectionsrr'rrtorch.nnr torch.funcr r torch.nn.functional functionalrreinopsr r rrx_transformers.x_transformersrrrrr0r2r8r=r?r@rTrWrYrZrlrrrrs6