o NiH@sRddlmZddlmZmZddlmZmZddlZddlm Z m Z m Z ddl m Z ddlm mZddlmZddlmZmZmZmZd d Zd d Zd dZdddZddZd?ddZd d!Z d@dAd&d'Z!dBd(d)Z"dCd+d,Z#dDd.d/Z$dEd2d3Z%dFd4d5Z&e'e#e$e%e&d6Z( - 7dGd8d9Z)Gd:d;d;e Z*dS)H) annotations)ceillog)TupleCallableN)nntensorTensor)Module) pad_sequence) rearrangerepeatpackunpackcCs|duSN)valrrY/home/ubuntu/.local/lib/python3.10/site-packages/x_transformers/autoregressive_wrapper.pyexistssrcCst|r|S|Sr)r)rdrrrdefaultrcOs|Srr)targskwargsrrridentitysr, cCs ||Sr)join)arr delimiterrrrrs rcCst|tr|S|f|Sr) isinstancetuple)rlengthrrr cast_tuplesr$csfdd}|S)Ncs2|j}||g|Ri|}|||Sr)trainingevaltrain)selfrr was_trainingoutfnrrinners  zeval_decorator..innerr)r,r-rr+reval_decorators r.#B ;cCs|j|dS)N)min)clampr)repsrrrr(rrcCsttt|  Sr)rtorch rand_likerrrr gumbel_noise+sr6?ư>cCs"t|}|t|||jddS)Ndim)r6maxargmax)logits temperaturer2noiserrr gumbel_sample.srAcs0|jD]}|jdkrfdd|jD|_qdS)Nacsg|]}|qSrr.0rr+rr 7sz$modify_cached_kv..)attn_intermediates layer_type cached_kv)cacher,interrr+rmodify_cached_kv4s  rKr9padtuple[int, int]cCsN|dkr|S|dkr| dn|j|d}d|}tj|g||R|dS)N)rrrr value)ndimFrM)rrMr;rPdims_from_rightzerosrrr pad_at_dim;s  rUc Csg|jdd|j|jR\}}}}|jdkr |jd|ks"J||ks*J||}|}tj||tjdd} tj||tjd} t||df|dd}||} || | | ddf} | S)Nr r)devicedtype).N)rPr;.) shaperWrXrQamaxr3arangelongrU) rlenspad_idbatchseq_lenrWrXpad_lens max_pad_len batch_arangeprompt_len_arangeoffsetalignedrrr align_rightCs(rg?cCs\tj|dd\}}tjtj|dddd}||k}tj|ddd}td||<|d ||S) NT) descendingr9r:r r9FrO-infr )r3sortcumsumrRsoftmaxrMfloatscatter)r>thres sorted_logitssorted_indices cum_probssorted_indices_to_removerrrtop_pWs  rv皙?cCsX|jd}t|t||}t||}t||\}}t|td}|d|||S)Nr9rkr ) rYrrr0r3topk full_likeroscatter_)r>frac_num_tokensk num_tokensrindprobsrrrtop_kcs  r@{Gz?cCs@|jdd}|jddd}t|||}t||ktd|SNr9r:Tr;keepdimrk)rnrZr3powwherero)r> min_p_pow min_p_ratior max_probslimitrrrtop_aps rcCs8|jdd}|jddd}||}t||ktd|Sr)rnrZr3rro)r>min_prrrrrrrys r)rvrrr?cCsJt||jddd}d||||}|||kt|jj }|S)zE Appendix A Algorithm 2 https://arxiv.org/abs/2309.09117 r9Trr )rrZ masked_fillr3finforXr<) expert_logitsamateur_logitsalphabetacutoffdiffscontrastive_decode_logitsrrrcontrastive_decode_fns rc seZdZ     d fdd Zeeddd d dd ed ed f d!ddZ eed d d e d d eedddd f d"ddZ  d#ddZ Z S)$AutoregressiveWrapperrrLFrwcsNt||_||_||_|j|_|dksJ||_||_|j|_||_ dS)Nr7) super__init__ pad_value ignore_indexnet max_seq_len mask_probadd_attn_z_lossadd_continuous_pred_headnext_embed_loss_weight)r(rrrrrr __class__rrrs   zAutoregressiveWrapper.__init__Nr7T prompt_lens Tensor | Nonefilter_logits_fnstr | Callable filter_kwargsdictc ) svt|rJd|j|dk|j}}t|gd\}}|j\}}t| tr:| tvs6Jdtt dt| } d}t|rLt |||j d}||}|}d}| oU|j j }tj|f|d}tj||d}t|D]}|dk}| r|jd k}| r|r|j jsJd |dd df}t|rt|fd d |j |fd ||d| \}}|r|}|ddd f}|jd d}|r|s| |fi| }||t|}|jd dj} |d | }!t|dd}||!}t|dd}t| d} |r|rt|fdd tj|| fd d}t|d|d}|jd }"|"krR|jd d d\}}#|dddf}|#dddf}$|"|dddf|$}$t|$d}%||%}t|d}t|s]qi||k}&|&jd drlnqit|rt |&d}'|'!j"d ddk}(|#|(|j }t|d|d}|d|df}t$||d\}|s|ddddfSt|d|d}t|d}||fS) Nzeos token not supported yetrL* nonly  are availabler^rWrr9the network cannot use cached key values when decoding outside the max sequence length. most likely because you are using absolute positional embedding. you can switch to rotary embeddings to resolve this issuecs|dd dddfS)N.r rr5rrrsz3AutoregressiveWrapper.beam_search..Treturn_intermediatesrI seq_start_posr:z b -> b beamsbeamsb ... -> (b beams) ...zb beams -> (b beams) 1cst|ddS)Nrr)r r5rrrr'sz6(b prev_beams) next_beams -> b (prev_beams next_beams))b)r;rizb beams -> (b beams)rjr z(b beams) seq -> b beams seq.z * beams nz(b beams) -> beams bzb beams n -> beams b n)%rrrWrrYr!strFILTER_LOGITS_FNrkeysrgrr can_cache_kvr3rTr[range can_cache_kv_outside_max_seq_lenrK log_softmaxr6rxindicesgatherr r catrlanyallrRrMrormrr))r(promptsr`rreturn_beams_and_scores eos_tokenr? stochasticrrrestrict_to_max_seq_lenrcache_kvrgreedyrW packed_shaper_ orig_seq_lenrr*rI should_cachescoresrciis_firstmax_len_exceededxr> new_cache log_probssamples next_scorescurr_num_beams sort_indicestop_beams_indicesflattened_beam_indices is_eos_tokensshifted_is_eos_tokensmaskr)rrr beam_searchs                 z!AutoregressiveWrapper.beam_searchr)rrrlist[Tensor] | Tensor amateur_modelModule | Tuple[Module] | Nonecontrastive_decode_kwargsdict | Tuple[dict]c ) s|j|dk} t|tr3t|dksJdt|rJdtdd|D|djd}t|dd }t|gd \}}g|j |jR\}}}t|t r`|t vs\Jd t t d t |}d}t|rrt|||jd }||}|}d}t|rt|}t| } t|t| ksJdgt|}t}t|D]\}}t|tr|j||<|qt|D]}|r|j dk}| r|r|jjsJd|dd df}t|r|jD]}|jdkrfdd|jD|_q|j|fd||d| \}}| r|jjr|}|dddf}t|rXtt||| D]=\}\}}} ||fd||d| \}!}"|!dddf}!|!j |j ksBJdt||!fi| }| rV|jrV|"||<q| rc|jddd}#n||fi| }$tj |$|dd}%t!"|%d}#t!j#||#fdd}t|sq||k}&|&j$dd%rnqt|rt&|&d}'|''j(dddk}(|)|(|j}|dd|df}t*||d \}|S)NrLrzprompts cannot be empty listzM`prompt_len` will be auto derived if prompts are passed in as list of TensorscSsg|]}|jdqSr)rYrCrrrrE{sz2AutoregressiveWrapper.generate..rT) batch_firstrrrrr9rrBcs(g|]}|dd dddfqS).r NrrCrrrrEs(rzBlogits dimension are not the same between amateur and expert modelrr:r rj)+rr!listlenrrrWr rrYrrrrrgrr$r enumeraterrr&rrrFrGrHrziprr=rRrnr3 multinomialrrrrMrormrr))r(rr`rr?rrrrrrrrrpsrrrWrr*rIamateur_cachesrmodule_rrrJr>ramateur amateur_cache!amateur_contrastive_decode_kwargsrnext_amateur_cachesamplefiltered_logitsrrrrrrrgenerate_s                 zAutoregressiveWrapper.generatecKs|jd|j|j|jf\}}}}||ddddf} } t| |k|j| } |jdkrjtj| j|j d} t | j j | dddf<t t||j|d} | j| ddj} t| d| d}|j|d|j| fd |||d |\}}|r|\}\}}n|}t|r|jd}|dd|df}|ddddf}|jjstjntj}|t|d | |d }|r||j}|r| |k}|ddddf}|ddddf}tj||d d}||}|||j }|s|S|||ffS)Nr rLrrr9r:r7)self_attn_kv_maskT)rreturn_attn_z_lossreturn_next_embed_predprepend_embedszb n c -> b c n)rnone) reduction)!rYrrrr3rrrrandnrWrrXr<r0intrxr zeros_likerpboolupdaterroutput_is_log_probrR cross_entropynll_lossr attn_z_lossdetachl1_lossmeanr)r(rreturn_outputsrrseqrradd_next_embed_lossinptargetrandnum_maskrrr*rIr>next_embed_pred init_embeds prepend_lenloss_fnloss embed_pred cont_targets cont_lossrrrforwardsZ         zAutoregressiveWrapper.forward)rrrLFrw)rrrrrr) rrrrrrrrrrrr)FN)__name__ __module__ __qualname__rr3no_gradr.rrrrrr __classcell__rrrrrsP&"r)r)r )r/)r7r8)r9rL)rMrNr)rh)rwN)rr)rw)rwr)+ __future__rmathrrtypingrrr3rrr torch.nnr torch.nn.functional functionalrRtorch.nn.utils.rnnr einopsr r rrrrrrr$r.r6rArKrUrgrvrrrrrrrrrrrsF