o iw@sddlZddlZddlmZddlmZmZddlmZddl m Z m Z m Z m Z ddlZddlmmZddlmZmZddlmZddlmZmZmZz dd lmZmZWneyned d d gZed d d gZYnweGdddZddZ ddZ!d.ddZ"e#         d/ddZ$d.ddZ%e#          d0dd Z&Gd!d"d"Z'ej(fd#e e)e ffd$d%Z*eGd&d'd'Z+e# (   )d1d*d+Z, )d2d,d-Z-dS)3N) namedtuple) dataclassfield)partial)CallableOptionalSequenceUnion) rearrangerepeat)Tensor)ProfilerActivityprofilerecord_function)GreedySearchDecoderOnlyOutputSampleDecoderOnlyOutputr sequencesscoresrc@sdeZdZUdZeed<eed<dZeed<dZeed<ee dZ e ed<d Z e e ed <d d Zd S) InferenceParamszInference parameters that are passed to the main model in order to efficienly calculate and store the context during inference. max_seqlenmax_batch_sizer seqlen_offsetbatch_size_offsetdefault_factorykey_value_memory_dictNlengths_per_samplecCs.||_||_d|_|jdur|jdSdS)Nr)rrrrzero_)selfrrrY/home/ubuntu/.local/lib/python3.10/site-packages/xformers/_flash_attn/utils/generation.pyreset#s  zInferenceParams.reset)__name__ __module__ __qualname____doc__int__annotations__rrrdictrrrr r!rrrr rs    rcCs,|t||ddk}||tddS)z 0 and top_p > 0, then top-k is applied first, then top-p. We assume that all sequences in the same batch have the same length. Arguments: input_ids: (batch, seq_len) max_length: int teacher_outputs (optional): (batch, seq_len). If provided, instead of sampling from the logits, the next token is taken from the teacher_outputs. Useful for testing. Returns: GreedySearchDecoderOnlyOutput or SampleDecoderOnlyOutput, with the following fields: sequences: (batch, max_length) scores: tuples of (batch, vocab_size) Nr7r_decoding_cache)tensor_parallelrrcs|jdk}|rtjdf|jtj|jd}nd}r|s*|||ddjjdd}n j|||jjdd}durC|ddfS|S)Nrr7dtyperD position_idsinference_paramsnum_last_tokensr5.) rr*fulllongrDr.rMrSrun) input_idsrZdecodingrYr.) batch_sizecgmodel vocab_sizerr get_logitss0 zdecode..get_logitscs@dus |jkrt|d}n dd|jf}|dS)Nr/r<rOr7)rrR unsqueeze)r.rZtoken)teacher_output_lenteacher_outputsrOr/r<rr sample_tokenss zdecode..sample_tokenscs<|jdkrdSdur|krdS|jdkrdSdS)NrFTr7)rall) current_tokenrZ) eos_token_id max_lengthrr should_stops zdecode..should_stop) enable_timingr)#Prompt processing + decoding time: .0fmsr5rr)rKhasattrrSupdate_graph_cacherZr!rr*cudaEvent distributedbarrierrecordappendr synchronizeprint elapsed_timerrcattuple)r_rcror/r<rOrnrjrdrTrbrq seqlen_ogrZrerkrpstartendrr output_clsr) rarbrnrorcrirjrOr/r<rdr decodebsN        rcCs|j\}}}|d} |j|| |fksJ|j|| fksJ|jtjtjfvs(J|dkr4|dks4Jd|dkr<||n|}|dkrH||n|}|dkrbt||d}t||t||t ||t ||tj |dd} tj |dd} dd } tj || | j d | | || | d d d df|k} | j dd}t|| | jdd}tj| d d d df| dd }tj|| d d dd fgdd}t|jdt|d |ddd}tj|ddjdd}t|d}||d d |f<||dfS)aAlgorithm 1 from [1] [1] Fast Inference from Transformers via Speculative Decoding Yaniv Leviathan, Matan Kalman, Yossi Matias https://arxiv.org/abs/2211.17192 Arguments: logits: Tensor of shape (batch_size, seqlen + 1, vocab_size) logits_draft: Tensor of shape (batch_size, seqlen, vocab_size) tokens_draft: Tensor of shape (batch_size, seqlen) Return: tokens: Tensor of shape (batch_size, seqlen + 1) num_generated_tokens: Tensor of shape (batch_size), with value in [1, seqlen + 1]. For each sequence in the batch, the number of valid tokens that were sampled by speculative sampling. r7r2r3rBrr)r5cSst|jdt|dddS)Nr)z ... -> ... 1r6indexz ... 1 -> ...)r gather)probstokensrrr sz$sample_speculative..rCN)rHz b -> b 1 d)drz b 1 d -> b drE)rr7)rKrWr*int64int32rNrHrIr1rAr9randrDrlwherer&argminclamprr rr rLrMFpad)r. logits_draft tokens_draftr/r<rObatch seqlen_p_1rdseqlenr probs_draftraccepted accepted_allfirst_rejected_idx probs_diffresample_probsresamplerrrr sample_speculativesF      ""  rc/ s |j\}}|dks Jd|dusJd| r[t|dsd|_t||j|||d| d|_|jj}|||t|ds>d|_t||j|||td|d| d|_|jj}|||n t||d }t||d }d%fd d }d&d d}t|||d}t t fi|}t ||| d}t ||| d}t ||||d}t ||||d}| rddl m }| d}| r| dkrtjtjt}|gg}}d}d} g}!||dkr||dd\}"}#||"||#nt|||d}$|||$d\}%}&| |$7} | r |tj||%gdd|$ddj}'t|&|'ddddf|tj||%gdd||$dd}(|d7}| rQ|tj||%gdd|$ddj})t|(|)t|(|&|%fi|\}"}*|!|*d| rot|"t|*||"ddd|*df||(ddd|*df|*d}+||+d|_|+dkr|jdn|j|_| rtj||dgdd},||,|*dddj}-t|d|-ddddf |j|dkrn.|j|dkr||dddddfdd\}"}#||"||#nt|||jd}$||dddddf|$d\}%}&| |$7} | rM|tj|,|%gdd|$ddj}'t|&|'ddddf|tj|dddddf|%gdd||$dd}(|d7}| r|tj|,|%gdd|$ddj})t|(|)t|(|&|%fi|\}"}*|!|*d| rt|"t|*||"ddd|*df||(ddd|*df|*d}+|j|+7_|+dkr|jdn|j|_| rtj|,|dgdd},||,|*dddj}-t|d|-ddddfq| rL| dkrtjtjtdt|dddtd|td t|!| d!d"d#tj|dd}tj|dd}| rv||j}-t||-dd|ddf|dkr}tnt }.|.||d$S)'a TD: WIP, for my own understanding, lightly tested. Only support batch_size == 1 for now. Speculative decoding, either greedy or with top-k or top-p sampling. If top-k = 0, don't limit the number of candidates (pure sampling). Top-k and top-p can be used together. If top_k > 0 and top_p > 0, then top-k is applied first, then top-p. We assume that all sequences in the same batch have the same length. Arguments: input_ids: (batch, seq_len) max_length: int Returns: GreedySearchDecoderOnlyOutput or SampleDecoderOnlyOutput, with the following fields: sequences: (batch, max_length) scores: tuples of (batch, vocab_size) r7z>Speculative decoding implementation only supports batch_size=1Nz@Speculative decoding implementation doesn't support eos_token_idrS)r7)decoding_seqlensrTrrUFc s|jdk}|r0|jd} tj|jdf|jtj|jd}|dddftj|tj|jd}nd}|r6|s@|||||dj } n||jdksIJ|j |||jdd| df} durg| ddfS| S)Nrr7TrVrX.) rrKr*r\rrDrrJr]r.rSr^) r_rZrcr[rbr`r cache_seqlensrYr.rdrr rePs<      z&decode_speculative..get_logitscSs|dksJ|gg}}t|D]*}|||d|dddf|j|djd7_|||ddqtj|ddddtj|ddfS)aSample `num_tokens` tokens from the model, given the previous logits. Also return the logits of the sampled tokens. Arguments: input_ids: (batch, seqlen) Return: tokens: (batch, num_tokens) scores: (batch, num_tokens), which contains @previous_logits and the logits of the next (num_tokens - 1) tokens. The logits of the last token isn't computed. r7r)Nr5)ranger}rrKrgr*rstack)r_ get_logits_fnrZ sample_fn num_tokensrrirrr rkus   $z)decode_speculative..sample_tokensrf)rcrb)rrrZr) AutoTokenizergpt2)rr5)r[r)TrrirsrtzNumber of calls to main model: zAcceptance rate: dz.2f%ru)r7Fr7)!rKrvrSrwrZr!rrr(rrR transformersrfrom_pretrainedr*rzr{rxr~timer}rHrr.rabsmaxritemrsumrr)/r_rc model_draftrospeculative_lookaheadr/r<rOrnrdrTrbrqdebugrarinference_params_draftrZrerksampling_kwargsrget_logits_mainget_logits_draftsample_tokens_mainsample_tokens_draftr tokenizerrrrnum_main_model_callsnum_draft_tokensnum_accepted_tokens_historyr scores_new n_spec_tokensr scores_draftscores_draft_refr. logits_refnum_generated_tokens num_generatedcur_ids scores_refrrrr decode_speculative sn !        %        $    ($    $$    (D   " ( rc@s*eZdZd ddZ     d dd ZdS) GenerationMixinNcKstN)NotImplementedError)rrarrWkwargsrrr allocate_inference_cache7sz(GenerationMixin.allocate_inference_cacher7r2r3Fc Ks4t|||f|||d|} |sd| _|r| S| jS)Nrf)rrr) rr_ror/r<rOreturn_dict_in_generate output_scoresroutputrrr generate:s zGenerationMixin.generater)r7r2r3FF)r"r#r$rrrrrr r6s rlayerscsNtjtjtjfvs J||d||ft|trt|}fdd|DS)Nrcsi|] }|tjdqS))rDrW)r*empty).0rrDrWkv_cache_shaperr Zsz,allocate_inference_cache..)r*float16bfloat16float32 isinstancer&r)rrnheadsheaddimrrDrWrrr rMs   rc@sdeZdZUdZeed<dZeed<dZdZe e dZ e ed<dZ dZ eeed<dZeeed<dS) DecodingCGCacherrrNr callablesrZr^)r"r#r$rr&r'rrDrWrr(rmempoolrZrrr^rrrrr r]s  rrrc  svdurttt|} | j} |dur| j}| |fjjfks-|jks-|jkri_d_ d_ t | |__||__t |drU||||} nt|jd|jj|jj} t|||jj|| |jj| |} tj|f|tj| d} t|||| | d_ tjj_ |D]}||fjvrt|j |||j |dj||f<qfdd}|_dj _S) Nrhead_dimrV)rrrrr)decoding_seqlenr n_warmupscs(|jdd\}}j||f|||S)Nr)rKr)r_rYrrarcacherr dispatchsz$update_graph_cache..dispatchr)rnextiter parametersrDrWrrrrrZgccollectrvrgetattrconfig hidden_sizenum_attention_headsnum_hidden_layersr*r\rrrxgraphsgraph_pool_handle capture_graphr^r)rcrrarrrrTrWr param_examplerD inf_cacherrrrrrr rwisp       rwc s^tt|j}tj||fdtj|dtj||fdtj|dj}||_jjdd<tj } | tj tj | &t|D] } ||djqI| tjrctjWdn1smwYtj | tj tj j|d||djWdn1swYfdd} |_| S)NrrVrX)poolcs2|jdd<||Sr)rcopy_replayrN) new_input_idsnew_position_idsrgraphrZr_r.rYrr r^s   zcapture_graph..run)rrrrDr*r\r]rrrxStream wait_streamcurrent_streamstreamrr.r~rzis_initializedr{ CUDAGraphr) rcrZrarrrrrDseqlen_offset_ogs_r^rrr rsL      r)r7r2r3) r7r2r3NNNr7FF) rr7r2r3NNr7FFF)rr7Nr)r7Nr).rr collectionsr dataclassesrr functoolsrtypingrrrr r*torch.nn.functionalnn functionalreinopsr r r torch.profilerr rrtransformers.generationrr ImportErrorrr1rArRinference_moderrrrrr&rrrwrrrrr s~      n; +    L