o ߥi@sdZddlZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z m Z mZddlZddlmmZddlZddlmZddlmZmZddlmZddlmZddlmZmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*dd l+m,Z,dd l-m.Z.m/Z/m0Z0ddl1m2Z2ddl3m4Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ejdej=?dej=@dej=Ade5BZ4dZCdZDdgZEGdddeZFddZGGdddejjHZIejJjKd d!ZLd"d#ZMGd$d%d%ejjHZNd&d'ZOejJjKd(d)ZP   d>d*d+ZQGd,d-d-ejjHZRGd.d/d/ejjHZSGd0d1d1ejjHZTGd2d3d3ejjHZUGd4d5d5e0e&ZVd6ZWd7ZXe)d8eWGd9d:d:eVZYe.jZe7j[e,j\d;Gd_sz0load_tf_weights_in_chatglm_6b..z Skipping z [A-Za-z]+_\d+z_(\d+)kernelgammaweight output_biasbetabiasoutput_weightssquad classifierr i _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )renumpy tensorflow ImportErrorrerrorospathabspathinfotrainlist_variables load_variableappendzipsplitr,join fullmatchgetattrAttributeErrorlenint transposeshapeAssertionErrorargsr* from_numpydata)modelconfigtf_checkpoint_pathrLnptftf_path init_varsnamesarraysnamerbarraypointerm_name scope_namesnumer0r0r1load_tf_weights_in_chatglm_6bBs                     rwcs0eZdZdZfddZdejfddZZS) PrefixEncoderz The torch.nn model to encode the prefix Input shape: (batch-size, prefix-length) Output shape: (batch-size, prefix-length, 2*layers*hidden) c st|j|_|jr7tj|j|j|_tj tj |j|jtj tj |j|j |jd|_ dStj|j|j |jd|_dS)NrJ)super__init__prefix_projectionr*r Embedding pre_seq_len hidden_size embedding SequentialLinearTanh num_layerstransr/rh __class__r0r1rzs     zPrefixEncoder.__init__prefixcCs,|jr||}||}|S||}|SN)r{rr)r/r prefix_tokenspast_key_valuesr0r0r1forwards    zPrefixEncoder.forward) r3r4r5__doc__rzr*Tensorr __classcell__r0r0rr1rxs rxcCs*d|dtd|dd||S)zOpenAI's gelu implementation.??gQ63E?gHm?)r*tanhxr0r0r1 gelu_impls  rcCst|Sr)rrr0r0r1gelusrcsDeZdZdejdffdd ZddZd d d Zfd d ZZ S)RotaryEmbedding'Fcsxtd|td|d|}|}||_|r(tj||_ d|_ n| d|d|_ d|_ d|_ ||_dS)NrrrJinv_freq)ryrzr*arangefloathalf learnabler Parameterrmax_seq_len_cachedregister_buffer cos_cached sin_cached precision)r/dimbaserrrrr0r1rzs   zRotaryEmbedding.__init__cCsdSrr0)r/ state_dictrlocal_metadatastrict missing_keysunexpected_keys error_msgsr0r0r1_load_from_state_dictz%RotaryEmbedding._load_from_state_dictr Nc Cs|dur |j|}|jdus||jkr{|jrdn||_tj||j|jjd}td||j}tj ||fdd |j}|j tj krE| }|dddddf}|dddddf}|j tj krm| }| }|jrt||fS|||_|_|jd|df|jd|dffS)N)devicedtypezi,j->ijr.)rbrrr*rrrreinsumcattorbfloat16rcossinrr) r/rseq_dimseq_lentfreqsembrrr0r0r1rs*      $zRotaryEmbedding.forwardcs8|jdur ||j|_|jdur||j|_t|Sr)rrry_apply)r/fnrr0r1rs     zRotaryEmbedding._apply)r N) r3r4r5r*rrzrrrrr0r0rr1rs  rcCsP|dd|jddf|d|jdddf}}tj| |f|jddS)N.rrJr r)rbr*rndim)rx1x2r0r0r1 rotate_halfs 6rcCsdt||ddt||dd}}||t||||t||}}||fS)Nr rJ)Frsqueeze unsqueezer)qkrr position_idr0r0r1apply_rotary_pos_emb_indexs rc CsV|dur|d|d} } tj| |fdd}tj| |fdd}|j\} } }}| r-||f}nd}t|d}|r@|t||}|d|d|d|df}||d|d|dd}||d|d|dd}tjddd|j |j d}tj || dd| dd dddd d }|j|}|j r||j _| ||}n#|dks||d |j }|}||}tj|dd}||}|d|d|d|df}||d|d|dd}||d|d|dd}t|| dd}|j|}|dddd}|dd |f}|j|}|||f}|S) Nrr rrJrrrgr)rEalpha)r*rrbrmathsqrtsizeviewzerosrrbaddbmmrascale_mask_softmaxscale contiguousall masked_fill_rsoftmaxtypebmmpermute)r/ query_layer key_layer value_layerattention_maskhidden_size_per_partitionlayer_id layer_pastscaling_attention_score use_cachepast_key past_valuerbnhr~presentquery_key_layer_scaling_coeff output_size matmul_resultattention_scoresattention_probsr context_layernew_context_layer_shapeoutputsr0r0r1 attention_fns              rc szeZdZddejdffdd ZeddZ ddd Z   dd ej d ej d e e ej ej fd e de f ddZ ZS) SelfAttentionNTcstt|||_||_||_||_||_||_t |r$|j|jdn|j|jdt j dd|_ d|_ |dur>|||_n||_||j|_tt jj|d|j||d|_tt jj|j|||d|_dS)NrJrF)rrrrrFr)ryrrzrr~rnum_attention_heads!num_attention_heads_per_partitionposition_encoding_2drr*r rotary_embrhidden_size_per_attention_headinner_hidden_sizer r rquery_key_valuedense)r/r~rrrrF params_dtyperrr0r1rzwsH     zSelfAttention.__init__cCs||d|S)Nr)r)rrr0r0r1attention_mask_funcs z!SelfAttention.attention_mask_funcFcCsF|d}|||}tj|||d}|r!tdd|DS|S)a#Split a tensor along its last dimension. Arguments: tensor: input tensor. num_partitions: number of partitions to split the tensor contiguous_split_chunks: If True, make each chunk contiguous in memory. r rcss|]}|VqdSr)r)r>chunkr0r0r1r@z.)rrr*rZtuple)r/tensornum_partitionscontiguous_split_chunkslast_dim last_dim_size tensor_listr0r0r1split_tensor_along_last_dims z)SelfAttention.split_tensor_along_last_dim hidden_statesrrroutput_attentionsc Cs||}|dd|jd|jf} |j| }||d\} } } |jr| jd| jdd\} }| jd| jdd\}}|j | | dd\}}|dddddf dd |dddddf dd }}t | ||||\} }t |||||\}}tj| |g| jdd} tj||g|jdd} n| dd}|j | | dd\}}t | | |||\} } t|| | | ||j|||d \}}}||}||f}|r||f7}|S) q hidden_states: [seq_len, batch, hidden_size] attention_mask: [(1, 1), seq_len, seq_len] NrrrJr r)rr) r/rrrrrrrr)rrrrrr rrrrmaxrarrr*concatrrr)r/r  position_idsrrrrr mixed_raw_layernew_tensor_shaperrrq1q2k1k2rrblock_position_idsrrroutputrr0r0r1rsZ            zSelfAttention.forwardFNFF)r3r4r5r*rrz staticmethodrr rrrboolrrr0r0rr1rus03  rcs$eZdZfddZddZZS)GEGLUcsttj|_dSr)ryrzrr activation_fnr/rr0r1rzs  zGEGLU.__init__cCs&|jd|jdd\}}|||S)NrJr r)rrr)r/rrrr0r0r1rsz GEGLU.forward)r3r4r5rzrrr0r0rr1r s rcs2eZdZdddeejffdd ZddZZS)GLUNTcsrtt|||_||_||_|durd|}||_ttj j |j|j||d|_ ttj j |j|j||d|_ dS)Nr) ryr rzractivation_funcr~rr r*r r dense_h_to_4h dense_4h_to_h)r/r~rrrFr"rrr0r1rzs* z GLU.__init__cCs"||}||}||}|S)z> hidden_states: [seq_len, batch, hidden_size] )r#r"r$)r/r intermediate_parallelrr0r0r1r:s   z GLU.forward) r3r4r5rr*rrzrrr0r0rr1r s r c sheZdZddedejddffdd Z   ddejdejd ee ejejfd e d e f d d Z Z S)GLMBlockNTc sntt|||_|||d|_| |_t|||||| |jd|_|||d|_| |_ t ||||| d|_ dS)Neps)rrFrr)rrFrr) ryr&rzrinput_layernormrr attentionpost_attention_layernormrr mlp) r/r~rlayernorm_epsilonrrr layernormuse_biasrrrrr0r1rzKs0   zGLMBlock.__init__Fr rrrr c Cs||}|j|||||||d} | d} | dd} d|jd} || | }||} || }| | |}|rA|f| } | S|f| dd} | S)r )rrrrr rr NrJr)r*r+rr,r-)r/r rrrrrr attention_inputattention_outputsattention_outputrr mlp_input mlp_outputrr0r0r1rzs,       zGLMBlock.forwardr) r3r4r5r r*rrzrrrrrrr0r0rr1r&Is.5r&csreZdZdZdZdZeZdZdgZ fddZ de j fd d Z d d Zdd dZdddZefddZZS)ChatGLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. FT transformerr&c s*tj|jfi|tt||dSr)ryrz name_or_pathr)r/rhkwargsrr0r1rzszChatGLMPreTrainedModel.__init__modulecCsdS)zInitialize the weights.Nr0)r/r:r0r0r1 _init_weightsrz$ChatGLMPreTrainedModel._init_weightsc sz|j\}}fdd|D}tj|||f|d}|t|D]\}}d||ddd|f<q |d|dk}|S)Ncg|] }|jjqSr0tolistindexrh bos_token_idr>seqrr0r1 z4ChatGLMPreTrainedModel.get_masks..rr r)rbr*onestril_ enumerate unsqueeze_r) r/r&r batch_size seq_lengthcontext_lengthsricontext_lengthr0rr1 get_maskss     z ChatGLMPreTrainedModel.get_masksc s|j\}fdd|D}jrPtjtjdd|d}t|D]\}} ||||| df<q%fdd|D} tj| dd} tj|| fdd}|Stjtjdd|d}|sst|D] \}} |||| d<qf|S)Ncr<r0r=rArr0r1rCrDz;ChatGLMPreTrainedModel.get_position_ids..rrr c s>g|]}ttj|tjdtj|tjddfqS)rr )r*rrlongr)r>rN)rrKr0r1rCs" r) rbrr*rrPrrepeatrHstack) r/r&mask_positionsrgmaskrJrLrrMrNrr0)rr/rKr1get_position_idss>      z'ChatGLMPreTrainedModel.get_position_idscCst|tr ||_dSdSr) isinstance ChatGLMModelgradient_checkpointing)r/r:valuer0r0r1_set_gradient_checkpointings  z2ChatGLMPreTrainedModel._set_gradient_checkpointingc s<|dd}|ddtt|jdd|i|}||_|S)a'Instantiate the model. Args: kwargs: Input args. model_dir: The model dir used to load the checkpoint and the label information. Returns: The loaded model, which is initialized by transformers.PreTrainedModel.from_pretrained model_dirNcfgpretrained_model_name_or_pathr0)popryrfrom_pretrainedr[)clsr9r[rgrr0r1 _instantiates z#ChatGLMPreTrainedModel._instantiater)r3r4r5ris_parallelizablesupports_gradient_checkpointingr! config_classbase_model_prefix_no_split_modulesrzr Moduler;rOrUrZ classmethodrarr0r0rr1r6s   $r6aM This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a: Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`ChatGLM6BTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zdThe bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.cseZdZdZdeffdd ZddZdejfdd Z ej fd d Z e e d eeeed          ddeejdeejdeejdeeeejejfdfdeejdeedeedeedeedeeejdfeffddZZS)rWa The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. rhcs$t||j_|j_tj_|j_|j_|j _ |j _ |j _ jj_ |j _ |j_|j_ttjjjjjd_d_fddtjfddtj D_tjj d_jdurD]}d|_qstj_t|_ tj!d_"dSdS) N)num_embeddings embedding_dimrFc s*tjjj|jjtdjjd S)NT)rrr/r0rr) r&r~rr.rrr rr)rrr0r1 get_layerwsz(ChatGLMModel.__init__..get_layercsg|]}|qSr0r0)r>r)rkr0r1rCsz)ChatGLMModel.__init__..r(g?)#ryrzmax_sequence_lengthr~r*rrr vocab_sizerr.rrrr}r{r r r|word_embeddingsrX ModuleListrangelayersr final_layernorm parameters requires_gradrrPrrxprefix_encoderDropoutdropout)r/rhparamr)rkr/r1rz_sD     zChatGLMModel.__init__cC|jSrrnrr0r0r1get_input_embeddingsz!ChatGLMModel.get_input_embeddingsnew_embeddingscC ||_dSrrzr/r}r0r0r1set_input_embeddings z!ChatGLMModel.set_input_embeddingscCsp|jd|d|}|||}|||j|jd|j |j |j }| |}| gd d}|S)NrrrJ)rJr rrr!)rrexpandrrurrr}rrr~rwrrZ)r/rJrrrrr0r0r1 get_prompts  zChatGLMModel.get_promptzbatch_size, sequence_length) checkpoint output_typerdNr&rrr. inputs_embedsrr output_hidden_states return_dictr(c  s<|dur|n|jj}|dur|n|jj}|dur|n|jj}| dur$| n|jj} |jr2|jr2|r2d}|dur>|dur>td|durL|jdd\} } n|dur[|jdd\} } } ntd|durh| |}|dur|j dur|j |jd|j |j d}n tdgt|j}|dur|j||j d}|dur|jj|jj} }||vr|n| ||vrdnd}fd d |D}|j|||j |d }|j dur|durt| d |d |j |j }|dk}tj||fdd}|dd }|rdnd}|rdnd}|rdnd}|durtjd d |j d}n||j }t|jD]V\}}|r.||f}||}|jrL|jrLtjj||||t ||||}n||||t ||||d}|d}|rh||d f}|rw|||rrdnd f}q"|!|}|r||f}| stdd||||fDSt"||||dS)NFzDYou cannot specify both input_ids and inputs_embeds at the same timerJz5You have to specify either input_ids or inputs_embedsr)rJrrrETcsg|] }|qSr0)r>r?rA mask_tokenr0r1rCz(ChatGLMModel.forward..)rSrrTr rrrrr0)rrrrrr css|] }|dur|VqdSrr0)r>vr0r0r1r@2s z'ChatGLMModel.forward..)last_hidden_staterr  attentions)#rhr rruse_return_dictrXtraining ValueErrorrbrnr}rrrrr_rqrO mask_token_idgmask_token_idrUr*rFrrrrrarrHutilsrrrrr)r/r&rrrrrr rrrJrK_MASKgMASK use_gmaskrSprefix_attention_maskr presentsall_self_attentionsall_hidden_statesrMlayerr layer_retr0rr1rs                  zChatGLMModel.forward) NNNNNNNNN)r3r4r5rr!rzr{r*rrrrrCHATGLM_6B_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOCrr6rrrrrrr0r0rr1rWKs\9    rW) module_namecsZeZdZdeffdd ZddZddZ  d>d ed ee e fd e d e d ee e ff ddZ    d?de jdee jdee jdee jdee jd ef ddZ          d@dee jdee jdee jdeee jdee jdee jdee dee dee dee fddZedeee je jfd fd!e jd eee je jfd ffd"d#Zd$d%Ze   & ' ( ) * dAd+e d,eee e fd-efd.d/Ze   & ( ) * dBd+e d,eee e fd-efd0d1Ze     d?d2eed3eed4ee d5ee!ee jgeeffd6d7Z"dCd8efd9d:Z#d;ed efd.r.csg|] }|jjqSr0)r?rhr@rArr0r1rCrcsg|] \}}||gqSr0r0)r> mask_positionrN)rKr0r1rCsrcSsg|]}|qSr0r0)r>rr0r0r1rCs)r&rrrrE)rrSrT)rbrhrrr>rrr*rrrrYrPrrOrU)r/r&rrrrr9rJrrrseqsrS last_tokenrLr0)rr/rKr1prepare_inputs_for_generationsx    z=ChatGLMForConditionalGeneration.prepare_inputs_for_generationrlabelsrr rrc  Cs:|dur|n|jj}| dur| n|jj} |j|||||||| | d } | d} || ddd} d}|dur{| tj } | dddddf}|dddf}t dd}||j }|| d| d| d}| | j} || j}| s| f| dd}|dur|f|S|St|| | j| j| jd S) N) r&rrrrrr rrrr rJ.ri) ignore_index)losslogitsrr r)rhrrr7rrrrr*float32r rrrrrrr r)r/r&rrrrrrr rrtransformer_outputsr  lm_logitsr shift_logits shift_labelsloss_fctrr0r0r1rsL       z'ChatGLMForConditionalGeneration.forward.beam_idxcstfdd|DS)aL This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. Output shares the same memory storage as `past`. c 3sH|]}|dd|dj|dd|djfVqdS)rr N) index_selectrr)r>rrr0r1r@s  zAChatGLMForConditionalGeneration._reorder_cache..)r)rrr0rr1_reorder_cache s z.ChatGLMForConditionalGeneration._reorder_cachecCs|}|dd}ddgddgddgd d gd d gg}|D] }td |dd|d|}td|dd|d|}q|S)Nu[[训练时间]]u2023年,u，!u！:u：;u；z\?u？z([\u4e00-\u9fff])%srz\1%sr z%s([\u4e00-\u9fff])z%s\1)stripreplacerLsub)r/responsepunktsitemr0r0r1process_responses    z0ChatGLMForConditionalGeneration.process_responser Tffffff?ffffff?queryhistory max_lengthc Ks|durg}| dur t} | t|||||| d| } |s#|} n d} t|D]\} \}}| d| ||7} q)| dt||7} || gdd}||j}|jd i|| }| dt|ddd}| |}| |}|||fg}||fS) N)r num_beams do_sampletop_p temperaturelogits_processor[Round {}] 问：{} 答：{} [Round {}] 问：{} 答：ptreturn_tensorsrr&r0) rrXr%rHrr_rrgenerater>decoder)r/rrrrrrrrrr9 gen_kwargspromptrM old_queryrinputsrr0r0r1_chat,s<       z%ChatGLMForConditionalGeneration._chatc ks |durg}|durt}|t|||||d| } |s#|} n d} t|D]\} \} }| d| | |7} q)| dt||7} || gdd}||j}|jd i|| D](}| dt|ddd}| |}| |}|||fg}||fVqZdS) N)rrrrrrrrrrrr&r0) rrXr%rHrr_rrstream_generater>rr)r/rrrrrrrrr9rrrMrrrr new_historyr0r0r1 stream_chatWs>       z+ChatGLMForConditionalGeneration.stream_chatgeneration_configrstopping_criteriaprefix_allowed_tokens_fnc +s|jd|jd}}|dur|j}t|}|jdi|} |j|j}} t| tr/| g} | dduo:|j du} | rN|j durNt d|j dtn|j durj|j ||_ | sjt d|j d|j dt||j kr|jjrud nd } td | d |d |j d|dur|nt}|dur|nt}|j|||||d}|j||d}||} ||jdd}d} |j|fi| }|di|dddd}|jdddddf}|||}| ||}tjj|dd}|jrt j!|dd"dnt j#|ddt j$|dddfgdd}|j%|| |jjd} |&t'fdd| D(}|)dks<|||r>dS|Vq)NrrrzUsing `max_length`'s default (z) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.zBoth `max_new_tokens` (=z) and `max_length`(=z) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)decoder_input_idsr&zInput length of z is z, but `max_length` is set to zX. This can lead to unexpected behavior. You should consider increasing `max_new_tokens`.)rinput_ids_seq_lengthencoder_input_idsrr)rrr TF)rr rr) num_samples)rc3s|]}|kVqdSrr0)r>rM next_tokensr0r1r@rzBChatGLMForConditionalGeneration.stream_generate..r0)*rbrcopydeepcopyupdater@ eos_token_idrVr`getrmax_new_tokenswarningswarn UserWarningrrhrwarningrr_get_logits_processor_get_stopping_criteria_get_logits_warpernewfill_rrr functionalrrr* multinomialrargmaxrrmulsumrPr)r/r&rrrrr9rrrrhas_default_max_lengthinput_ids_string logits_warperunfinished_sequencesr' model_inputsrnext_token_logitsnext_token_scoresprobsr0rr1rs            z/ChatGLMForConditionalGeneration.stream_generatebitscKsX|dkrdSddlm}|jrtd|Sd|_||j_||j|fd|i||_|S)Nrr )rzAlready quantized.Tr) quantizationrrrrTrhrr7)r/rrr9rr0r0r1rs   z(ChatGLMForConditionalGeneration.quantizeinputc Cs|d}|d}d|vr|d}nd}d|vr|d}nd}d|vr'|d}nd}d |vr2|d }nd }t|tjkr?|}|j|j||||||d \}}td tj |tj |iS) Ntextrrrrrrr rT)rrrrzGeneration finished.) rr*rr>rrrrTrRESPONSEHISTORY) r/rrrrrrrrr0r0r1chats6     z$ChatGLMForConditionalGeneration.chat)FF)NNNN) NNNNNNNNNN)Nrr TrrN)NrTrrNr)&r3r4r5r!rzrrrrstrrrrr*r6rrdictrrr7rrrrno_gradrr`rrrrrrrrrrr0r0rr1r>s   & H    =  * (  nr)NTF)^rrrrQrLsysrtypingrrrrrrrr*torch.nn.functionalr rrtorch.utils.checkpointtorch.nnr r torch.nn.utilsr &transformers.generation.logits_processr transformers.generation.utilsrrrrtransformers.modeling_outputsrrrtransformers.modeling_utilsrtransformers.utilsrrrmodelscope.metainformodelscope.modelsrrrmodelscope.outputsrmodelscope.utilsrloggingmodelscope.utils.constantr configurationr! tokenizationr"platform_C_jit_set_profiling_mode_jit_set_profiling_executor_jit_override_can_fuse_on_cpu_jit_override_can_fuse_on_gpu get_loggerrr(CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LISTr%rwrgrxjitscriptrrrrrrrrr r&r6CHATGLM_6B_START_DOCSTRINGrrWregister_moduler chatglm_6brr0r0r0r1s$                J 6  v 1f_ 2p