o
    ߥi'&                     @   sd  d dl Z d dlmZmZmZmZmZmZ d dlZd dl	m
Z
 d dlm
  mZ d dlZd dl	mZ d dlmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dl m!Z! ddl"m#Z# ddl$m%Z%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 erd dl2m3Z3 e! Z4e#j5ej6ej7de#j5ej8ej7dG dd de&Z9dS )    N)TYPE_CHECKINGCallableListOptionalTupleUnion)CrossEntropyLoss)GenerationConfigPreTrainedTokenizerStoppingCriteriaList)LogitsProcessorList)GenerateOutput)CausalLMOutputWithPast)Models)
OutputKeys)Tasks)
get_logger   )MODELS   )	QWenModelQWenPreTrainedModel)
BatchTokensTypeHistoryTypeStopWordsLogitsProcessordecode_tokens	get_batchget_stop_words_idsmake_context	pad_batchswitchtop_k_logits)BaseStreamer)module_namec                !       s  e Zd ZdgZdgZ fddZdd Zdd Z				d3d
dZ																												d4de	e
j de	eee
j   de	e
j de	e
j de	e
j de	e
j de	e
j de	e
j de	e
j de	e
j de	e de	e de	e de	e deeef fddZedeee
j  de
jdeee
j  fddZ	 	!d5d"ed#ed$e	e d%ed&edeeef fd'd(Z														d6d)e	e
j d*e	e d+e	e d,e	e d-e	eee
jgee f  d.e	e d/e	d0 deee
jf f fd1d2Z   Z!S )7QWenForTextGenerationz"h\.\d+\.attn\.rotary_emb\.inv_freqzh\.\d+\.attn\.masked_biasc                    s   t  | t|| _tj|j|jdd| _|j	r |j
r J d|j	r-| j  | j  |j
r:| j  | j  |   d S )NF)biasz,In config, bf16 and fp16 cannot both be true)super__init__r   transformernnLinearn_embd
vocab_sizelm_headbf16fp16bfloat16half	post_init)selfconfig	__class__ ^/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/qwen/text_generation.pyr'   ,   s   




zQWenForTextGeneration.__init__c                 C   s   | j S Nr-   )r3   r7   r7   r8   get_output_embeddings:   s   z+QWenForTextGeneration.get_output_embeddingsc                 C   s
   || _ d S r9   r:   )r3   new_embeddingsr7   r7   r8   set_output_embeddings=   s   
z+QWenForTextGeneration.set_output_embeddingsNc           	      K   s   | dd }|r"|d d df d}|d ur"|d d df d}| dd }| dd }|d urU|d u rU| dd }||dkd |rT|d d df d}nd }|d urd|d u rdd|i}nd|i}||| d	|||d
 |S )Ntoken_type_idsattention_maskposition_idsr   r   inputs_embeds	input_ids	use_cache)past_key_valuesrD   rA   r@   r>   )get	unsqueezelongcumsummasked_fill_update)	r3   rC   rE   rB   kwargsr>   r@   rA   model_inputsr7   r7   r8   prepare_inputs_for_generation@   s2   
z3QWenForTextGeneration.prepare_inputs_for_generationrC   rE   r@   r>   rA   	head_maskrB   encoder_hidden_statesencoder_attention_masklabelsrD   output_attentionsoutput_hidden_statesreturn_dictreturnc                 C   s   |d ur|n| j j}| j|||||||||	||||d}|d }| |}d }|
d urZ|
|j}
|dd dd d f  }|
ddd f  }t }||d|	d|d}|sp|f|dd   }|d urn|f| S |S t
|||j|j|jdS )N)rE   r@   r>   rA   rO   rB   rP   rQ   rD   rS   rT   rU   r   .r?   r   )losslogitsrE   hidden_states
attentions)r4   use_return_dictr(   r-   todevice
contiguousr   viewsizer   rE   rY   rZ   )r3   rC   rE   r@   r>   rA   rO   rB   rP   rQ   rR   rD   rS   rT   rU   transformer_outputsrY   	lm_logitsrW   shift_logitsshift_labelsloss_fctoutputr7   r7   r8   forwardd   sL   
zQWenForTextGeneration.forwardbeam_idxc                    s   t  fdd| D S )Nc                 3   s&    | ]}t  fd d|D V  qdS )c                 3   s$    | ]}| d  |jV  qdS )r   N)index_selectr\   r]   ).0
past_staterh   r7   r8   	<genexpr>   s
    
zAQWenForTextGeneration._reorder_cache.<locals>.<genexpr>.<genexpr>Ntuple)rj   
layer_pastrl   r7   r8   rm      s    
z7QWenForTextGeneration._reorder_cache.<locals>.<genexpr>rn   )rE   rh   r7   rl   r8   _reorder_cache   s   z$QWenForTextGeneration._reorder_cacheYou are a helpful assistant.T	tokenizerqueryhistorysystemappend_historyc                 C   s   |d u rg }t ||||d| jjd\}}t| jj|}t|g| j}	| j|	|dd}
t	|
d |t
|t
|| jjdd}|rH|||f tj|tj|iS )Ni   )ru   rv   max_window_sizechat_formatF)stop_words_idsreturn_dict_in_generater   )raw_text_lencontext_lengthry   verbose)r   generation_configry   r   torchtensorr\   r]   generater   lenappendr   RESPONSEHISTORY)r3   rs   rt   ru   rv   rw   raw_textcontext_tokensrz   rC   outputsresponser7   r7   r8   chat   s<   	
	zQWenForTextGeneration.chatinputsr   logits_processorstopping_criteriaprefix_allowed_tokens_fnsynced_gpusstreamerr"   c                    s   | dd }	|	d u r|d urt|dd }	|	d u rt| jdd }	|	d ur:t|	| jjd}
|d u r5t|
g}n||
 t j||||||fd|i|S )Nrz   )rz   eos_token_idr   )	popgetattrr   r   r   r   r   r&   r   )r3   r   r   r   r   r   r   r   rL   rz   stop_words_logits_processorr5   r7   r8   r      s:   
zQWenForTextGeneration.generate)NN)NNNNNNNNNNNNNN)rr   T)NNNNNNN)"__name__
__module____qualname___keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpectedr'   r;   r=   rN   r   r   
LongTensorr   TensorFloatTensorboolr   r   rg   staticmethodrq   r
   strr   r   r	   r   r   r   intr   r   r   __classcell__r7   r7   r5   r8   r$   &   s    
&	


=

.	r$   ):warningstypingr   r   r   r   r   r   r   torch.nnr)   torch.nn.functional
functionalFtorch.utils.checkpointr   transformersr	   r
   r   &transformers.generation.logits_processr   transformers.generation.utilsr   transformers.modeling_outputsr   modelscope.metainfor   modelscope.outputsr   modelscope.utils.constantr   modelscope.utils.loggerr    r   backboner   r   qwen_generation_utilsr   r   r   r   r   r   r   r   r    r!   !transformers.generation.streamersr"   loggerregister_moduletext_generationqwen_7br   r$   r7   r7   r7   r8   <module>   s0    0