o
    ߥi'                     @   s   d dl mZ d dlZd dlm  mZ dd Z		dddZd de	d	 fd
dZ
dd Z								ddedededede	de	dedefddZdd Z									d dededede	de	dededefddZdS )!    )ListNc              
   C   s.  |   \}}|r|}nd}ttj|||f| jd|d||}tj|tj| jd}|d	| }|r9|
 }|s=|rt|D ]M}	||	| |	 |kf }
|rS|

 }
d}t|
  d D ]0}|
| }|rud||	d|d dd|d f< |r||	|d df  |d | 8  < |d }q]qA|dk }||fS )z4Build masks and position id for left to right model.   )device)dtyper   r   Ng      ?)sizetorchtrilonesr   viewarangelong	unsqueeze	expand_asclonerange)data	eod_tokenreset_position_idsreset_attention_maskmicro_batch_size
seq_lengthatt_mask_batchattention_maskposition_idsb	eod_index
prev_indexji r   \/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/codegeex/inference.pyget_ltor_masks_and_position_ids	   sH   	
 $r!   Fc                 C   s0   |  |d  }t||||\}}|||fS )z#Generate batch from context tokens.)r
   
contiguouscudar!   )context_tokensr   r   r   r   tokensr   r   r   r   r    	get_batchB   s   
r'           Infc           
      C   s   |dkr| t | |d d k }|| |< |dkr^t j| ddd\}}t jtj|dddd}||k}|dd	df  |dd
d	f< d|d< t|dD ]}	||	 ||	  }|| |	 |< qM| S )zThis function has been mostly taken from huggingface conversational
    ai code at
        https://medium.com/huggingface/how-to-build-a-state-of-the-art-
             conversational-ai-with-transfer-learning-2d818ac26313r   ).r"   Nr(   Tr"   )
descendingdimr+   .Nr   ).r   )	r   topksortcumsumFsoftmaxr   r   r   )
logitstop_ktop_pfilter_valueindices_to_removesorted_logitssorted_indicescumulative_probssorted_indices_to_remover   r   r   r    top_k_logitsV   s*   

r;   c                 C   sD   g }| D ]}t |}||k r||g||   || q| |fS )N)lenextendappend)batchpad_idr   context_lengthsr&   context_lengthr   r   r    	pad_batchx   s   rC         ?return_scoresprompt_lengthr   bad_idstemperaturetoppr-   greedyc                 c   s    t ||j|\}}tj|}tj|}|  }t|||j\}}}t| |||||||||||	|
||d}|D ]\}}|d7 }|d urU|d d d |f |fV  q;dV  q;d S )N)	r   out_seq_lengthrE   rF   rG   rH   rI   r-   rJ   r   )NN)	rC   eos_token_idr   r$   
LongTensorminitemr'   sample_sequence_batch)model	tokenizerr   rK   r%   rE   rF   r   rG   rH   rI   r-   rJ   rA   context_tokens_tensorcontext_length_tensorrB   r&   r   r   batch_token_iteratorlengthsr   r   r    get_token_stream   sH   
rW   c                 C   s   | | }d| |  ||  S )Nr   )type_as)val1val2booleanr   r   r    switch   s   
r\   	recomputec           )   
   c   s\   |    t  |  }|j}d}|}d }|d}t|g 	 }|}|d u r=|d }||| kr=|| }t
|g 	 | }|	rUt|g 	 }||kr|rs| ||||
|d}|d d |d d d f }nP|dkr|d d d |f }|d d d |f }n|d d |d f |d}|d d |d f |d}| ||||d|
|d\}}|d d df |d }|d ur|D ]
}d|d d |f< q|rtj|ddd}n)| }|	rtj|dd} || }t|||d	}tj|dd}!tj|!dd
d}||k}"t|d d |f d||"}#|sB|	rB|dd}$| d|$d}%|%|" }%|%|   }%||%7 }|#|d d |f< ||k |" @ }&|&| @  }'|||'d< ||&B }t|}(|	rv|||ffV  n||fV  |d7 }|d7 }|(rn||ksZW d    d S W d    d S W d    d S 1 sw   Y  d S )Nr   r   )rF   rB   r"   T)
layer_pastget_key_valuerF   rB   ir,   )r3   r4   )num_samples)evalr   no_gradrN   rO   rL   r   zerosbyter$   r	   r   floatr
   r#   argmaxlog_softmaxr;   r0   r1   multinomialr\   gatherboollogical_notall))rQ   rR   r%   rA   r   r   r   rK   maxlenrE   rF   rG   rH   rI   r-   r]   rJ   rB   eos_idcounterorg_context_lengthr^   
batch_sizeis_doner&   rV   scoresr2   
tokens2usepositions2usebad_idprevorig_log_probs	log_probsstarted
new_tokensindices
new_scores
done_tokenjust_finisheddoner   r   r    rP      s   


	


\$rP   )FF)FNNNrD   rD   r(   F)	NFNNrD   rD   r(   FF)typingr   r   torch.nn.functionalnn
functionalr0   r!   r'   re   r;   rC   rj   intrW   r\   rP   r   r   r   r    <module>   sz   =
"	

6
