o
    ߥi&                     @   st   d Z ddlZddlZddlmZ G dd dejjZG dd dejjZG d	d
 d
ejjZ	G dd dejjZ
dS )zMultiple choice model.    N   )GLMModelc                       s`   e Zd Z		ddef fddZdd	d
ZdddZddedefddZ				dddZ
  ZS )GLMForMultiTokenClozeT        language_modelc                    $   t t|   || _|| _|| _d S N)superr   __init__modeltake_softmaxlength_penaltyselfr   r   r   	__class__ _/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/mglm/model/downstream.pyr
      s   
zGLMForMultiTokenCloze.__init__N Fc                 C      | j |||}|S r   r   
state_dictr   destinationprefix	keep_varssdr   r   r   r         z GLMForMultiTokenCloze.state_dictc                 C      | j j||dS N)strictr   load_state_dictr   r   r    r   r   r   r"         z%GLMForMultiTokenCloze.load_state_dictr   recursec                 C   r   N)r   r%   r   named_parametersr   r   r%   r   r   r   r(      r$   z&GLMForMultiTokenCloze.named_parametersc                 C   s  |d u r|  |||S d }t|jdkrc|jd d \}}|d|d}|jdg| dd  R  }|jdg| dd  R  }|d|d}|d|d}|d urc|d|d}| j ||||d^}	}
| jrztjjj	|	dd}	tj
|dtj|jd}|d|}tj
|dtj|jd}|d|}|	|||f }|| jdd}| jd	kr||jdd| j  }|d ur|d|}|g|
R S )
N      
prompt_posdimr   dtypedevicer   r   )r   lenshapereshapesizer   torchnn
functionallog_softmaxarangelongr3   	unsqueeze	expand_assumr   view)r   	input_idsposition_idsattention_mask
target_ids
logit_maskr.   num_choices
batch_sizeoutputsmems	batch_idsseq_idslogitsr   r   r   forward    sD   

zGLMForMultiTokenCloze.forwardTr   Nr   FTr   TNNN)__name__
__module____qualname__r   r
   r   r"   strboolr(   rN   __classcell__r   r   r   r   r   
   s    
	
r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )GLMForMultiTokenClozeFastTr   c                    r   r   )r	   rZ   r
   r   r   r   r   r   r   r   r
   J   s   
z"GLMForMultiTokenClozeFast.__init__c	                    s  | j |||ddd^}	}
| \ }|d}g |
D ]!}|ddddj  g| dd  R  }| qd fdd	}|d|}|jdg| d	d  R  }|||d|}|d|d}|d|d}| j |||gR  ^}	}
| jrtjj	j
|	dd
}	tj|dtj|jd}|d|}tj|dtj|jd}|d|}|	|||f }|| jdd
}| jdkr||jdd
| j  }d ur|d}|g|
R S )NTF)return_memorydetach_memoryr,   r   r   c                    s   d  d| | f}t|}tj||j|jddd}||ddk }|d 	d| d}|	  dd}tj
||fdd}|d}|S )Nr   r   )r3   r2   r,   r+   r/   )new_onesr8   trilr<   r3   r2   rA   r>   floatexpandcat)
seq_lengthsepmemory_lengthmidsmaskrH   enc_memsrG   r   r   build_dec_mask_matrixe   s   


z@GLMForMultiTokenClozeFast.forward.<locals>.build_dec_mask_matrixr+   r/   r1   r   )r   )r   r7   r>   r`   r6   appendr   r8   r9   r:   r;   r<   r=   r3   r?   r@   r   rA   )r   rB   rC   rD   dec_input_idsdec_position_idsdec_attention_maskdec_target_idsdec_logit_maskrI   rJ   max_dec_lenmax_enc_lenhiddenrj   rK   rL   rM   r   rh   r   rN   P   st   


z!GLMForMultiTokenClozeFast.forwardrO   rT   rU   rV   r
   rN   rY   r   r   r   r   rZ   H   s    rZ   c                       sV   e Zd Zd fdd	ZdddZdd	d
ZddedefddZ			dddZ	  Z
S )GLMForSingleTokenClozeFc                    s   t    || _|| _d S r   )r	   r
   r   r   )r   r   r   r   r   r   r
      s   

zGLMForSingleTokenCloze.__init__Nr   c                 C   r   r   r   r   r   r   r   r      r   z!GLMForSingleTokenCloze.state_dictTc                 C   r   r   r!   r#   r   r   r   r"      r$   z&GLMForSingleTokenCloze.load_state_dictr   r%   c                 C   r   r&   r'   r)   r   r   r   r(      r$   z'GLMForSingleTokenCloze.named_parametersc                 C   s   |d u r|  |||S t|jdksJ | j ||||d^}}tj|d|j|jd}	||	|f }
| jr?tj	j
j|
dd}n|
}|	d|}	||	|f }||
g|R S )Nr+   r-   r   r1   r,   r/   r   )r   r4   r5   r8   r<   r7   r2   r3   r   r9   r:   r;   r>   r?   )r   rB   rC   rD   rE   rF   r.   rI   rJ   rK   target_logitstarget_proboutputr   r   r   rN      s(   
zGLMForSingleTokenCloze.forward)FrP   rQ   rR   rS   )rT   rU   rV   r
   r   r"   rW   rX   r(   rN   rY   r   r   r   r   ru      s    

ru   c                       s(   e Zd Z	d fdd	Zdd Z  ZS )GLMForSequenceClassificationr   c                    sN   t    || _|| _|| _tj||| _tj	|| _
tj||| _d S r   )r	   r
   
pool_tokenr   	num_classr8   r9   Linear
pool_layerDropoutmultichoice_dropoutmultichoice_head)r   r   hidden_sizehidden_dropoutrz   r{   r   r   r   r
      s   
z%GLMForSequenceClassification.__init__c                 C   sV  d }t |jdkr@| jdksJ |jd d \}}|d|d}|jdg| dd  R  }|jdg| dd  R  }| |||^}}| jdkr`|tj|d|j	|j
d|f }n)| jdkry|tj|d|j	|j
d|d f }n| jd	kr|d d df }ntt| |}| |}	| |	}
|d ur|
d|}
|
g|R S )
Nr*   r   r+   r,   startr   r1   padcls)r4   r5   r{   r6   r7   r   rz   r8   r<   r2   r3   NotImplementedErrortanhr}   r   r   rA   )r   rB   rC   rD   rG   rH   rI   rJ   rx   multichoice_outputrM   r   r   r   rN      sF   




z$GLMForSequenceClassification.forward)r   rt   r   r   r   r   ry      s    ry   )__doc__r8   torch.nnmodeling_glmr   r9   Moduler   rZ   ru   ry   r   r   r   r   <module>   s   >N.