o
    i                     @   s  d Z ddlZddlmZ ddlmZmZ ddlZddlmZ ddl	m
Z
mZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' e%(e)Z*da+dd Z,dd Z-dSddZ.dSddZ/dSddZ0dd Z1G dd dej2j3Z4G d d! d!ej2j3Z5G d"d# d#Z6dTd$d%Z7d&d' Z8			dUd(d)Z9G d*d+ d+ej:Z;G d,d- d-ej:Z<G d.d/ d/ej:Z=G d0d1 d1ej:Z>G d2d3 d3ej:Z?G d4d5 d5ej:Z@G d6d7 d7eZAG d8d9 d9ej:ZBG d:d; d;ej:ZCG d<d= d=ej:ZDG d>d? d?ej:ZEe!G d@dA dAeZFe!G dBdC dCeFZGe!G dDdE dEeFZHG dFdG dGej:ZIe!dHdIG dJdK dKeFZJe!G dLdM dMeFZKe!G dNdO dOeFZLe!G dPdQ dQeFZMg dRZNdS )VzPyTorch MRA model.    N)Path)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)load   )ACT2FN)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringis_cuda_platformis_ninja_availableis_torch_cuda_availablelogging   )	MraConfigc                     sD   t t jjjd d   fdd} | g d}td|ddad S )	Nkernelsmrac                    s    fdd| D S )Nc                    s   g | ]} | qS  r    ).0file
src_folderr    X/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/mra/modeling_mra.py
<listcomp>4       z:load_cuda_kernels.<locals>.append_root.<locals>.<listcomp>r    )filesr#   r    r%   append_root3   s   z&load_cuda_kernels.<locals>.append_root)zcuda_kernel.cuzcuda_launch.cuztorch_extension.cppcuda_kernelT)verbose)r   __file__resolveparentr	   mra_cuda_kernel)r)   	src_filesr    r#   r%   load_cuda_kernels/   s   r1   c                 C   s   t |  dkrtdt | dkrtd| ddkr#td| ddkr.td| jd	d
jdd	}| }| }| }t	||||\}}|dd	dddddddf }||fS )z8
    Computes maximum values for softmax stability.
       z.sparse_qk_prod must be a 4-dimensional tensor.   'indices must be a 2-dimensional tensor.    z>The size of the second dimension of sparse_qk_prod must be 32.r
   z=The size of the third dimension of sparse_qk_prod must be 32.dimN)
lensize
ValueErrormaxvalues	transpose
contiguousintr/   	index_max)sparse_qk_prodindicesquery_num_blockkey_num_block
index_valsmax_valsmax_vals_scatterr    r    r%   
sparse_max;   s   $rJ   r5   c                 C   s   t |  dkrtdt | dkrtd| jd |jd kr&td| j\}}|| }tj|dtj|jd}| |||} | |dddf ||  ddf } | S )zN
    Converts attention mask to a sparse mask for high resolution logits.
    r3   z$mask must be a 2-dimensional tensor.r4   r   zBmask and indices must have the same size in the zero-th dimension.dtypedeviceN)	r:   r;   r<   shapetorcharangelongrM   reshape)maskrD   
block_size
batch_sizeseq_len	num_block	batch_idxr    r    r%   sparse_maskW   s   
&rY   c           	      C   s"  |   \}}}|  \}}}|| dkrtd|| dkr"td| ||| ||dd} |||| ||dd}t|   dkrJtdt|  dkrVtdt|  d	krbtd
|  ddkrmtd| ddkrxtd|  } | }| }| }t| || S )z7
    Performs Sampled Dense Matrix Multiplication.
    r   zTquery_size (size of first dimension of dense_query) must be divisible by block_size.Pkey_size (size of first dimension of dense_key) must be divisible by block_size.r9   r6   r2   z+dense_query must be a 4-dimensional tensor.)dense_key must be a 4-dimensional tensor.r3   r4   r
   r5   z.The third dimension of dense_query must be 32.z,The third dimension of dense_key must be 32.)	r;   r<   rR   r?   r:   r@   rA   r/   mm_to_sparse)	dense_query	dense_keyrD   rT   rU   
query_sizer8   _key_sizer    r    r%   r\   n   s.   r\   c           	      C   s  |  \}}}|| dkrtd|  d|krtd|  d|kr'td|||| ||dd}t|   d	krAtd
t|  d	krMtdt|  dkrYtd| ddkrdtd|  } | }| }| }t| |||}|dd||| |}|S )zP
    Performs matrix multiplication of a sparse matrix with a dense matrix.
    r   rZ   r3   zQThe size of the second dimension of sparse_query must be equal to the block_size.r
   zPThe size of the third dimension of sparse_query must be equal to the block_size.r9   r6   r2   ,sparse_query must be a 4-dimensional tensor.r[   r4   r5   z8The size of the third dimension of dense_key must be 32.)	r;   r<   rR   r?   r:   r@   rA   r/   sparse_dense_mm)	sparse_queryrD   r^   rE   rT   rU   ra   r8   dense_qk_prodr    r    r%   rc      s.   rc   c                 C   s    | | | t j| |dd  S )Nfloorrounding_mode)rO   divrQ   )rD   dim_1_blockdim_2_blockr    r    r%   transpose_indices   s    rl   c                   @   s2   e Zd Zedd Zedd Zed	ddZdS )
MraSampledDenseMatMulc                 C   &   t ||||}| ||| || _|S N)r\   save_for_backwardrT   )ctxr]   r^   rD   rT   rC   r    r    r%   forward      zMraSampledDenseMatMul.forwardc                 C   sj   | j \}}}| j}|d| }|d| }t|||}t|dd|||}	t||||}
|
|	d d fS Nr   r9   r6   )saved_tensorsrT   r;   rl   rc   r?   )rq   gradr]   r^   rD   rT   rE   rF   	indices_Tgrad_key
grad_queryr    r    r%   backward   s   zMraSampledDenseMatMul.backwardr5   c                 C      t | |||S ro   )rm   apply)r]   r^   rD   rT   r    r    r%   operator_call      z#MraSampledDenseMatMul.operator_callNr5   __name__
__module____qualname__staticmethodrr   rz   r}   r    r    r    r%   rm      s    


rm   c                   @   s0   e Zd Zedd Zedd Zedd ZdS )MraSparseDenseMatMulc                 C   rn   ro   )rc   rp   rE   )rq   rd   rD   r^   rE   rC   r    r    r%   rr      rs   zMraSparseDenseMatMul.forwardc           
      C   s`   | j \}}}| j}|d|d }t|||}t|dd|||}t|||}	|	d |d fS rt   )ru   rE   r;   rl   rc   r?   r\   )
rq   rv   rd   rD   r^   rE   rF   rw   rx   ry   r    r    r%   rz      s   zMraSparseDenseMatMul.backwardc                 C   r{   ro   )r   r|   )rd   rD   r^   rE   r    r    r%   r}      r~   z"MraSparseDenseMatMul.operator_callNr   r    r    r    r%   r      s    

	r   c                   @   s   e Zd Zedd ZdS )MraReduceSumc                 C   s  |   \}}}}t|   dkrtdt|  dkr td|   \}}}}|  \}}| jdd|| |} tj| dtj|jd}tj	||dd	 |d d d f |  || }	tj
|| |f| j| jd}
|
d|	| |||}|||| }|S )
Nr2   rb   r3   r4   r7   r   rK   rf   rg   )r;   r:   r<   sumrR   rO   rP   rQ   rM   ri   zerosrL   	index_add)rd   rD   rE   rF   rU   rW   rT   r`   rX   global_idxestempoutputr    r    r%   r}      s$   &
zMraReduceSum.operator_callN)r   r   r   r   r}   r    r    r    r%   r      s    r   c                 C   s  |   \}}}|| }d}	|durl||||jdd}
| ||||jdd|
dddddf d  }|||||jdd|
dddddf d  }|durk|||||jdd|
dddddf d  }	n5|tj||tj| jd }
| ||||jdd}|||||jdd}|dur|||||jdd}	t||	ddt
| }|jdddj}|dur|d	|
dddddf |
dddddf  d
k    }||
||	fS )z/
    Compute low resolution approximation.
    Nr9   r7   r6   ư>rK   T)r8   keepdims     @g      ?)r;   rR   r   rO   onesfloatrM   meanmatmulr?   mathsqrtr=   r>   )querykeyrT   rS   valuerU   rV   head_dimnum_block_per_row	value_hattoken_count	query_hatkey_hatlow_resolution_logitlow_resolution_logit_row_maxr    r    r%   get_low_resolution_logit  s6   :r   c                 C   sT  | j \}}}|dkr3|d }tj||| jd}	tjtj|	| d|d}
| |
dddddf d  } |dkrk| ddd|ddf d | ddd|ddf< | ddddd|f d | ddddd|f< tj| |d|ddd	d
}|j}|dkr|j	j
ddj	}| |ddddf k }||fS |dkrd}||fS t| d)zZ
    Compute the indices of the subset of components to be used in the approximation.
    r   r3   rM   )diagonalNg     @r9   TF)r8   largestsortedfullr7   sparsez# is not a valid approx_model value.)rN   rO   r   rM   triltriutopkrR   rD   r>   minr   r<   )r   
num_blocksapprox_modeinitial_prior_first_n_blocksinitial_prior_diagonal_n_blocksrU   total_blocks_per_rowr`   offset	temp_maskdiagonal_mask
top_k_valsrD   	thresholdhigh_resolution_maskr    r    r%   get_block_idxes6  s.   r   c	           $      C   s  t du rt|  S |  \}	}
}}|	|
 }|| dkr!td|| }| |||} ||||}||||}|dure| |dddddf  } ||dddddf  }||dddddf  }|dkrvt| ||||\}}}}n(|dkrt  t| |||\}}}}W d   n1 sw   Y  nt	dt  || }t
|||||\}}W d   n1 sw   Y  tj| |||dt| }t||||\}}|| }|dur|dd	t||dddddddf    }t|}t||||}t||||}|dkrt|| d|  |dddddf  }t||dddddddf d	d	|d	|||}|jd
ddddddf d	d	|||}|d	d	|||| } |durq| | } t| | dk  }!||!dddddf  }||! }t|  | dk  }"||"dddddf  }||" }|| |dddddf |dddddf  d  }#n|dkr||dddddf d  }#nt	d|dur|#|dddddf  }#|#|	|
||}#|#S )z0
    Use Mra to approximate self-attention.
    Nr   z4sequence length must be divisible by the block_size.r   r   z&approx_mode must be "full" or "sparse")rT   r   r   r9   r7   r   z-config.approx_mode must be "full" or "sparse")r/   rO   
zeros_likerequires_grad_r;   r<   rR   r   no_grad	Exceptionr   rm   r}   r   r   rJ   rY   expr   r   r   repeatr   r   )$r   r   r   rS   r   r   rT   r   r   rU   num_headrV   r   
meta_batchr   r   r   r   r   r`   low_resolution_logit_normalizedrD   r   high_resolution_logitrH   rI   high_resolution_attnhigh_resolution_attn_outhigh_resolution_normalizerlow_resolution_attnlow_resolution_attn_outlow_resolution_normalizerlog_correctionlow_resolution_corrhigh_resolution_corrcontext_layerr    r    r%   mra2_attention\  s   




.

"
.
.
 
r   c                       s*   e Zd ZdZ fddZdddZ  ZS )MraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t    tj|j|j|jd| _t|jd |j| _	t|j
|j| _tj|j|jd| _t|j| _| dt|jdd  t|dd| _| jdtj| j tj| jjd	d
d d S )N)padding_idxr3   epsposition_ids)r   r9   position_embedding_typeabsolutetoken_type_idsrK   F)
persistent)super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrO   rP   expandgetattrr   r   r   r;   rQ   rM   selfconfig	__class__r    r%   r     s   

zMraEmbeddings.__init__Nc                 C   s   |d ur	|  }n|  d d }|d }|d u r$| jd d d |f }|d u rNt| drC| jd d d |f }||d |}|}ntj|tj| jjd}|d u rW| 	|}| 
|}	||	 }
| jdkrn| |}|
|7 }
| |
}
| |
}
|
S )Nr9   r   r   r   rK   r   )r;   r   hasattrr   r   rO   r   rQ   rM   r   r   r   r   r   r   )r   	input_idsr   r   inputs_embedsinput_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr   
embeddingsr   r    r    r%   rr     s,   







zMraEmbeddings.forward)NNNNr   r   r   __doc__r   rr   __classcell__r    r    r   r%   r     s    r   c                       s(   e Zd Zd fdd	ZdddZ  ZS )MraSelfAttentionNc              
      sf  t    |j|j dkrt|dstd|j d|j dtd u}t rNt rNt	 rN|sNzt
  W n tyM } ztd|  W Y d }~nd }~ww |j| _t|j|j | _| j| j | _t|j| j| _t|j| j| _t|j| j| _t|j| _|d ur|n|j| _|jd |j | _t| jt|jd d | _|j| _|j| _|j | _ d S )	Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zGCould not load the custom kernel for multi-scale deformable attention: r5   r3   )!r   r   r   num_attention_headsr   r<   r/   r   r   r   r1   r   loggerwarningrA   attention_head_sizeall_head_sizer   Linearr   r   r   r   attention_probs_dropout_probr   r   r   block_per_rowrW   r   r   r   r   )r   r   r   kernel_loadeder   r    r%   r   
  s:   


zMraSelfAttention.__init__c              
   C   s  |j \}}}| ||d| j| jdd}| ||d| j| jdd}| ||d| j| jdd}d|d  }| 	d| jd
|| j | }d}	| j|	k r|| j||	| j f}
tj|tj|
|jdgdd}tj|tj|
|jdgdd}tj|tj|
|jdgdd}t| | | | | j| j| j| jd	}| j|	k r|d d d d d d d | jf }|
|| j|| j}|d
ddd }| d d | jf }|j| }|f}|S )Nr9   r   r3         ?r   r5   r   r7   )r   r   r   r   r
   r6   )rN   r   viewr   r   r?   r   r   squeezer   rR   rA   rO   catr   rM   r   r   rW   r   r   r   permuter@   r;   r   )r   hidden_statesattention_maskrU   rV   r`   query_layer	key_layervalue_layergpu_warp_sizepad_sizer   new_context_layer_shapeoutputsr    r    r%   rr   -  sV   	

"
zMraSelfAttention.forwardro   r   r   r   r   rr   r   r    r    r   r%   r   	  s    #r   c                       8   e Zd Z fddZdejdejdejfddZ  ZS )MraSelfOutputc                    sB   t    t|j|j| _tj|j|jd| _t|j	| _
d S Nr   )r   r   r   r  r   denser   r   r   r   r   r   r   r    r%   r   n     
zMraSelfOutput.__init__r  input_tensorreturnc                 C   &   |  |}| |}| || }|S ro   r  r   r   r   r  r  r    r    r%   rr   t     

zMraSelfOutput.forwardr   r   r   r   rO   Tensorrr   r   r    r    r   r%   r  m      $r  c                       s0   e Zd Zd fdd	Zdd ZdddZ  ZS )	MraAttentionNc                    s.   t    t||d| _t|| _t | _d S )N)r   )r   r   r   r   r  r   setpruned_heads)r   r   r   r   r    r%   r   |  s   

zMraAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   r7   )r:   r   r   r   r   r%  r   r   r   r   r   r  r   union)r   headsindexr    r    r%   prune_heads  s   zMraAttention.prune_headsc                 C   s2   |  ||}| |d |}|f|dd   }|S Nr   r   )r   r   )r   r  r  self_outputsattention_outputr  r    r    r%   rr     s   zMraAttention.forwardro   )r   r   r   r   r)  rr   r   r    r    r   r%   r#  {  s    r#  c                       2   e Zd Z fddZdejdejfddZ  ZS )MraIntermediatec                    sD   t    t|j|j| _t|jt	rt
|j | _d S |j| _d S ro   )r   r   r   r  r   intermediate_sizer  
isinstance
hidden_actstrr   intermediate_act_fnr   r   r    r%   r     s
   
zMraIntermediate.__init__r  r  c                 C      |  |}| |}|S ro   )r  r3  r   r  r    r    r%   rr        

zMraIntermediate.forwardr   r    r    r   r%   r.    s    r.  c                       r  )	MraOutputc                    sB   t    t|j|j| _tj|j|jd| _t	|j
| _d S r  )r   r   r   r  r/  r   r  r   r   r   r   r   r   r   r    r%   r     r  zMraOutput.__init__r  r  r  c                 C   r  ro   r  r  r    r    r%   rr     r  zMraOutput.forwardr   r    r    r   r%   r7    r"  r7  c                       s.   e Zd Z fddZdddZdd Z  ZS )	MraLayerc                    sB   t    |j| _d| _t|| _|j| _t|| _t	|| _
d S Nr   )r   r   chunk_size_feed_forwardseq_len_dimr#  	attentionadd_cross_attentionr.  intermediater7  r   r   r   r    r%   r     s   


zMraLayer.__init__Nc                 C   sB   |  ||}|d }|dd  }t| j| j| j|}|f| }|S r*  )r<  r   feed_forward_chunkr:  r;  )r   r  r  self_attention_outputsr,  r  layer_outputr    r    r%   rr     s   
zMraLayer.forwardc                 C   s   |  |}| ||}|S ro   )r>  r   )r   r,  intermediate_outputrA  r    r    r%   r?    s   
zMraLayer.feed_forward_chunkro   )r   r   r   r   rr   r?  r   r    r    r   r%   r8    s    
	r8  c                       s.   e Zd Z fddZ				dddZ  ZS )	
MraEncoderc                    s:   t     | _t fddt jD | _d| _d S )Nc                    s   g | ]}t  qS r    )r8  )r!   r`   r   r    r%   r&     r'   z'MraEncoder.__init__.<locals>.<listcomp>F)	r   r   r   r   
ModuleListrangenum_hidden_layerslayergradient_checkpointingr   r   rD  r%   r     s   
 
zMraEncoder.__init__NFTc           
      C   st   |rdnd }t | jD ]\}}|r||f }|||}	|	d }q|r'||f }|s4tdd ||fD S t||dS )Nr    r   c                 s   s    | ]	}|d ur|V  qd S ro   r    )r!   vr    r    r%   	<genexpr>  s    z%MraEncoder.forward.<locals>.<genexpr>)last_hidden_stater  )	enumeraterH  tupler   )
r   r  r  	head_maskoutput_hidden_statesreturn_dictall_hidden_statesilayer_modulelayer_outputsr    r    r%   rr     s   



zMraEncoder.forward)NNFTr  r    r    r   r%   rC    s    	rC  c                       r-  )MraPredictionHeadTransformc                    sV   t    t|j|j| _t|jtrt	|j | _
n|j| _
tj|j|jd| _d S r  )r   r   r   r  r   r  r0  r1  r2  r   transform_act_fnr   r   r   r   r    r%   r     s   
z#MraPredictionHeadTransform.__init__r  r  c                 C   s"   |  |}| |}| |}|S ro   )r  rW  r   r5  r    r    r%   rr     s   


z"MraPredictionHeadTransform.forwardr   r    r    r   r%   rV    s    	rV  c                       s,   e Zd Z fddZdd Zdd Z  ZS )MraLMPredictionHeadc                    sL   t    t|| _tj|j|jdd| _t	t
|j| _| j| j_d S )NF)bias)r   r   rV  	transformr   r  r   r   decoder	ParameterrO   r   rY  r   r   r    r%   r     s
   

zMraLMPredictionHead.__init__c                 C   s   | j | j_ d S ro   )rY  r[  r   r    r    r%   _tie_weights  s   z MraLMPredictionHead._tie_weightsc                 C   r4  ro   )rZ  r[  r5  r    r    r%   rr     r6  zMraLMPredictionHead.forward)r   r   r   r   r^  rr   r   r    r    r   r%   rX    s    rX  c                       r-  )MraOnlyMLMHeadc                    s   t    t|| _d S ro   )r   r   rX  predictionsr   r   r    r%   r   '  s   
zMraOnlyMLMHead.__init__sequence_outputr  c                 C   s   |  |}|S ro   )r`  )r   ra  prediction_scoresr    r    r%   rr   +  s   
zMraOnlyMLMHead.forwardr   r    r    r   r%   r_  &  s    r_  c                   @   s.   e Zd ZU eed< dZdZdejfddZ	dS )MraPreTrainedModelr   r   Tmodulec                 C   s   | j j}t|tjr"|jjjd|d |jdur |jj	  dS dS t|tj
rC|jjjd|d |jdurA|jj|j 	  dS dS t|tjrX|jj	  |jjd dS t|tre|jj	  dS dS )zInitialize the weightsg        )r   stdNr  )r   initializer_ranger0  r   r  weightdatanormal_rY  zero_r   r   r   fill_rX  )r   rd  re  r    r    r%   _init_weights7  s"   


z MraPreTrainedModel._init_weightsN)
r   r   r   r   __annotations__base_model_prefixsupports_gradient_checkpointingr   Modulerl  r    r    r    r%   rc  0  s
   
 rc  c                       s   e Zd Z fddZdd Zdd Zdd Ze																dd
ee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee dee deeef fddZ  ZS )MraModelc                    s2   t  | || _t|| _t|| _|   d S ro   )r   r   r   r   r   rC  encoder	post_initr   r   r    r%   r   M  s
   

zMraModel.__init__c                 C   s   | j jS ro   r   r   r]  r    r    r%   get_input_embeddingsW  s   zMraModel.get_input_embeddingsc                 C   s   || j _d S ro   rt  )r   r   r    r    r%   set_input_embeddingsZ  s   zMraModel.set_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrr  rH  r<  r)  )r   heads_to_prunerH  r'  r    r    r%   _prune_heads]  s   zMraModel._prune_headsNr   r  r   r   rO  r   rP  rQ  r  c	                 C   s|  |d ur|n| j j}|d ur|n| j j}|d ur |d ur td|d ur/| || | }	n|d ur<| d d }	ntd|	\}
}|d urK|jn|j}|d u r[tj|
|f|d}|d u rt	| j
drz| j
jd d d |f }||
|}|}n	tj|	tj|d}| ||	}| || j j}| j
||||d}| j|||||d}|d	 }|s|f|d
d   S t||j|j|jdS )NzDYou cannot specify both input_ids and inputs_embeds at the same timer9   z5You have to specify either input_ids or inputs_embedsr   r   rK   )r   r   r   r   )r  rO  rP  rQ  r   r   )rL  r  
attentionscross_attentions)r   rP  use_return_dictr<   %warn_if_padding_and_no_attention_maskr;   rM   rO   r   r   r   r   r   r   rQ   get_extended_attention_maskget_head_maskrG  rr  r   r  rz  r{  )r   r   r  r   r   rO  r   rP  rQ  r   rU   r   rM   r   r   extended_attention_maskembedding_outputencoder_outputsra  r    r    r%   rr   e  sZ   
zMraModel.forward)NNNNNNNN)r   r   r   r   ru  rv  ry  r   r   rO   r!  boolr   rN  r   rr   r   r    r    r   r%   rq  K  sB    
	

rq  c                       s   e Zd ZddgZ fddZdd Zdd Ze																		dd
ee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee dee deeef fddZ  ZS )MraForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasc                    s,   t  | t|| _t|| _|   d S ro   )r   r   rq  r   r_  clsrs  r   r   r    r%   r     s   

zMraForMaskedLM.__init__c                 C   s
   | j jjS ro   )r  r`  r[  r]  r    r    r%   get_output_embeddings  s   
z$MraForMaskedLM.get_output_embeddingsc                 C   s   || j j_|j| j j_d S ro   )r  r`  r[  rY  )r   new_embeddingsr    r    r%   set_output_embeddings  s   
z$MraForMaskedLM.set_output_embeddingsNr   r  r   r   rO  r   labelsrP  rQ  r  c
              
   C   s   |	dur|	n| j j}	| j||||||||	d}
|
d }| |}d}|dur7t }||d| j j|d}|	sM|f|
dd  }|durK|f| S |S t|||
j|
j	dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        Nr  r   r   rO  r   rP  rQ  r   r9   r   losslogitsr  rz  )
r   r|  r   r  r   r  r   r   r  rz  )r   r   r  r   r   rO  r   r  rP  rQ  r  ra  rb  masked_lm_lossloss_fctr   r    r    r%   rr     s4   
zMraForMaskedLM.forward	NNNNNNNNN)r   r   r   _tied_weights_keysr   r  r  r   r   rO   r!  r  r   rN  r   rr   r   r    r    r   r%   r    sH    		

r  c                       s(   e Zd ZdZ fddZdd Z  ZS )MraClassificationHeadz-Head for sentence-level classification tasks.c                    sF   t    t|j|j| _t|j| _t|j|j	| _
|| _d S ro   )r   r   r   r  r   r  r   r   r   
num_labelsout_projr   r   r   r    r%   r     s
   

zMraClassificationHead.__init__c                 K   sR   |d d dd d f }|  |}| |}t| jj |}|  |}| |}|S )Nr   )r   r  r   r   r1  r  )r   featureskwargsxr    r    r%   rr     s   



zMraClassificationHead.forwardr   r    r    r   r%   r    s    r  z
    MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )custom_introc                          e Zd Z fddZe									ddeej deej deej deej deej d	eej d
eej dee dee de	e
ef fddZ  ZS )MraForSequenceClassificationc                    s4   t  | |j| _t|| _t|| _|   d S ro   )r   r   r  rq  r   r  
classifierrs  r   r   r    r%   r     s
   

z%MraForSequenceClassification.__init__Nr   r  r   r   rO  r   r  rP  rQ  r  c
              
   C   sf  |	dur|	n| j j}	| j||||||||	d}
|
d }| |}d}|dur| j jdu rP| jdkr6d| j _n| jdkrL|jtjksG|jtj	krLd| j _nd| j _| j jdkrnt
 }| jdkrh|| | }n+|||}n%| j jdkrt }||d| j|d}n| j jdkrt }|||}|	s|f|
dd  }|dur|f| S |S t|||
j|
jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   
regressionsingle_label_classificationmulti_label_classificationr9   r  )r   r|  r   r  problem_typer  rL   rO   rQ   rA   r   r  r   r  r   r   r  rz  )r   r   r  r   r   rO  r   r  rP  rQ  r  ra  r  r  r  r   r    r    r%   rr   !  sR   


"


z$MraForSequenceClassification.forwardr  )r   r   r   r   r   r   rO   r!  r  r   rN  r   rr   r   r    r    r   r%   r    sB    		

r  c                       r  )MraForMultipleChoicec                    sD   t  | t|| _t|j|j| _t|jd| _| 	  d S r9  )
r   r   rq  r   r   r  r   pre_classifierr  rs  r   r   r    r%   r   h  s
   
zMraForMultipleChoice.__init__Nr   r  r   r   rO  r   r  rP  rQ  r  c
              
   C   s  |	dur|	n| j j}	|dur|jd n|jd }
|dur%|d|dnd}|dur4|d|dnd}|durC|d|dnd}|durR|d|dnd}|dure|d|d|dnd}| j||||||||	d}|d }|dddf }| |}t |}| 	|}|d|
}d}|durt
 }|||}|	s|f|dd  }|dur|f| S |S t|||j|jdS )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r9   r6   r  r   r  )r   r|  rN   r  r;   r   r  r   ReLUr  r   r   r  rz  )r   r   r  r   r   rO  r   r  rP  rQ  num_choicesr  hidden_statepooled_outputr  reshaped_logitsr  r  r   r    r    r%   rr   r  sN   +


zMraForMultipleChoice.forwardr  )r   r   r   r   r   r   rO   r!  r  r   rN  r   rr   r   r    r    r   r%   r  f  sB    
	

r  c                       r  )MraForTokenClassificationc                    sJ   t  | |j| _t|| _t|j| _t	|j
|j| _|   d S ro   )r   r   r  rq  r   r   r   r   r   r  r   r  rs  r   r   r    r%   r     s   
z"MraForTokenClassification.__init__Nr   r  r   r   rO  r   r  rP  rQ  r  c
              
   C   s  |	dur|	n| j j}	| j||||||||	d}
|
d }| |}| |}d}|durdt }|durW|ddk}|d| j}t	||dt
|j|}|||}n||d| j|d}|	sz|f|
dd  }|durx|f| S |S t|||
j|
jdS )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   r9   r   r  )r   r|  r   r   r  r   r  r  rO   wheretensorignore_indextype_asr   r  rz  )r   r   r  r   r   rO  r   r  rP  rQ  r  ra  r  r  r  active_lossactive_logitsactive_labelsr   r    r    r%   rr     sD   

z!MraForTokenClassification.forwardr  )r   r   r   r   r   r   rO   r!  r  r   rN  r   rr   r   r    r    r   r%   r    sB    	

r  c                       s   e Zd Z fddZe										ddeej deej deej deej deej d	eej d
eej deej dee dee de	e
ef fddZ  ZS )MraForQuestionAnsweringc                    sB   t  | d|_|j| _t|| _t|j|j| _| 	  d S )Nr3   )
r   r   r  rq  r   r   r  r   
qa_outputsrs  r   r   r    r%   r     s   
z MraForQuestionAnswering.__init__Nr   r  r   r   rO  r   start_positionsend_positionsrP  rQ  r  c              
   C   s>  |
d ur|
n| j j}
| j|||||||	|
d}|d }| |}|jddd\}}|d}|d}d }|d ur}|d ur}t| dkrJ|d}t| dkrW|d}|d}|d|}|d|}t	|d}|||}|||}|| d }|
s||f|dd   }|d ur|f| S |S t
||||j|jdS )	Nr  r   r   r9   r7   )r  r3   )r  start_logits
end_logitsr  rz  )r   r|  r   r  splitr  r:   r;   clampr   r   r  rz  )r   r   r  r   r   rO  r   r  r  rP  rQ  r  ra  r  r  r  
total_lossignored_indexr  
start_lossend_lossr   r    r    r%   rr   &  sN   








zMraForQuestionAnswering.forward)
NNNNNNNNNN)r   r   r   r   r   r   rO   r!  r  r   rN  r   rr   r   r    r    r   r%   r    sH    	

r  )r  r  r  r  r  r8  rq  rc  r   )NN)r5   r   r   )Or   r   pathlibr   typingr   r   rO   r   torch.nnr   r   r   torch.utils.cpp_extensionr	   activationsr   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   r   configuration_mrar   
get_loggerr   r   r/   r1   rJ   rY   r\   rc   rl   autogradFunctionrm   r   r   r   r   r   rp  r   r   r  r#  r.  r7  r8  rC  rV  rX  r_  rc  rq  r  r  r  r  r  r  __all__r    r    r    r%   <module>   sz    



((
(-
s:d!%
gHOgIM