o
    wi                     @   s  d Z ddlZddlmZ ddlmZmZ ddlZddlZddlm	Z	 ddl
mZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( e&)e*Z+da,dd Z-dd Z.dSddZ/dSddZ0dSddZ1dd Z2G dd dej3j4Z5G d d! d!ej3j4Z6G d"d# d#Z7dTd$d%Z8d&d' Z9			dUd(d)Z:G d*d+ d+e	j;Z<G d,d- d-e	j;Z=G d.d/ d/e	j;Z>G d0d1 d1e	j;Z?G d2d3 d3e	j;Z@G d4d5 d5e	j;ZAG d6d7 d7eZBG d8d9 d9e	j;ZCG d:d; d;e	j;ZDG d<d= d=e	j;ZEG d>d? d?e	j;ZFe"G d@dA dAeZGe"G dBdC dCeGZHe"G dDdE dEeGZIG dFdG dGe	j;ZJe"dHdIG dJdK dKeGZKe"G dLdM dMeGZLe"G dNdO dOeGZMe"G dPdQ dQeGZNg dRZOdS )VzPyTorch MRA model.    N)Path)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)load   )ACT2FN)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringis_cuda_platformis_ninja_availableis_torch_cuda_availablelogging   )	MraConfigc                     sD   t t jjjd d   fdd} | g d}td|ddad S )	Nkernelsmrac                    s    fdd| D S )Nc                    s   g | ]} | qS  r    ).0file
src_folderr    a/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/mra/modeling_mra.py
<listcomp>5       z:load_cuda_kernels.<locals>.append_root.<locals>.<listcomp>r    )filesr#   r    r%   append_root4   s   z&load_cuda_kernels.<locals>.append_root)zcuda_kernel.cuzcuda_launch.cuztorch_extension.cppcuda_kernelT)verbose)r   __file__resolveparentr	   mra_cuda_kernel)r)   	src_filesr    r#   r%   load_cuda_kernels0   s   r1   c                 C   s   t |  dkrtdt | dkrtd| ddkr#td| ddkr.td| jd	d
jdd	}| }| }| }t	||||\}}|dd	dddddddf }||fS )z8
    Computes maximum values for softmax stability.
       z.sparse_qk_prod must be a 4-dimensional tensor.   'indices must be a 2-dimensional tensor.    z>The size of the second dimension of sparse_qk_prod must be 32.r
   z=The size of the third dimension of sparse_qk_prod must be 32.dimN)
lensize
ValueErrormaxvalues	transpose
contiguousintr/   	index_max)sparse_qk_prodindicesquery_num_blockkey_num_block
index_valsmax_valsmax_vals_scatterr    r    r%   
sparse_max<   s   $rJ   r5   c                 C   s   t |  dkrtdt | dkrtd| jd |jd kr&td| j\}}|| }tj|dtj|jd}| |||} | |dddf ||  ddf } | S )zN
    Converts attention mask to a sparse mask for high resolution logits.
    r3   z$mask must be a 2-dimensional tensor.r4   r   zBmask and indices must have the same size in the zero-th dimension.dtypedeviceN)	r:   r;   r<   shapetorcharangelongrM   reshape)maskrD   
block_size
batch_sizeseq_len	num_block	batch_idxr    r    r%   sparse_maskX   s   
&rY   c           	      C   s"  |   \}}}|  \}}}|| dkrtd|| dkr"td| ||| ||dd} |||| ||dd}t|   dkrJtdt|  dkrVtdt|  d	krbtd
|  ddkrmtd| ddkrxtd|  } | }| }| }t| || S )z7
    Performs Sampled Dense Matrix Multiplication.
    r   zTquery_size (size of first dimension of dense_query) must be divisible by block_size.Pkey_size (size of first dimension of dense_key) must be divisible by block_size.r9   r6   r2   z+dense_query must be a 4-dimensional tensor.)dense_key must be a 4-dimensional tensor.r3   r4   r
   r5   z.The third dimension of dense_query must be 32.z,The third dimension of dense_key must be 32.)	r;   r<   rR   r?   r:   r@   rA   r/   mm_to_sparse)	dense_query	dense_keyrD   rT   rU   
query_sizer8   _key_sizer    r    r%   r\   o   s.   r\   c           	      C   s  |  \}}}|| dkrtd|  d|krtd|  d|kr'td|||| ||dd}t|   d	krAtd
t|  d	krMtdt|  dkrYtd| ddkrdtd|  } | }| }| }t| |||}|dd||| |}|S )zP
    Performs matrix multiplication of a sparse matrix with a dense matrix.
    r   rZ   r3   zQThe size of the second dimension of sparse_query must be equal to the block_size.r
   zPThe size of the third dimension of sparse_query must be equal to the block_size.r9   r6   r2   ,sparse_query must be a 4-dimensional tensor.r[   r4   r5   z8The size of the third dimension of dense_key must be 32.)	r;   r<   rR   r?   r:   r@   rA   r/   sparse_dense_mm)	sparse_queryrD   r^   rE   rT   rU   ra   r8   dense_qk_prodr    r    r%   rc      s.   rc   c                 C   s    | | | t j| |dd  S )Nfloorrounding_mode)rO   divrQ   )rD   dim_1_blockdim_2_blockr    r    r%   transpose_indices   s    rl   c                   @   s2   e Zd Zedd Zedd Zed	ddZdS )
MraSampledDenseMatMulc                 C   &   t ||||}| ||| || _|S N)r\   save_for_backwardrT   )ctxr]   r^   rD   rT   rC   r    r    r%   forward      zMraSampledDenseMatMul.forwardc                 C   sj   | j \}}}| j}|d| }|d| }t|||}t|dd|||}	t||||}
|
|	d d fS Nr   r9   r6   )saved_tensorsrT   r;   rl   rc   r?   )rq   gradr]   r^   rD   rT   rE   rF   	indices_Tgrad_key
grad_queryr    r    r%   backward   s   zMraSampledDenseMatMul.backwardr5   c                 C      t | |||S ro   )rm   apply)r]   r^   rD   rT   r    r    r%   operator_call      z#MraSampledDenseMatMul.operator_callNr5   __name__
__module____qualname__staticmethodrr   rz   r}   r    r    r    r%   rm      s    


rm   c                   @   s0   e Zd Zedd Zedd Zedd ZdS )MraSparseDenseMatMulc                 C   rn   ro   )rc   rp   rE   )rq   rd   rD   r^   rE   rC   r    r    r%   rr      rs   zMraSparseDenseMatMul.forwardc           
      C   s`   | j \}}}| j}|d|d }t|||}t|dd|||}t|||}	|	d |d fS rt   )ru   rE   r;   rl   rc   r?   r\   )
rq   rv   rd   rD   r^   rE   rF   rw   rx   ry   r    r    r%   rz      s   zMraSparseDenseMatMul.backwardc                 C   r{   ro   )r   r|   )rd   rD   r^   rE   r    r    r%   r}      r~   z"MraSparseDenseMatMul.operator_callNr   r    r    r    r%   r      s    

	r   c                   @   s   e Zd Zedd ZdS )MraReduceSumc                 C   s  |   \}}}}t|   dkrtdt|  dkr td|   \}}}}|  \}}| jdd|| |} tj| dtj|jd}tj	||dd	 |d d d f |  || }	tj
|| |f| j| jd}
|
d|	| |||}|||| }|S )
Nr2   rb   r3   r4   r7   r   rK   rf   rg   )r;   r:   r<   sumrR   rO   rP   rQ   rM   ri   zerosrL   	index_add)rd   rD   rE   rF   rU   rW   rT   r`   rX   global_idxestempoutputr    r    r%   r}      s$   &
zMraReduceSum.operator_callN)r   r   r   r   r}   r    r    r    r%   r      s    r   c                 C   s  |   \}}}|| }d}	|durl||||jdd}
| ||||jdd|
dddddf d  }|||||jdd|
dddddf d  }|durk|||||jdd|
dddddf d  }	n5|tj||tj| jd }
| ||||jdd}|||||jdd}|dur|||||jdd}	t||	ddt
| }|jdddj}|dur|d	|
dddddf |
dddddf  d
k    }||
||	fS )z/
    Compute low resolution approximation.
    Nr9   r7   r6   ư>rK   T)r8   keepdims     @g      ?)r;   rR   r   rO   onesfloatrM   meanmatmulr?   mathsqrtr=   r>   )querykeyrT   rS   valuerU   rV   head_dimnum_block_per_row	value_hattoken_count	query_hatkey_hatlow_resolution_logitlow_resolution_logit_row_maxr    r    r%   get_low_resolution_logit  s6   :r   c                 C   sT  | j \}}}|dkr3|d }tj||| jd}	tjtj|	| d|d}
| |
dddddf d  } |dkrk| ddd|ddf d | ddd|ddf< | ddddd|f d | ddddd|f< tj| |d|ddd	d
}|j}|dkr|j	j
ddj	}| |ddddf k }||fS |dkrd}||fS t| d)zZ
    Compute the indices of the subset of components to be used in the approximation.
    r   r3   rM   )diagonalNg     @r9   TF)r8   largestsortedfullr7   sparsez# is not a valid approx_model value.)rN   rO   r   rM   triltriutopkrR   rD   r>   minr   r<   )r   
num_blocksapprox_modeinitial_prior_first_n_blocksinitial_prior_diagonal_n_blocksrU   total_blocks_per_rowr`   offset	temp_maskdiagonal_mask
top_k_valsrD   	thresholdhigh_resolution_maskr    r    r%   get_block_idxes7  s.   r   c	           $      C   s  t du rt|  S |  \}	}
}}|	|
 }|| dkr!td|| }| |||} ||||}||||}|dure| |dddddf  } ||dddddf  }||dddddf  }|dkrvt| ||||\}}}}n(|dkrt  t| |||\}}}}W d   n1 sw   Y  nt	dt  || }t
|||||\}}W d   n1 sw   Y  tj| |||dt| }t||||\}}|| }|dur|dd	t||dddddddf    }t|}t||||}t||||}|dkrt|| d|  |dddddf  }t||dddddddf d	d	|d	|||}|jd
ddddddf d	d	|||}|d	d	|||| } |durq| | } t| | dk  }!||!dddddf  }||! }t|  | dk  }"||"dddddf  }||" }|| |dddddf |dddddf  d  }#n|dkr||dddddf d  }#nt	d|dur|#|dddddf  }#|#|	|
||}#|#S )z0
    Use Mra to approximate self-attention.
    Nr   z4sequence length must be divisible by the block_size.r   r   z&approx_mode must be "full" or "sparse")rT   r   r   r9   r7   r   z-config.approx_mode must be "full" or "sparse")r/   rO   
zeros_likerequires_grad_r;   r<   rR   r   no_grad	Exceptionr   rm   r}   r   r   rJ   rY   expr   r   r   repeatr   r   )$r   r   r   rS   r   r   rT   r   r   rU   num_headrV   r   
meta_batchr   r   r   r   r   r`   low_resolution_logit_normalizedrD   r   high_resolution_logitrH   rI   high_resolution_attnhigh_resolution_attn_outhigh_resolution_normalizerlow_resolution_attnlow_resolution_attn_outlow_resolution_normalizerlog_correctionlow_resolution_corrhigh_resolution_corrcontext_layerr    r    r%   mra2_attention]  s   




.

"
.
.
 
r   c                       s*   e Zd ZdZ fddZdddZ  ZS )MraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t    tj|j|j|jd| _t|jd |j| _	t|j
|j| _tj|j|jd| _t|j| _| dt|jdd  t|dd| _| jdtj| j tj| jjd	d
d d S )N)padding_idxr3   epsposition_ids)r   r9   position_embedding_typeabsolutetoken_type_idsrK   F)
persistent)super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrO   rP   expandgetattrr   r   r   r;   rQ   rM   selfconfig	__class__r    r%   r     s   

zMraEmbeddings.__init__Nc                 C   s   |d ur	|  }n|  d d }|d }|d u r$| jd d d |f }|d u rNt| drC| jd d d |f }||d |}|}ntj|tj| jjd}|d u rW| 	|}| 
|}	||	 }
| jdkrn| |}|
|7 }
| |
}
| |
}
|
S )Nr9   r   r   r   rK   r   )r;   r   hasattrr   r   rO   r   rQ   rM   r   r   r   r   r   r   )r   	input_idsr   r   inputs_embedsinput_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr   
embeddingsr   r    r    r%   rr     s,   







zMraEmbeddings.forward)NNNNr   r   r   __doc__r   rr   __classcell__r    r    r   r%   r     s    r   c                       0   e Zd Zd fdd	Zdd ZdddZ  ZS )	MraSelfAttentionNc              
      sf  t    |j|j dkrt|dstd|j d|j dtd u}t rNt rNt	 rN|sNzt
  W n tyM } ztd|  W Y d }~nd }~ww |j| _t|j|j | _| j| j | _t|j| j| _t|j| j| _t|j| j| _t|j| _|d ur|n|j| _|jd |j | _t| jt|jd d | _|j| _|j| _|j | _ d S )	Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zGCould not load the custom kernel for multi-scale deformable attention: r5   r3   )!r   r   r   num_attention_headsr   r<   r/   r   r   r   r1   r   loggerwarningrA   attention_head_sizeall_head_sizer   Linearr   r   r   r   attention_probs_dropout_probr   r   r   block_per_rowrW   r   r   r   r   )r   r   r   kernel_loadeder   r    r%   r     s:   


zMraSelfAttention.__init__c                 C   s6   |  d d | j| jf }|j| }|ddddS )Nr9   r   r3   r   r
   )r;   r   r   viewpermute)r   layernew_layer_shaper    r    r%   transpose_for_scores.  s   
z%MraSelfAttention.transpose_for_scoresc              
   C   s  |  |}| | |}| | |}| |}| \}}}	}
d|d  }| d|d|| |	 }d}|
|k rt|||	||
 f}t	j
|t	j||jdgdd}t	j
|t	j||jdgdd}t	j
|t	j||jdgdd}t| | | | | j| j| j| jd}|
|k r|d d d d d d d |
f }||||	|
}|d	d
dd }| d d | jf }|j| }|f}|S )N      ?r   r   r5   r   r9   r7   )r   r   r   r   r3   r
   r6   )r   r  r   r   r;   squeezer   rR   rA   rO   catr   rM   r   r   rW   r   r   r   r  r@   r  r  )r   hidden_statesattention_maskmixed_query_layer	key_layervalue_layerquery_layerrU   	num_headsrV   r   gpu_warp_sizepad_sizer   new_context_layer_shapeoutputsr    r    r%   rr   3  s@   

  
zMraSelfAttention.forwardro   )r   r   r   r   r  rr   r   r    r    r   r%   r   
  s    #r   c                       8   e Zd Z fddZdejdejdejfddZ  ZS )MraSelfOutputc                    sB   t    t|j|j| _tj|j|jd| _t|j	| _
d S Nr   )r   r   r   r  r   denser   r   r   r   r   r   r   r    r%   r   h     
zMraSelfOutput.__init__r  input_tensorreturnc                 C   &   |  |}| |}| || }|S ro   r  r   r   r   r  r  r    r    r%   rr   n     

zMraSelfOutput.forwardr   r   r   r   rO   Tensorrr   r   r    r    r   r%   r  g      $r  c                       r   )	MraAttentionNc                    s.   t    t||d| _t|| _t | _d S )N)r   )r   r   r   r   r  r   setpruned_heads)r   r   r   r   r    r%   r   v  s   

zMraAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   r7   )r:   r   r   r   r   r*  r   r   r   r   r   r  r  union)r   headsindexr    r    r%   prune_heads|  s   zMraAttention.prune_headsc                 C   s2   |  ||}| |d |}|f|dd   }|S Nr   r   )r   r   )r   r  r  self_outputsattention_outputr  r    r    r%   rr     s   zMraAttention.forwardro   )r   r   r   r   r.  rr   r   r    r    r   r%   r(  u  s    r(  c                       2   e Zd Z fddZdejdejfddZ  ZS )MraIntermediatec                    sD   t    t|j|j| _t|jt	rt
|j | _d S |j| _d S ro   )r   r   r   r  r   intermediate_sizer  
isinstance
hidden_actstrr   intermediate_act_fnr   r   r    r%   r     s
   
zMraIntermediate.__init__r  r   c                 C      |  |}| |}|S ro   )r  r8  r   r  r    r    r%   rr        

zMraIntermediate.forwardr%  r    r    r   r%   r3    s    r3  c                       r  )	MraOutputc                    sB   t    t|j|j| _tj|j|jd| _t	|j
| _d S r  )r   r   r   r  r4  r   r  r   r   r   r   r   r   r   r    r%   r     r  zMraOutput.__init__r  r  r   c                 C   r!  ro   r"  r#  r    r    r%   rr     r$  zMraOutput.forwardr%  r    r    r   r%   r<    r'  r<  c                       s.   e Zd Z fddZdddZdd Z  ZS )	MraLayerc                    sB   t    |j| _d| _t|| _|j| _t|| _t	|| _
d S Nr   )r   r   chunk_size_feed_forwardseq_len_dimr(  	attentionadd_cross_attentionr3  intermediater<  r   r   r   r    r%   r     s   


zMraLayer.__init__Nc                 C   sB   |  ||}|d }|dd  }t| j| j| j|}|f| }|S r/  )rA  r   feed_forward_chunkr?  r@  )r   r  r  self_attention_outputsr1  r  layer_outputr    r    r%   rr     s   
zMraLayer.forwardc                 C   s   |  |}| ||}|S ro   )rC  r   )r   r1  intermediate_outputrF  r    r    r%   rD    s   
zMraLayer.feed_forward_chunkro   )r   r   r   r   rr   rD  r   r    r    r   r%   r=    s    
	r=  c                       s.   e Zd Z fddZ				dddZ  ZS )	
MraEncoderc                    s:   t     | _t fddt jD | _d| _d S )Nc                    s   g | ]}t  qS r    )r=  )r!   r`   r   r    r%   r&     r'   z'MraEncoder.__init__.<locals>.<listcomp>F)	r   r   r   r   
ModuleListrangenum_hidden_layersr	  gradient_checkpointingr   r   rI  r%   r     s   
 
zMraEncoder.__init__NFTc           
      C   st   |rdnd }t | jD ]\}}|r||f }|||}	|	d }q|r'||f }|s4tdd ||fD S t||dS )Nr    r   c                 s   s    | ]	}|d ur|V  qd S ro   r    )r!   vr    r    r%   	<genexpr>  s    z%MraEncoder.forward.<locals>.<genexpr>)last_hidden_stater  )	enumerater	  tupler   )
r   r  r  	head_maskoutput_hidden_statesreturn_dictall_hidden_statesilayer_modulelayer_outputsr    r    r%   rr     s   



zMraEncoder.forward)NNFT)r   r   r   r   rr   r   r    r    r   r%   rH    s    	rH  c                       r2  )MraPredictionHeadTransformc                    sV   t    t|j|j| _t|jtrt	|j | _
n|j| _
tj|j|jd| _d S r  )r   r   r   r  r   r  r5  r6  r7  r   transform_act_fnr   r   r   r   r    r%   r     s   
z#MraPredictionHeadTransform.__init__r  r   c                 C   s"   |  |}| |}| |}|S ro   )r  r[  r   r:  r    r    r%   rr      s   


z"MraPredictionHeadTransform.forwardr%  r    r    r   r%   rZ    s    	rZ  c                       s,   e Zd Z fddZdd Zdd Z  ZS )MraLMPredictionHeadc                    sL   t    t|| _tj|j|jdd| _t	t
|j| _| j| j_d S )NF)bias)r   r   rZ  	transformr   r  r   r   decoder	ParameterrO   r   r]  r   r   r    r%   r   	  s
   

zMraLMPredictionHead.__init__c                 C   s   | j | j_ d S ro   )r]  r_  r   r    r    r%   _tie_weights  s   z MraLMPredictionHead._tie_weightsc                 C   r9  ro   )r^  r_  r:  r    r    r%   rr     r;  zMraLMPredictionHead.forward)r   r   r   r   rb  rr   r   r    r    r   r%   r\    s    r\  c                       r2  )MraOnlyMLMHeadc                    s   t    t|| _d S ro   )r   r   r\  predictionsr   r   r    r%   r   !  s   
zMraOnlyMLMHead.__init__sequence_outputr   c                 C   s   |  |}|S ro   )rd  )r   re  prediction_scoresr    r    r%   rr   %  s   
zMraOnlyMLMHead.forwardr%  r    r    r   r%   rc     s    rc  c                   @   s    e Zd ZeZdZdZdd ZdS )MraPreTrainedModelr   Tc                 C   s   t |tjr |jjjd| jjd |jdur|jj	  dS dS t |tj
rC|jjjd| jjd |jdurA|jj|j 	  dS dS t |tjrX|jj	  |jjd dS dS )zInitialize the weightsg        )r   stdNr  )r5  r   r  weightdatanormal_r   initializer_ranger]  zero_r   r   r   fill_)r   moduler    r    r%   _init_weights1  s   

z MraPreTrainedModel._init_weightsN)r   r   r   r   config_classbase_model_prefixsupports_gradient_checkpointingrp  r    r    r    r%   rg  *  s
    rg  c                       s   e Zd Z fddZdd Zdd Zdd Ze																dd
ee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee dee deeef fddZ  ZS )MraModelc                    s2   t  | || _t|| _t|| _|   d S ro   )r   r   r   r   r   rH  encoder	post_initr   r   r    r%   r   D  s
   

zMraModel.__init__c                 C   s   | j jS ro   r   r   ra  r    r    r%   get_input_embeddingsN  s   zMraModel.get_input_embeddingsc                 C   s   || j _d S ro   rw  )r   r   r    r    r%   set_input_embeddingsQ  s   zMraModel.set_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsru  r	  rA  r.  )r   heads_to_pruner	  r,  r    r    r%   _prune_headsT  s   zMraModel._prune_headsNr   r  r   r   rS  r   rT  rU  r   c	                 C   s|  |d ur|n| j j}|d ur|n| j j}|d ur |d ur td|d ur/| || | }	n|d ur<| d d }	ntd|	\}
}|d urK|jn|j}|d u r[tj|
|f|d}|d u rt	| j
drz| j
jd d d |f }||
|}|}n	tj|	tj|d}| ||	}| || j j}| j
||||d}| j|||||d}|d	 }|s|f|d
d   S t||j|j|jdS )NzDYou cannot specify both input_ids and inputs_embeds at the same timer9   z5You have to specify either input_ids or inputs_embedsr   r   rK   )r   r   r   r   )r  rS  rT  rU  r   r   )rP  r  
attentionscross_attentions)r   rT  use_return_dictr<   %warn_if_padding_and_no_attention_maskr;   rM   rO   r   r   r   r   r   r   rQ   get_extended_attention_maskget_head_maskrL  ru  r   r  r}  r~  )r   r   r  r   r   rS  r   rT  rU  r   rU   r   rM   r   r   extended_attention_maskembedding_outputencoder_outputsre  r    r    r%   rr   \  sZ   
zMraModel.forward)NNNNNNNN)r   r   r   r   rx  ry  r|  r   r   rO   r&  boolr   rR  r   rr   r   r    r    r   r%   rt  B  sB    
	

rt  c                       s   e Zd ZddgZ fddZdd Zdd Ze																		dd
ee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee	j
 dee dee deeef fddZ  ZS )MraForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasc                    s,   t  | t|| _t|| _|   d S ro   )r   r   rt  r   rc  clsrv  r   r   r    r%   r     s   

zMraForMaskedLM.__init__c                 C   s
   | j jjS ro   )r  rd  r_  ra  r    r    r%   get_output_embeddings  s   
z$MraForMaskedLM.get_output_embeddingsc                 C   s   || j j_|j| j j_d S ro   )r  rd  r_  r]  )r   new_embeddingsr    r    r%   set_output_embeddings  s   
z$MraForMaskedLM.set_output_embeddingsNr   r  r   r   rS  r   labelsrT  rU  r   c
              
   C   s   |	dur|	n| j j}	| j||||||||	d}
|
d }| |}d}|dur7t }||d| j j|d}|	sM|f|
dd  }|durK|f| S |S t|||
j|
j	dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        Nr  r   r   rS  r   rT  rU  r   r9   r   losslogitsr  r}  )
r   r  r   r  r   r  r   r   r  r}  )r   r   r  r   r   rS  r   r  rT  rU  r  re  rf  masked_lm_lossloss_fctr   r    r    r%   rr     s4   
zMraForMaskedLM.forward	NNNNNNNNN)r   r   r   _tied_weights_keysr   r  r  r   r   rO   r&  r  r   rR  r   rr   r   r    r    r   r%   r    sH    		

r  c                       s(   e Zd ZdZ fddZdd Z  ZS )MraClassificationHeadz-Head for sentence-level classification tasks.c                    sF   t    t|j|j| _t|j| _t|j|j	| _
|| _d S ro   )r   r   r   r  r   r  r   r   r   
num_labelsout_projr   r   r   r    r%   r     s
   

zMraClassificationHead.__init__c                 K   sR   |d d dd d f }|  |}| |}t| jj |}|  |}| |}|S )Nr   )r   r  r   r   r6  r  )r   featureskwargsxr    r    r%   rr     s   



zMraClassificationHead.forwardr   r    r    r   r%   r    s    r  z
    MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )custom_introc                          e Zd Z fddZe									ddeej deej deej deej deej d	eej d
eej dee dee de	e
ef fddZ  ZS )MraForSequenceClassificationc                    s4   t  | |j| _t|| _t|| _|   d S ro   )r   r   r  rt  r   r  
classifierrv  r   r   r    r%   r     s
   

z%MraForSequenceClassification.__init__Nr   r  r   r   rS  r   r  rT  rU  r   c
              
   C   sf  |	dur|	n| j j}	| j||||||||	d}
|
d }| |}d}|dur| j jdu rP| jdkr6d| j _n| jdkrL|jtjksG|jtj	krLd| j _nd| j _| j jdkrnt
 }| jdkrh|| | }n+|||}n%| j jdkrt }||d| j|d}n| j jdkrt }|||}|	s|f|
dd  }|dur|f| S |S t|||
j|
jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   
regressionsingle_label_classificationmulti_label_classificationr9   r  )r   r  r   r  problem_typer  rL   rO   rQ   rA   r   r  r   r  r   r   r  r}  )r   r   r  r   r   rS  r   r  rT  rU  r  re  r  r  r  r   r    r    r%   rr     sR   


"


z$MraForSequenceClassification.forwardr  )r   r   r   r   r   r   rO   r&  r  r   rR  r   rr   r   r    r    r   r%   r    sB    		

r  c                       r  )MraForMultipleChoicec                    sD   t  | t|| _t|j|j| _t|jd| _| 	  d S r>  )
r   r   rt  r   r   r  r   pre_classifierr  rv  r   r   r    r%   r   _  s
   
zMraForMultipleChoice.__init__Nr   r  r   r   rS  r   r  rT  rU  r   c
              
   C   s  |	dur|	n| j j}	|dur|jd n|jd }
|dur%|d|dnd}|dur4|d|dnd}|durC|d|dnd}|durR|d|dnd}|dure|d|d|dnd}| j||||||||	d}|d }|dddf }| |}t |}| 	|}|d|
}d}|durt
 }|||}|	s|f|dd  }|dur|f| S |S t|||j|jdS )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r9   r6   r  r   r  )r   r  rN   r  r;   r   r  r   ReLUr  r   r   r  r}  )r   r   r  r   r   rS  r   r  rT  rU  num_choicesr  hidden_statepooled_outputr  reshaped_logitsr  r  r   r    r    r%   rr   i  sN   +


zMraForMultipleChoice.forwardr  )r   r   r   r   r   r   rO   r&  r  r   rR  r   rr   r   r    r    r   r%   r  ]  sB    
	

r  c                       r  )MraForTokenClassificationc                    sJ   t  | |j| _t|| _t|j| _t	|j
|j| _|   d S ro   )r   r   r  rt  r   r   r   r   r   r  r   r  rv  r   r   r    r%   r     s   
z"MraForTokenClassification.__init__Nr   r  r   r   rS  r   r  rT  rU  r   c
              
   C   s  |	dur|	n| j j}	| j||||||||	d}
|
d }| |}| |}d}|durdt }|durW|ddk}|d| j}t	||dt
|j|}|||}n||d| j|d}|	sz|f|
dd  }|durx|f| S |S t|||
j|
jdS )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   r9   r   r  )r   r  r   r   r  r   r  r  rO   wheretensorignore_indextype_asr   r  r}  )r   r   r  r   r   rS  r   r  rT  rU  r  re  r  r  r  active_lossactive_logitsactive_labelsr   r    r    r%   rr     sD   

z!MraForTokenClassification.forwardr  )r   r   r   r   r   r   rO   r&  r  r   rR  r   rr   r   r    r    r   r%   r    sB    	

r  c                       s   e Zd Z fddZe										ddeej deej deej deej deej d	eej d
eej deej dee dee de	e
ef fddZ  ZS )MraForQuestionAnsweringc                    sB   t  | d|_|j| _t|| _t|j|j| _| 	  d S )Nr3   )
r   r   r  rt  r   r   r  r   
qa_outputsrv  r   r   r    r%   r     s   
z MraForQuestionAnswering.__init__Nr   r  r   r   rS  r   start_positionsend_positionsrT  rU  r   c              
   C   s>  |
d ur|
n| j j}
| j|||||||	|
d}|d }| |}|jddd\}}|d}|d}d }|d ur}|d ur}t| dkrJ|d}t| dkrW|d}|d}|d|}|d|}t	|d}|||}|||}|| d }|
s||f|dd   }|d ur|f| S |S t
||||j|jdS )	Nr  r   r   r9   r7   )r  r3   )r  start_logits
end_logitsr  r}  )r   r  r   r  splitr  r:   r;   clampr   r   r  r}  )r   r   r  r   r   rS  r   r  r  rT  rU  r  re  r  r  r  
total_lossignored_indexr  
start_lossend_lossr   r    r    r%   rr     sN   








zMraForQuestionAnswering.forward)
NNNNNNNNNN)r   r   r   r   r   r   rO   r&  r  r   rR  r   rr   r   r    r    r   r%   r    sH    	

r  )r  r  r  r  r  r=  rt  rg  r   )NN)r5   r   r   )Pr   r   pathlibr   typingr   r   rO   torch.utils.checkpointr   torch.nnr   r   r   torch.utils.cpp_extensionr	   activationsr   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   r   configuration_mrar   
get_loggerr   r   r/   r1   rJ   rY   r\   rc   rl   autogradFunctionrm   r   r   r   r   r   Moduler   r   r  r(  r3  r<  r=  rH  rZ  r\  rc  rg  rt  r  r  r  r  r  r  __all__r    r    r    r%   <module>   s|    



((
(-
s:]!%
gHOgIM