o
    eih                     @   s  d Z ddlmZ ddlZddlmZ ddlmZmZmZ ddl	m
Z ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4 e-5e6Z7G dd dej8Z9		dPdej8dej:dej:dej:dej:dB de;dB de;d e'e+ fd!d"Z<G d#d$ d$ej8Z=G d%d& d&ej8Z>G d'd( d(ej8Z?G d)d* d*ej8Z@G d+d, d,ej8ZAG d-d. d.ej8ZBG d/d0 d0ej8ZCG d1d2 d2eZDG d3d4 d4ej8ZEG d5d6 d6ej8ZFe,G d7d8 d8e%ZGe,d9d:G d;d< d<eGZHe,d=d:G d>d? d?eGeZIe,G d@dA dAeGZJG dBdC dCej8ZKe,dDd:G dEdF dFeGZLe,G dGdH dHeGZMe,G dIdJ dJeGZNG dKdL dLej8ZOe,G dMdN dNeGZPg dOZQdS )QzPyTorch X-MOD model.    )CallableN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FNgelu)CacheDynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)TransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs   )
XmodConfigc                       s   e Zd ZdZ fddZ					ddejdB dejdB dejdB d	ejdB d
edej	fddZ
edd ZedddZ  ZS )XmodEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t    tj|j|j|jd| _t|j|j| _	tj
|j|jd| _
t|j| _| jdt|jddd | jdtj| j tjddd |j| _tj|j|j| jd| _d S )	N)padding_idxepsposition_idsr$   F)
persistenttoken_type_ids)dtype)super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangemax_position_embeddingsexpandzerosr*   sizelongr'   position_embeddingsselfconfig	__class__ d/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/xmod/modeling_xmod.pyr1   6   s   
zXmodEmbeddings.__init__Nr   	input_idsr.   r*   inputs_embedspast_key_values_lengthreturnc                 C   s  |d u r|d ur|  || j|}n| || j}|d ur!| }n| d d }|\}}|d u rZt| drO| j|jd d}	tj	|	d|d}	|	||}ntj
|tj| jjd}|d u rc| |}| |}
||
 }| |}|| }| |}| |}|S )Nr,   r.   r   r$   )dimindexr/   device)"create_position_ids_from_input_idsr'   &create_position_ids_from_inputs_embedsrD   hasattrr.   rB   shaper?   gatherrC   rE   r*   rU   r6   r8   rF   r9   r=   )rH   rN   r.   r*   rO   rP   input_shape
batch_size
seq_lengthbuffered_token_type_idsr8   
embeddingsrF   rL   rL   rM   forwardJ   s2   






zXmodEmbeddings.forwardc                 C   sJ   |   dd }|d }tj|d || d tj| jd}|d|S )z
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr,   r$   rT   r   )rD   r?   r@   rE   rU   	unsqueezerB   )rO   r'   r[   sequence_lengthr*   rL   rL   rM   rW   z   s   
z5XmodEmbeddings.create_position_ids_from_inputs_embedsc                 C   s6   |  | }tj|dd|| | }| | S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r$   rR   )neintr?   cumsumtype_asrE   )rN   r'   rP   maskincremental_indicesrL   rL   rM   rV      s   z1XmodEmbeddings.create_position_ids_from_input_ids)NNNNr   )r   )__name__
__module____qualname____doc__r1   r?   
LongTensorFloatTensorre   Tensorr`   staticmethodrW   rV   __classcell__rL   rL   rJ   rM   r&   3   s2    
0
r&           modulequerykeyvalueattention_maskscalingr=   kwargsc           
      K   s   |d u r| dd }t||dd| }|d ur|| }tjj|dd}tjj||| jd}t||}	|	dd	 }	|	|fS )Nr,            r   rc   )ptrainingr$   )
rD   r?   matmul	transposer   
functionalsoftmaxr=   r~   
contiguous)
rt   ru   rv   rw   rx   ry   r=   rz   attn_weightsattn_outputrL   rL   rM   eager_attention_forward   s   
r   c                       sd   e Zd Zd fdd	Z			ddejdejdB dedB dejdB d	ee	 d
e
ej fddZ  ZS )XmodSelfAttentionFNc                    s   t    |j|j dkrt|dstd|j d|j d|| _|j| _t|j|j | _| j| j | _	| jd | _
t|j| j	| _t|j| j	| _t|j| j	| _t|j| _|j| _|| _|| _d S Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r{   )r0   r1   r4   num_attention_headsrX   
ValueErrorrI   re   attention_head_sizeall_head_sizery   r   Linearru   rv   rw   r;   attention_probs_dropout_probr=   
is_decoder	is_causal	layer_idxrH   rI   r   r   rJ   rL   rM   r1      s&   


zXmodSelfAttention.__init__hidden_statesrx   past_key_valuescache_positionrz   rQ   c                 K   s  |j d d }g |d| jR }| |j| dd}| |j| dd}	| |j| dd}
|d urP|}t|trC|j	}|
|	|
| jd|i\}	}
t| jjt}|| ||	|
|f| jsddn| jj| jd|\}}|jg |dR   }||fS )Nr,   r$   r|   r   rs   r=   ry   )rY   r   ru   viewr   rv   rw   
isinstancer   self_attention_cacheupdater   r   get_interfacerI   _attn_implementationr   r~   r=   r}   ry   reshaper   )rH   r   rx   r   r   rz   r[   hidden_shapequery_layer	key_layervalue_layercurrent_past_key_valuesattention_interfacer   r   rL   rL   rM   r`      s@   


zXmodSelfAttention.forwardFNNNN)rj   rk   rl   r1   r?   rp   ro   r   r   r   tupler`   rr   rL   rL   rJ   rM   r      s$    r   c                       sd   e Zd Zd fdd	Z			ddejdejdB dejdB dedB d	ee	 d
e
ej fddZ  ZS )XmodCrossAttentionFNc                    s   t    |j|j dkrt|dstd|j d|j d|| _|j| _t|j|j | _| j| j | _	| jd | _
t|j| j	| _t|j| j	| _t|j| j	| _t|j| _|| _|| _d S r   )r0   r1   r4   r   rX   r   rI   re   r   r   ry   r   r   ru   rv   rw   r;   r   r=   r   r   r   rJ   rL   rM   r1     s$   


zXmodCrossAttention.__init__r   encoder_hidden_statesrx   r   rz   rQ   c                 K   sN  |j d d \}}|j d }||d| jf}	||d| jf}
| |j|	 dd}|d ur3|j| jnd}|d urL|rL|jj	| j j
}|jj	| j j}n-| |j|
 dd}| |j|
 dd}|d ury|j||| j\}}d|j| j< t| jjt}|| ||||f| jsdn| jj| jd|\}}|||d }||fS )Nr,   r$   r|   FTrs   r   )rY   r   ru   r   r   
is_updatedgetr   cross_attention_cachelayerskeysvaluesrv   rw   r   r   r   rI   r   r   r~   r=   r}   ry   r   r   )rH   r   r   rx   r   rz   bsztgt_lensrc_lenq_input_shapekv_input_shaper   r   r   r   r   r   r   rL   rL   rM   r`     sB   	


zXmodCrossAttention.forwardr   r   )rj   rk   rl   r1   r?   rp   ro   r   r   r   r   r`   rr   rL   rL   rJ   rM   r     s$    r   c                       s8   e Zd Z fddZdejdejdejfddZ  ZS )XmodSelfOutputc                    sB   t    t|j|j| _tj|j|jd| _t|j	| _
d S Nr(   )r0   r1   r   r   r4   denser9   r:   r;   r<   r=   rG   rJ   rL   rM   r1   T  s   
zXmodSelfOutput.__init__r   input_tensorrQ   c                 C   s    |  |}| |}|| }|S N)r   r=   )rH   r   r   rL   rL   rM   r`   Z  s   

zXmodSelfOutput.forwardrj   rk   rl   r1   r?   rp   r`   rr   rL   rL   rJ   rM   r   R  s    $r   c                       s   e Zd Zd fdd	Z					ddejdejdB dejdB dejdB d	eeej  dB d
ejdB dee	 deej fddZ
  ZS )XmodAttentionFNc                    sB   t    || _|rtnt}||||d| _t|| _|j| _d S )Nr   r   )	r0   r1   is_cross_attentionr   r   rH   r   outputpre_norm)rH   rI   r   r   r   attention_classrJ   rL   rM   r1   b  s   

zXmodAttention.__init__r   rx   r   encoder_attention_maskr   r   rz   rQ   c                 K   sj   |}| j r| j|}| js|n|}| j|f||||d|\}	}
| |	|}	| j s1| j|	}	|	|
fS )N)r   rx   r   r   )r   r   r9   r   rH   )rH   r   rx   r   r   r   r   rz   residualattention_outputr   rL   rL   rM   r`   k  s$   

zXmodAttention.forward)FNFNNNNN)rj   rk   rl   r1   r?   rp   ro   r   r   r   r`   rr   rL   rL   rJ   rM   r   a  s0    	r   c                       2   e Zd Z fddZdejdejfddZ  ZS )XmodIntermediatec                    sD   t    t|j|j| _t|jt	rt
|j | _d S |j| _d S r   )r0   r1   r   r   r4   intermediate_sizer   r   
hidden_actstrr	   intermediate_act_fnrG   rJ   rL   rM   r1     s
   
zXmodIntermediate.__init__r   rQ   c                 C   s   |  |}| |}|S r   )r   r   rH   r   rL   rL   rM   r`     s   

zXmodIntermediate.forwardr   rL   rL   rJ   rM   r     s    r   c                       r   )XmodAdapterc                    sd   t    |j|j | _t|j| j| _t| j|j| _t	|j
tr,t|j
 | _d S |j
| _d S r   )r0   r1   r4   adapter_reduction_factorbottleneck_sizer   r   dense1dense2r   r   r   r	   adapter_act_fnrG   rJ   rL   rM   r1     s   
zXmodAdapter.__init__r   rQ   c                 C   s"   |  |}| |}| |}|S r   )r   r   r   r   rL   rL   rM   r`     s   


zXmodAdapter.forwardr   rL   rL   rJ   rM   r     s    
r   c                       sT   e Zd Z fddZdejdejdejdejfddZdejdejfd	d
Z  ZS )
XmodOutputc                    s   t    t|j|j| _tj|j|jd| _|j	| _	t
|j| _|jr1tj|j|jd| _nd | _|j| _ti | _|jD ]}t|| jt|< qAd S r   )r0   r1   r   r   r   r4   r   r9   r:   ln_before_adapterr;   r<   r=   adapter_layer_normadapter_reuse_layer_norm
ModuleDictadapter_modules	languagesr   r   )rH   rI   languagerJ   rL   rM   r1     s   

zXmodOutput.__init__r   r   lang_idsrQ   c                 C   s,   |  |}| |}|| }| ||}|S r   )r   r=   lang_adapter)rH   r   r   r   rL   rL   rM   r`     s
   

zXmodOutput.forwardc           
      C   s   | j s|}| jd ur| |}n| jr| |}| j r|}t|}t| j D ]\}}||k}|| }| j| |}	|	||< q)| 	|}||7 }|S r   )
r   r   r   r9   r?   
zeros_like	enumerater   r   r=   )
rH   r   r   r   new_hidden_statesadapter_idxlang_key	lang_masklang_hidden_statesadapted_lang_hidden_statesrL   rL   rM   r     s"   




zXmodOutput.lang_adapter)	rj   rk   rl   r1   r?   rp   r`   r   rr   rL   rL   rJ   rM   r     s    "r   c                       s   e Zd Zd fdd	Z					ddejdejdejdB dejdB dejdB d	eeej  dB d
ejdB dee	 deej fddZ
dd Z  ZS )	XmodLayerNc                    s   t    |j| _d| _t||j|d| _|j| _|j| _| jr3| js*t|  dt|d|dd| _	t
|| _t|| _|j| _d S )Nr$   r   z> should be used as a decoder model if cross attention is addedFT)r   r   r   )r0   r1   chunk_size_feed_forwardseq_len_dimr   r   	attentionadd_cross_attentionr   crossattentionr   intermediater   r   r   )rH   rI   r   rJ   rL   rM   r1     s$   


zXmodLayer.__init__r   r   rx   r   r   r   r   rz   rQ   c                 K   s   | j ||f||d|\}	}
|	}| jr7|d ur7t| ds%td|  d| j|d ||fd|i|\}}
|}|}| jrB| j|}t| j	| j
| j|}| |||}| js\| j|}|S )N)r   r   r   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r   )r   r   rX   r   r   r   r   r9   r   feed_forward_chunkr   r   )rH   r   r   rx   r   r   r   r   rz   self_attention_output_r   cross_attention_outputr   intermediate_outputlayer_outputrL   rL   rM   r`     sN   



zXmodLayer.forwardc                 C   s
   |  |S r   )r   )rH   r   rL   rL   rM   r   $  s   
zXmodLayer.feed_forward_chunkr   r   )rj   rk   rl   r1   r?   rp   ro   r   r   r   r`   r   rr   rL   rL   rJ   rM   r     s6    	

4r   c                       s   e Zd Z fddZ						ddejdejdejdB dejdB dejdB d	eeej  dB d
edB dejdB de	e
 deej eB fddZ  ZS )XmodEncoderc                    sZ   t     | _t fddt jD | _ j| _	| j	r+tj
 j jd| _
d S d S )Nc                    s   g | ]}t  |d qS ))r   )r   ).0irI   rL   rM   
<listcomp>,  s    z(XmodEncoder.__init__.<locals>.<listcomp>r(   )r0   r1   rI   r   
ModuleListrangenum_hidden_layerslayerr   is_pre_normr9   r4   r:   rG   rJ   r   rM   r1   )  s   
 zXmodEncoder.__init__Nr   r   rx   r   r   r   	use_cacher   rz   rQ   c	              	   K   sX   t | jD ]\}
}||||||||fi |	}q| jr | |}t||r(|dS d dS )N)last_hidden_stater   )r   r   r   r9   r   )rH   r   r   rx   r   r   r   r   r   rz   r   layer_modulerL   rL   rM   r`   1  s(   
zXmodEncoder.forward)NNNNNN)rj   rk   rl   r1   r?   rp   ro   r   boolr   r   r   r`   rr   rL   rL   rJ   rM   r   (  s:    	
r   c                       r   )
XmodPoolerc                    s*   t    t|j|j| _t | _d S r   )r0   r1   r   r   r4   r   Tanh
activationrG   rJ   rL   rM   r1   T  s   
zXmodPooler.__init__r   rQ   c                 C   s(   |d d df }|  |}| |}|S Nr   )r   r  )rH   r   first_token_tensorpooled_outputrL   rL   rM   r`   Y  s   

zXmodPooler.forwardr   rL   rL   rJ   rM   r   S  s    r   c                       sj   e Zd ZeZdZdZg dZdZdZ	dZ
dZeeedZe  fddZdefdd	Zd
d Z  ZS )XmodPreTrainedModelrobertaT)r&   r   r   )r   
attentionscross_attentionsc                    sf   t  | t|trt|j dS t|tr1t|j	t
|j	jd d t|j dS dS )zInitialize the weightsr,   r+   N)r0   _init_weightsr   
XmodLMHeadinitzeros_biasr&   copy_r*   r?   r@   rY   rB   r.   )rH   rt   rJ   rL   rM   r
  r  s   

"z!XmodPreTrainedModel._init_weightsr   c                 C   s8   || j jvrt|  d| dt| j j || j _dS )z
        Set the default language code for the model. This is used when the language is not specified in the input.

        Args:
            language (`str`): The language code, such as `"en_XX"` or `"de_DE"`.
        z does not have an adapter for z. Supported languages: N)rI   r   r   listdefault_language)rH   r   rL   rL   rM   set_default_language|  s
   z(XmodPreTrainedModel.set_default_languagec                 C   s|   t d | jj D ]}d|_qt d | jjjD ] }|jj	dur/|jj	 D ]}d|_q)|jj
 D ]}d|_q5qdS )z
        Freeze the embeddings and language adapters of the model. Usually, this is applied before the model is
        fine-tuned on a downstream task.
        zFreezing embeddingsFzFreezing adaptersN)loggerinfor  r_   
parametersrequires_gradencoderr   r   r   r   )rH   	parameterr   rL   rL   rM   'freeze_embeddings_and_language_adapters  s   

z;XmodPreTrainedModel.freeze_embeddings_and_language_adapters)rj   rk   rl   r%   config_classbase_model_prefixsupports_gradient_checkpointingno_split_modules_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr   r   r   _can_record_outputsr?   no_gradr
  r   r  r  rr   rL   rL   rJ   rM   r  b  s"    	r  a0  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    )custom_introc                       s   e Zd Zd fdd	Zdd Zdd Zeee											dd	e	j
dB d
e	jdB de	j
dB de	j
dB de	j
dB de	j
dB de	j
dB de	j
dB dee	j dB dedB de	j
dB dee dee	j
 eB fddZdd Z  ZS )	XmodModelTc                    sJ   t  | || _d| _t|| _t|| _|rt|nd| _	| 
  dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)r0   r1   rI   gradient_checkpointingr&   r_   r   r  r   pooler	post_init)rH   rI   add_pooling_layerrJ   rL   rM   r1     s   

zXmodModel.__init__c                 C      | j jS r   r_   r6   rH   rL   rL   rM   get_input_embeddings     zXmodModel.get_input_embeddingsc                 C      || j _d S r   r+  )rH   rw   rL   rL   rM   set_input_embeddings     zXmodModel.set_input_embeddingsNrN   r   rx   r.   r*   rO   r   r   r   r   r   rz   rQ   c                 K   s  | j jr|
dur
|
n| j j}
nd}
|
r2|	du r2|dus| j jr,tt| j dt| j dnt| j d}	|du |duA r>td|durI|j}|j}n
|j}|jdd }|\}}|dur^|jn|j}|	duri|		 nd}|du ryt
j||| |d}|du r| j jdu rtdt| jjd jj }|| j j}|t
j||d }| j|||||d	}| j||||||	d
\}}| j|f|||||	|
||d|}|d }| jdur| |nd}t|||jdS )  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        NFr   z:You must specify exactly one of input_ids or inputs_embedsr,   r   )rU   zPInput language unknown. Please call `XmodPreTrainedModel.set_default_language()`)rN   r*   r.   rO   rP   )rx   r   embedding_outputr   r   r   )r   rx   r   r   r   r   r   r*   )r   pooler_outputr   )rI   r   r   is_encoder_decoderr   r   r   rU   rY   get_seq_lengthr?   r@   r  r  r  r   r   r   r   rS   onesr_   _create_attention_masksr'  r   r   )rH   rN   r   rx   r.   r*   rO   r   r   r   r   r   rz   rU   r[   r\   r]   rP   adapter_languagesdefault_lang_idr3  encoder_outputssequence_outputr  rL   rL   rM   r`     s|   

	
zXmodModel.forwardc                 C   sP   | j jrt| j ||||d}nt| j ||d}|d ur$t| j |||d}||fS )N)rI   rO   rx   r   r   )rI   rO   rx   )rI   rO   rx   r   )rI   r   r   r   )rH   rx   r   r3  r   r   r   rL   rL   rM   r8  $  s*   	z!XmodModel._create_attention_masks)T)NNNNNNNNNNN)rj   rk   rl   r1   r-  r0  r"   r#   r   r?   rp   rn   r  ro   r   r   r   r   r   r`   r8  rr   rL   rL   rJ   rM   r%    s\    	
_r%  zQ
    X-MOD Model with a `language modeling` head on top for CLM fine-tuning.
    c                "       s  e Zd ZdddZ fddZdd Zdd	 Zee	
	
	
	
	
	
	
	
	
	
	
	
	dde	j
d
B de	j
d
B de	jd
B de	j
d
B de	j
d
B de	jd
B de	jd
B de	jd
B de	j
d
B deee	j  d
B ded
B de	jd
B dee	jB dee dee	j eB fddZ  ZS )XmodForCausalLM)roberta.embeddings.word_embeddings.weightlm_head.biaszlm_head.decoder.weightzlm_head.decoder.biasc                    s@   t  | |jstd t|dd| _t|| _| 	  d S )NzLIf you want to use `XmodLMHeadModel` as a standalone, add `is_decoder=True.`Fr)  
r0   r1   r   r  warningr%  r  r  lm_headr(  rG   rJ   rL   rM   r1   S  s   

zXmodForCausalLM.__init__c                 C   r*  r   rD  decoderr,  rL   rL   rM   get_output_embeddings`  r.  z%XmodForCausalLM.get_output_embeddingsc                 C   r/  r   rE  rH   new_embeddingsrL   rL   rM   set_output_embeddingsd  r1  z%XmodForCausalLM.set_output_embeddingsNr   rN   r   rx   r.   r*   rO   r   r   labelsr   r   r   logits_to_keeprz   rQ   c                 K   s   |	durd}| j |f||||||||
||dd|}|j}t|tr)t| dn|}| |dd|ddf }d}|	durM| jd||	| jjd|}t	|||j
|j|j|jdS )aS  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XmodForCausalLM, AutoConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
        >>> config = AutoConfig.from_pretrained("facebook/xmod-base")
        >>> config.is_decoder = True
        >>> model = XmodForCausalLM.from_pretrained("facebook/xmod-base", config=config)
        >>> model.set_default_language("en_XX")

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)r   rx   r.   r*   rO   r   r   r   r   r   return_dict)logitsrK  r3   )lossrN  r   r   r  r	  rL   )r  r   r   re   slicerD  loss_functionrI   r3   r   r   r   r  r	  )rH   rN   r   rx   r.   r*   rO   r   r   rK  r   r   r   rL  rz   outputsr   slice_indicesrN  rO  rL   rL   rM   r`   g  sB   -zXmodForCausalLM.forward)NNNNNNNNNNNNr   )rj   rk   rl   _tied_weights_keysr1   rG  rJ  r!   r   r?   rn   ro   r   r   rp   re   r   r   r   r`   rr   rL   rL   rJ   rM   r=  G  sj    	
r=  c                       s   e Zd ZdddZ fddZdd Zdd	 Zee	
	
	
	
	
	
	
	
	
dde	j
d
B de	j
d
B de	jd
B de	j
d
B de	j
d
B de	jd
B de	jd
B de	jd
B de	j
d
B dee dee	j eB fddZ  ZS )XmodForMaskedLMr>  r?  r@  c                    s@   t  | |jrtd t|dd| _t|| _| 	  d S )NzkIf you want to use `XmodForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrA  rB  rG   rJ   rL   rM   r1     s   
zXmodForMaskedLM.__init__c                 C   r*  r   rE  r,  rL   rL   rM   rG    r.  z%XmodForMaskedLM.get_output_embeddingsc                 C   r/  r   rE  rH  rL   rL   rM   rJ    r1  z%XmodForMaskedLM.set_output_embeddingsNrN   r   rx   r.   r*   rO   r   r   rK  rz   rQ   c
                 K   sx   | j |f|||||||dd|
}|d }| |}d}|	dur2t }||d| jj|	d}t|||j|jdS )a  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        T)r   rx   r.   r*   rO   r   r   rM  r   Nr,   rO  rN  r   r  )	r  rD  r   r   rI   r3   r   r   r  )rH   rN   r   rx   r.   r*   rO   r   r   rK  rz   rR  r<  prediction_scoresmasked_lm_lossloss_fctrL   rL   rM   r`     s4   

zXmodForMaskedLM.forward)	NNNNNNNNN)rj   rk   rl   rT  r1   rG  rJ  r!   r   r?   rn   ro   r   r   r   rp   r   r`   rr   rL   rL   rJ   rM   rU    sR    	
rU  c                       (   e Zd ZdZ fddZdd Z  ZS )r  z*Roberta Head for masked language modeling.c                    sZ   t    t|j|j| _tj|j|jd| _t|j|j	| _
tt|j	| _d S r   )r0   r1   r   r   r4   r   r9   r:   
layer_normr3   rF  	Parameterr?   rC   r  rG   rJ   rL   rM   r1     s
   
zXmodLMHead.__init__c                 K   s*   |  |}t|}| |}| |}|S r   )r   r
   r[  rF  rH   featuresrz   xrL   rL   rM   r`     s
   


zXmodLMHead.forwardrj   rk   rl   rm   r1   r`   rr   rL   rL   rJ   rM   r    s    r  z
    X-MOD Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                          e Zd Z fddZee							ddejdB dejdB dejdB dejdB dejdB d	ejdB d
ejdB de	e
 deej eB fddZ  ZS )XmodForSequenceClassificationc                    s>   t  | |j| _|| _t|dd| _t|| _|   d S NFrA  )	r0   r1   
num_labelsrI   r%  r  XmodClassificationHead
classifierr(  rG   rJ   rL   rM   r1   ,  s   
z&XmodForSequenceClassification.__init__NrN   r   rx   r.   r*   rO   rK  rz   rQ   c              	   K   s,  | j |f|||||dd|}	|	d }
| |
}d}|dur| jjdu rI| jdkr/d| j_n| jdkrE|jtjks@|jtjkrEd| j_nd| j_| jjdkrgt	 }| jdkra||
 |
 }n+|||}n%| jjdkr~t }||d	| j|d	}n| jjdkrt }|||}t|||	j|	jd
S )a  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Tr   rx   r.   r*   rO   rM  r   Nr$   
regressionsingle_label_classificationmulti_label_classificationr,   rV  )r  rf  rI   problem_typerd  r/   r?   rE   re   r   squeezer   r   r   r   r   r  rH   rN   r   rx   r.   r*   rO   rK  rz   rR  r<  rN  rO  rY  rL   rL   rM   r`   7  sN   



"


z%XmodForSequenceClassification.forwardNNNNNNN)rj   rk   rl   r1   r!   r   r?   rn   ro   r   r   r   rp   r   r`   rr   rL   rL   rJ   rM   rb  $  s<    	
rb  c                       s   e Zd Z fddZee							ddejdB dejdB dejdB dejdB dejdB d	ejdB d
ejdB de	e
 deej eB fddZ  ZS )XmodForMultipleChoicec                    s@   t  | t|| _t|j| _t|j	d| _
|   d S )Nr$   )r0   r1   r%  r  r   r;   r<   r=   r   r4   rf  r(  rG   rJ   rL   rM   r1   |  s
   
zXmodForMultipleChoice.__init__NrN   r   r.   rx   rK  r*   rO   rz   rQ   c              	   K   sX  |dur	|j d n|j d }	|dur|d|dnd}
|dur.||d|d nd}|dur=|d|dnd}|durL|d|dnd}|dur[|d|dnd}|durn|d|d|dnd}| j|
f|||||dd|}|d }| |}| |}|d|	}d}|durt }|||}t|||j	|j
dS )	a|  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        lang_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr$   r,   r   T)r   r*   r.   rx   rO   rM  rV  )rY   r   rD   repeatr  r=   rf  r   r   r   r  )rH   rN   r   r.   rx   rK  r*   rO   rz   num_choicesflat_input_idsflat_lang_idsflat_position_idsflat_token_type_idsflat_attention_maskflat_inputs_embedsrR  r  rN  reshaped_logitsrO  rY  rL   rL   rM   r`     sH   .&



zXmodForMultipleChoice.forwardrn  )rj   rk   rl   r1   r!   r   r?   rn   ro   r   r   r   rp   r   r`   rr   rL   rL   rJ   rM   ro  y  s<    
	
ro  c                       ra  )XmodForTokenClassificationc                    sb   t  | |j| _t|dd| _|jd ur|jn|j}t|| _	t
|j|j| _|   d S rc  )r0   r1   rd  r%  r  classifier_dropoutr<   r   r;   r=   r   r4   rf  r(  rH   rI   r{  rJ   rL   rM   r1     s   z#XmodForTokenClassification.__init__NrN   r   rx   r.   r*   rO   rK  rz   rQ   c              	   K   s|   | j |f|||||dd|}	|	d }
| |
}
| |
}d}|dur4t }||d| j|d}t|||	j|	jdS )a  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Trg  r   Nr,   rV  )	r  r=   rf  r   r   rd  r   r   r  rm  rL   rL   rM   r`     s2   

z"XmodForTokenClassification.forwardrn  )rj   rk   rl   r1   r!   r   r?   rn   ro   r   r   r   rp   r   r`   rr   rL   rL   rJ   rM   rz    s<    	
rz  c                       rZ  )re  z-Head for sentence-level classification tasks.c                    sT   t    t|j|j| _|jd ur|jn|j}t|| _	t|j|j
| _d S r   )r0   r1   r   r   r4   r   r{  r<   r;   r=   rd  out_projr|  rJ   rL   rM   r1   $  s   
zXmodClassificationHead.__init__c                 K   sL   |d d dd d f }|  |}| |}t|}|  |}| |}|S r  )r=   r   r?   tanhr}  r]  rL   rL   rM   r`   -  s   




zXmodClassificationHead.forwardr`  rL   rL   rJ   rM   re  !  s    	re  c                       s   e Zd Z fddZee								ddejdB dejdB dejdB dejdB dejdB d	ejdB d
ejdB dejdB de	e
 deej eB fddZ  ZS )XmodForQuestionAnsweringc                    s@   t  | |j| _t|dd| _t|j|j| _| 	  d S rc  )
r0   r1   rd  r%  r  r   r   r4   
qa_outputsr(  rG   rJ   rL   rM   r1   :  s
   z!XmodForQuestionAnswering.__init__NrN   r   rx   r.   r*   rO   start_positionsend_positionsrz   rQ   c	              	   K   s
  | j |f|||||dd|	}
|
d }| |}|jddd\}}|d }|d }d}|durz|durzt| dkrG|d}t| dkrT|d}|d}|d|}|d|}t|d}|||}|||}|| d	 }t	||||
j
|
jd
S )r2  Trg  r   r$   r,   rc   N)ignore_indexr|   )rO  start_logits
end_logitsr   r  )r  r  splitrl  r   lenrD   clampr   r   r   r  )rH   rN   r   rx   r.   r*   rO   r  r  rz   rR  r<  rN  r  r  
total_lossignored_indexrY  
start_lossend_lossrL   rL   rM   r`   D  sJ   






z XmodForQuestionAnswering.forward)NNNNNNNN)rj   rk   rl   r1   r!   r   r?   rn   ro   r   r   r   rp   r   r`   rr   rL   rL   rJ   rM   r  7  sB    
	
r  )r=  rU  ro  r  rb  rz  r%  r  )Nrs   )Rrm   collections.abcr   r?   r   torch.nnr   r   r    r   r  activationsr	   r
   cache_utilsr   r   r   
generationr   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r    utils.genericr!   r"   utils.output_capturingr#   configuration_xmodr%   
get_loggerrj   r  Moduler&   rp   floatr   r   r   r   r   r   r   r   r   r   r   r  r%  r=  rU  r  rb  ro  rz  re  r  __all__rL   rL   rL   rM   <module>   s   (

q
JM*/M+7  nSOdBK