o
    wiˊ                     @   s  d Z ddlZddlmZmZ ddlZddlmZ ddlZddlm	Z	m
Z
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, e# rddl-m.Z.m/Z/ G dd de*Z0G dd de(Z1G dd de&Z2G dd de'Z3e"G dd deZ4e"G dd  d e4Z5G d!d" d"ee!Z6e"d#d$G d%d& d&e4eZ7e"G d'd( d(e4Z8e"d)d$G d*d+ d+e4Z9g d,Z:dS )-zPyTorch BioGPT model.    N)OptionalUnion)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)CacheEncoderDecoderCache)GenerationMixin)AttentionMaskConverter)FlashAttentionKwargs))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentions SequenceClassifierOutputWithPastTokenClassifierOutput)PreTrainedModel)Unpack)
LossKwargsauto_docstringis_torch_flex_attn_availablelogger   )BartAttentionBartDecoderLayerBartScaledWordEmbedding)OPTLearnedPositionalEmbedding   )BioGptConfig)	BlockMaskmake_flex_block_causal_maskc                       s8   e Zd Z		ddejdedeej f fddZ  ZS )	 BioGptLearnedPositionalEmbeddingr   Nattention_maskpast_key_values_lengthposition_idsc                    s   t  ||| dS )z3`input_ids_shape` is expected to be [bsz x seqlen].N)superforward)selfr"   r#   r$   	__class__ f/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/biogpt/modular_biogpt.pyr&   <   s   z(BioGptLearnedPositionalEmbedding.forwardr   N)	__name__
__module____qualname__torch
LongTensorintr   r&   __classcell__r*   r*   r(   r+   r!   ;   s    r!   c                   @      e Zd ZdS )BioGptScaledWordEmbeddingNr-   r.   r/   r*   r*   r*   r+   r5   F       r5   c                   @   r4   )BioGptAttentionNr6   r*   r*   r*   r+   r8   J   r7   r8   c                       s   e Zd Zddedee f fddZ							ddejd	eej d
eej dee	 dee
 dee
 deej deej dee deejeeejejf  f fddZ  ZS )BioGptDecoderLayerNconfig	layer_idxc              	      sv   t  | |j| _t| j|j|jdd||d| _|j| _	t
|j | _t| j|j| _t|j| j| _| `| `d S )NT)	embed_dim	num_headsdropout
is_decoder	is_causalr:   r;   )r%   __init__hidden_sizer<   r8   num_attention_headsattention_probs_dropout_prob	self_attnhidden_dropout_probr>   r   
hidden_actactivation_fnnnLinearintermediate_sizefc1fc2encoder_attnencoder_attn_layer_norm)r'   r:   r;   r(   r*   r+   rA   O   s"   	zBioGptDecoderLayer.__init__FThidden_statesr"   layer_head_maskpast_key_valueoutput_attentions	use_cacher$   cache_positionflash_attn_kwargsreturnc	              
   K   s   |}
|  |}| jd|||||||d|	\}}}tjj|| j| jd}|
| }|}
| |}| |}| |}tjj|| j	| jd}| 
|}tjj|| j| jd}|
| }|f}|rc||f7 }|rj||f7 }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence. It is used to update the
                cache in the correct position and to infer the complete sequence length.
        )rP   rR   r"   rQ   rS   r$   rU   ptrainingNr*   )self_attn_layer_normrE   rI   
functionalr>   rZ   final_layer_normrL   rH   activation_dropoutrM   )r'   rP   r"   rQ   rR   rS   rT   r$   rU   rV   residualself_attn_weightsoutputsr*   r*   r+   r&   e   s:   







zBioGptDecoderLayer.forwardN)NNNFTNN)r-   r.   r/   r   r   r2   rA   r0   Tensorr	   boolr1   r   r   tupleFloatTensorr&   r3   r*   r*   r(   r+   r9   N   s<    	
r9   c                   @   s   e Zd ZeZdZdZdZdZdZ	dZ
dZdd Zdeeejdf  dejdejd	efd
dZedejdededejdejdefddZdS )BioGptPreTrainedModelbiogptTc                 C   s   t |tjr |jjjd| jjd |jdur|jj	  dS dS t |tj
rC|jjjd| jjd |jdurA|jj|j 	  dS dS t |tjrX|jj	  |jjd dS dS )zInitialize the weights        )meanstdN      ?)
isinstancerI   rJ   weightdatanormal_r:   initializer_rangebiaszero_	Embeddingpadding_idx	LayerNormfill_)r'   moduler*   r*   r+   _init_weights   s   

z#BioGptPreTrainedModel._init_weightsr"   r   input_tensorrU   past_key_valuesc                 C   sb  | j jdkr*t|tjrt|}|S |d u r(ttj|jd |jd f|jd}|S | j jdkr>|d ur<|dk	 r<|S d S |d urF|
 nd}|d urO|jnd}| j jdkre|setj|||| jd	red S |j}|jd }|rt| }	nt|tjr|jd
 n|| d }	| j|||	|||jd d}
| j jdkr|d ur|jjdv rt|j}t|
|}
|
S )Nflex_attentionr   r   )sizedeviceflash_attention_2ri   Fsdpa)inputs_embedsr#   is_training)sequence_lengthtarget_lengthdtyperU   
batch_size)cudaxpunpu)r:   _attn_implementationrm   r0   rc   r    onesshaper~   anyget_seq_lengthis_compileabler   _ignore_causal_mask_sdparZ   r   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positiontypefinfomin_unmask_unattended)r'   r"   rz   rU   r{   past_seen_tokensusing_compilable_cacher   r   r   causal_mask	min_dtyper*   r*   r+   _update_causal_mask   s`   





z)BioGptPreTrainedModel._update_causal_maskr   r   r   r   c                 K   sD  | dur|   dkr| }|S t|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9 }|ddddddf 	|ddd}| dur|
 }| jd }	|ddddddd|	f | ddddddf |j }
|
dk}
|ddddddd|	f |
||ddddddd|	f< |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuer   r~   r   )diagonalr~   r   r   )dimr0   r   r   fullr~   triuarangereshapeexpandcloner   tomasked_fill)r"   r   r   r   rU   r   kwargsr   r   mask_lengthpadding_maskr*   r*   r+   r     s,    $
6  zKBioGptPreTrainedModel._prepare_4d_causal_attention_mask_with_cache_positionN)r-   r.   r/   r   config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_2_supports_sdpa_supports_flex_attn_supports_cache_class_supports_static_cachery   r   r   r0   rc   r	   r   staticmethodr2   r   r   r*   r*   r*   r+   rg      sB    
Lrg   c                       s   e Zd Zdef fddZdd Zdd Ze											dd	ee	j
 d
ee	j dee	j dee	j deeee	j   dee dee	j
 dee dee dee dee	j dee deeef fddZ  ZS )BioGptModelr:   c                    s   t     | _ j| _ j| _ j| _ j| _	 j
r"t jnd}t j| j| j	|d| _t j| j| _t fddt jD | _t| j| _d| _|   d S )Nrl   )embed_scalec                    s   g | ]}t  |d qS ))r;   )r9   ).0ir:   r*   r+   
<listcomp>[  s    z(BioGptModel.__init__.<locals>.<listcomp>F)r%   rA   r:   	layerdroprF   r>   rB   r<   pad_token_idru   scale_embeddingmathsqrtr5   
vocab_sizeembed_tokensr!   max_position_embeddingsembed_positionsrI   
ModuleListrangenum_hidden_layerslayersrv   
layer_normgradient_checkpointing	post_init)r'   r:   r   r(   r   r+   rA   M  s    zBioGptModel.__init__c                 C      | j S rb   r   r'   r*   r*   r+   get_input_embeddingsb     z BioGptModel.get_input_embeddingsc                 C   
   || _ d S rb   r   r'   valuer*   r*   r+   set_input_embeddingse     
z BioGptModel.set_input_embeddingsN	input_idsr"   	head_maskr   r{   rT   r$   rS   output_hidden_statesreturn_dictrU   rV   rW   c           !      K   s|  |d ur|n| j j}|	d ur|	n| j j}	|d ur|n| j j}|
d ur$|
n| j j}
|d u |d uA r4td|d urF|}|j}|d|d }n|d ur^| d d }|d d d d df }ntd|d u rk| 	|}| j
rz| jrz|rztd d}d}|rt|tsd}td t|}| d d \}}|d ur| nd}|d u rtj||| |jd	}|d u r|| }tj|||jd	}t|tr|jn|}| ||||}|d u rtj|d
d}|| d
  }|d d |d f }| j|||d}|| }tjj|| j| jd}| j
r| jr|rtd d}|	r dnd }|r'dnd }d }|r0dnd }t| j D ]Q\}}|	rC||f7 }| jrTt!g }|| j"k rTq7||f||d ura|| nd |||||d|}|d }|r}||rzdnd
 }|r||d
 f7 }q7|	r||f7 }| #|}|r|nd } |r|$ } |
st%dd || |||fD S t&|| |||dS )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timer   zEYou have to specify either decoder_input_ids or decoder_inputs_embedsz[`use_cache=True` is incompatible with gradient checkpointing`. Setting `use_cache=False`...FTzPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.r   r   r   )r   )r$   rX   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...r*   )r"   rQ   rR   rS   rT   r$   rU   r   c                 s   s    | ]	}|d ur|V  qd S rb   r*   )r   vr*   r*   r+   	<genexpr>  s    z&BioGptModel.forward.<locals>.<genexpr>)last_hidden_stater{   rP   
attentionscross_attentions)'r:   rS   r   rT   use_return_dict
ValueErrorr   viewr}   r   r   rZ   r   warning_oncerm   r	   r
   from_legacy_cacher   r0   r   r~   r   self_attention_cacher   cumsumlongr   rI   r\   r>   	enumerater   randr   r   to_legacy_cachere   r   )!r'   r   r"   r   r   r{   rT   r$   rS   r   r   rU   rV   inputinput_shapereturn_legacy_cacher   
seq_lengthr#   mask_seq_lengthself_attn_cacher   	positionsrP   all_hidden_statesall_self_attnsall_cross_attentionsnext_decoder_cacheidxdecoder_layerdropout_probabilitylayer_outputs
next_cacher*   r*   r+   r&   h  s   



	

zBioGptModel.forward)NNNNNNNNNNN)r-   r.   r/   r   rA   r   r   r   r   r0   r1   rf   re   rc   rd   r   r   r   r   r&   r3   r*   r*   r(   r+   r   K  sV    	

r   c                   @   r4   )KwargsForCausalLMNr6   r*   r*   r*   r+   r     s    r   zR
    BioGPT Model with a `language modeling` head on top for CLM fine-tuning.
    )custom_introc                        s   e Zd ZdgZ fddZdd Zdd Ze												dd	ee	j
 d
ee	j dee	j dee	j deeee	j   dee	j
 dee dee	j
 dee dee dee dee	j dee deeef fddZedd Z  ZS )BioGptForCausalLMzoutput_projection.weightc                    s8   t  | t|| _tj|j|jdd| _| 	  d S NF)rr   )
r%   rA   r   rh   rI   rJ   rB   r   output_projectionr   r'   r:   r(   r*   r+   rA     s   
zBioGptForCausalLM.__init__c                 C   r   rb   r   r   r*   r*   r+   get_output_embeddings  r   z'BioGptForCausalLM.get_output_embeddingsc                 C   r   rb   r   )r'   new_embeddingsr*   r*   r+   set_output_embeddings  r   z'BioGptForCausalLM.set_output_embeddingsNr   r"   r   r   r{   labelsrT   r$   rS   r   r   rU   r   rW   c                 K   s   |dur|n| j j}| j|f|||||||	|
||d
|}|d }| |}d}|dur;| j||fd| j ji|}|sQ|f|dd  }|durO|f| S |S t|||j|j|j	|j
dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)
r"   r   r   r{   rT   r$   rS   r   r   rU   r   r   r   )losslogitsr{   rP   r   r   )r:   r   rh   r   loss_functionr   r   r{   rP   r   r   )r'   r   r"   r   r   r{   r  rT   r$   rS   r   r   rU   r   ra   sequence_outputprediction_scoreslm_lossoutputr*   r*   r+   r&   "  sP   
zBioGptForCausalLM.forwardc                    s.   d}| D ]}|t  fdd|D f7 }q|S )Nr*   c                 3   s$    | ]}| d  |jV  qdS r,   )index_selectr   r~   )r   
past_statebeam_idxr*   r+   r   h  s   " z3BioGptForCausalLM._reorder_cache.<locals>.<genexpr>)re   )r{   r  reordered_past
layer_pastr*   r  r+   _reorder_cachec  s   z BioGptForCausalLM._reorder_cacheNNNNNNNNNNNN)r-   r.   r/   _tied_weights_keysrA   r  r  r   r   r0   r1   rf   re   rc   rd   r   r   r   r   r&   r   r  r3   r*   r*   r(   r+   r     sb    		

@r   c                        s   e Zd Z fddZe													ddeej deej deej deej dee	e	ej
   d	eej d
eej dee deej dee dee dee deej
 dee	ef fddZ  ZS )BioGptForTokenClassificationc                    sj   t  | |j| _t|| _t|dr|jd ur|j}n|j}t	|| _
t|j|j| _|   d S )Nclassifier_dropout)r%   rA   
num_labelsr   rh   hasattrr  rF   rI   Dropoutr>   rJ   rB   
classifierr   )r'   r:   r  r(   r*   r+   rA   o  s   
z%BioGptForTokenClassification.__init__Nr   token_type_idsr"   r   r{   r   r  rT   r$   rS   r   r   rU   rW   c                 C   s  |dur|n| j j}| j|||||||	|
|||d}|d }| |}| |}d}|durgt }|durZ|ddk}|d| j}t	||dt
|j|}|||}n||d| j|d}|s}|f|dd  }|dur{|f| S |S t|||j|jdS )  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N
r{   r"   r   r   rT   r$   rS   r   r   rU   r   r   r   r   )r  r  rP   r   )r:   r   rh   r>   r  r   r   r  r0   wheretensorignore_indextype_asr   rP   r   )r'   r   r  r"   r   r{   r   r  rT   r$   rS   r   r   rU   transformer_outputsrP   r  r  loss_fctactive_lossactive_logitsactive_labelsr  r*   r*   r+   r&   }  sJ   

z$BioGptForTokenClassification.forward)NNNNNNNNNNNNN)r-   r.   r/   rA   r   r   r0   r1   rf   re   rc   rd   r   r   r&   r3   r*   r*   r(   r+   r  m  sZ    	

r  a  
    The BioGpt Model transformer with a sequence classification head on top (linear layer).

    [`BioGptForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it is required to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       s   e Zd Zdef fddZe												ddeej deej	 deej	 dee
e
ej   d	eej	 d
eej dee deej dee dee dee deej dee
ef fddZdd Zdd Z  ZS )BioGptForSequenceClassificationr:   c                    s@   t  | |j| _t|| _tj|j| jdd| _| 	  d S r   )
r%   rA   r  r   rh   rI   rJ   rB   scorer   r   r(   r*   r+   rA     s
   
z(BioGptForSequenceClassification.__init__Nr   r"   r   r{   r   r  rT   r$   rS   r   r   rU   rW   c                 C   s  |dur|n| j j}| j||||||||	|
||d}|d }| |}|dur1|jdd \}}n	|jdd \}}| j jdu rCd}n"|durYt|| j jdd 	|j
}nd}t| jj d |tj||j
d|f }d}|dur| j jdu r| jdkrd	| j _n| jdkr|jtjks|jtjkrd
| j _nd| j _| j jd	krt }| jdkr|| | }n+|||}n%| j jd
krt }||d| j|d}n| j jdkrt }|||}|s|f|dd  }|dur|f| S |S t|||j|j|jdS )r  Nr  r   r   r   r   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r   
regressionsingle_label_classificationmulti_label_classification)r  r  r{   rP   r   )r:   r   rh   r(  r   r   r0   nesumr   r~   r   r   r)   r-   r   problem_typer  r   r   r2   r   squeezer   r   r   r   r{   rP   r   )r'   r   r"   r   r{   r   r  rT   r$   rS   r   r   rU   r"  rP   r  r   r   pooled_logitsr  r#  r  r*   r*   r+   r&     sr   
$

"


z'BioGptForSequenceClassification.forwardc                 C   s   | j jS rb   rh   r   r   r*   r*   r+   r   7  s   z4BioGptForSequenceClassification.get_input_embeddingsc                 C   s   || j _d S rb   r1  r   r*   r*   r+   r   :  s   z4BioGptForSequenceClassification.set_input_embeddingsr  )r-   r.   r/   r   rA   r   r   r0   r1   rf   re   rc   rd   r   r   r&   r   r   r3   r*   r*   r(   r+   r'    sX    		

\r'  )r   r  r'  r   rg   );__doc__r   typingr   r   r0   torch.nnrI   torch.utils.checkpointr   r   r   activationsr   cache_utilsr	   r
   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   bart.modeling_bartr   r   r   opt.modeling_optr   configuration_biogptr   integrations.flex_attentionr   r    r!   r5   r8   r9   rg   r   r   r   r  r'  __all__r*   r*   r*   r+   <module>   sV   \ ! =]Tn