o
    ei                     @   s  d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$ e rqddlm%Z% e"&e'Z(de)de)dej*fddZ+dd Z,dej*dej*fddZ-dej*dej*dej*dej*fd d!Z.G d"d# d#ej/Z0G d$d% d%e0Z1e0e1d&Z2G d'd( d(ej/Z3G d)d* d*eZ4e!G d+d, d,eZ5e!G d-d. d.e5Z6e!d/d0G d1d2 d2e5eZ7e!d3d0G d4d5 d5e5Z8e!G d6d7 d7e5Z9g d8Z:dS )9zPyTorch GPT-J model.    N)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)create_causal_mask)!flash_attn_supports_top_left_maskis_flash_attn_available)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPast)PreTrainedModel)auto_docstringlogging   )
GPTJConfig)_flash_attention_forwardnum_posdimreturnc                 C   s`   ddt jd|dt jd|   }t dt j| t jd | }t jt |t |fddS )	Ng      ?i'  r      )dtypezi , j -> i jr   r   )torcharangeint64einsumfloatcatsincos)r   r   inv_freqsinusoid_inp r*   d/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/gptj/modeling_gptj.pycreate_sinusoidal_positions/   s    "r,   c                 C   s   |  |j|jd ddS Nr   r   )todevicerepeatshape)embed_positionsposition_idsr*   r*   r+   get_embed_positions5   s   r4   xc                 C   sb   | d d d d d d d d df }| d d d d d d dd df }t j| |fdd} | dS )Nr   r   r   )r    stackflatten)r5   x1x2r*   r*   r+   rotate_every_two9   s   ""
r<   tensorr&   r'   c                 C   s`   t |d d d d d d d f dd}t |d d d d d d d f dd}| | t| |  S )Nr   r   )r    repeat_interleaver<   )r=   r&   r'   r*   r*   r+   apply_rotary_pos_emb@   s   &&r?   c                       s   e Zd Zd fdd	Zdd Zdd Z	ddd	Zd
d Z						ddej	de
dB dej	dB dejdB dedB dedB dejdB deejeej f eejeej eejdf f B dB fddZ  ZS )GPTJAttentionNc                    sB  t    || _|j| _t|j| _t|j	| _
d| _|| _|d u r/td| jj d |j| _|j| _| j| j | _| j| j | jkrTtd| j d| j dt| j| _tj| j| jdd| _tj| j| jdd| _tj| j| jdd| _tj| j| jdd| _|j| _| jp| j| _| j d	t!| j| jdd
 d S )NTzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).Fbiasr2   )
persistent)"super__init__configmax_position_embeddingsmax_positionsr   Dropout
attn_pdropattn_dropoutresid_pdropresid_dropout	is_causal	layer_idxloggerwarning_once	__class____name__hidden_size	embed_dimnum_attention_headshead_dim
ValueErrormathsqrt
scale_attnLineark_projv_projq_projout_proj
rotary_dimpos_embd_dimregister_bufferr,   )selfrF   rO   rR   r*   r+   rE   G   s<   


zGPTJAttention.__init__c                 C   s|   |  dd ||f }||}|r|S t|jdkr%|dddddS t|jdkr4|ddddS td	t|j )
zO
        Splits hidden dim into attn_head_size and num_attention_heads
        Nr6      r   r   r   r      3Input tensor rank should be one of [4, 5], but is: )sizeviewlenr1   permuterX   )rd   r=   rV   attn_head_sizerotary	new_shaper*   r*   r+   _split_headsl   s   
zGPTJAttention._split_headsc                 C   s   t |jdkr|ddddd }nt |jdkr%|dddd }n
tdt |j | dd	 || f }||S )
zR
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        rf   r   r   r   r   rg   rh   Nr7   )rk   r1   rl   
contiguousrX   ri   rj   )rd   r=   rV   rm   ro   r*   r*   r+   _merge_heads{   s   
zGPTJAttention._merge_headsc                 C   s   | tj}| tj}t||dd}|| j }|d ur#|| }tjj|dd}| |j	}| 
|}t||}||fS )Nr6   r7   r   )r.   r    float32matmul	transposer[   r   
functionalsoftmaxr   rK   )rd   querykeyvalueattention_maskattn_weightsattn_outputr*   r*   r+   _attn   s   

zGPTJAttention._attnc                 C   s8   | j }|j|jkr||j}|| _ ||jd ddS r-   )r2   r/   r.   r0   r1   )rd   r3   r2   r*   r*   r+   _get_embed_positions   s
   z"GPTJAttention._get_embed_positionsFhidden_states
layer_pastr{   r3   	use_cacheoutput_attentionscache_positionr   .c                 C   s*  |  |}| |}	| |}
| || j| jd}| |	| j| jd}	| |
| j| jd}
| |}|ddd|j	d }t
|d||	j}t
j||j	d d dd\}}| jd ur|	d d d d d d d | jf }|	d d d d d d | jd f }|d d d d d d d | jf }|d d d d d d | jd f }t|||}t|||}t
j||gdd}	t
j||gdd}nt|	||}	t|||}|	dddd}	|dddd}|d ur||| j|d	}||	|
| j|\}	}
| ||	|
|\}}| || j| j}| |}| |}||fS )
NTFr6   r   r   r   r   r   r&   r'   partial_rotation_sizer   )r_   r]   r^   rp   rV   rW   r   	unsqueezer0   r1   r    gatherr.   r   splitra   r?   r%   rl   updaterO   r~   rr   r`   rM   )rd   r   r   r{   r3   r   r   r   rx   ry   rz   r2   repeated_position_idssincosr&   r'   k_rotk_passq_rotq_passcache_kwargsr}   r|   r*   r*   r+   forward   sF   




""""

zGPTJAttention.forwardNNNNFFN)rS   
__module____qualname__rE   rp   rr   r~   r   r    FloatTensorr	   
LongTensorbooltupleTensorr   __classcell__r*   r*   re   r+   r@   F   sD    %


r@   c                       s   e Zd ZdZ fddZ						ddejdedB dejdB d	ejdB d
e	dB de	dB dejdB de
eje
ej f e
eje
ej e
ejdf f B dB fddZ  ZS )GPTJFlashAttention2aD  
    GPTJ flash attention module. This module inherits from `GPTJAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                    s   t  j|i | t | _d S r   )rD   rE   r   _flash_attn_uses_top_left_mask)rd   argskwargsre   r*   r+   rE      s   zGPTJFlashAttention2.__init__NFr   r   r{   r3   r   r   r   r   .c              
   C   s:  |  |}| |}	| |}
| || j| jd}| |	| j| jd}	| |
| j| jd}
| |}|ddd|j	d }t
|d||	j}t
j||j	d d dd\}}| jd ur|	d d d d d d d | jf }|	d d d d d d | jd f }|d d d d d d d | jf }|d d d d d d | jd f }t|||}t|||}t
j||gdd}	t
j||gdd}nt|	||}	t|||}|	dddd}	|dddd}|d ur||| j|d	}||	|
| j|\}	}
|	dddd }	|dddd }|
dddd }
|j}|jjd
kr |jjnd}|t
jkr]t
|r4t
|}nt| jdr@| jj}n| j jj}td| d ||}|	|}	|
|}
| j re| jj!nd}|j	d }t"||	|
|||| j#| j$d}|%|j	d |j	d |j	d |j	d  }| &|}| '|}||fS )NTFr6   r   r   r   r   r   r   mpscpu_is_quantizedzThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .g        )dropoutrN   use_top_left_mask)(r_   r]   r^   rp   rV   rW   r   r   r0   r1   r    r   r.   r   r   ra   r?   r%   rl   r   rO   rq   r/   typers   is_autocast_enabledget_autocast_dtypehasattrrF   weightrP   rQ   trainingrJ   r   rN   r   reshaper`   rM   )rd   r   r   r{   r3   r   r   r   rx   ry   rz   r2   r   r   r&   r'   r   r   r   r   r   input_dtypedevice_typetarget_dtypeattention_dropoutquery_lengthr|   r}   r*   r*   r+   r      s   




""""





"

zGPTJFlashAttention2.forwardr   )rS   r   r   __doc__rE   r    r   r	   r   r   r   r   r   r   r*   r*   re   r+   r      s<    
r   )eagerflash_attention_2c                       s6   e Zd Z fddZdejdB dejfddZ  ZS )GPTJMLPc                    sJ   t    |j}t||| _t||| _t|j | _	t
|j| _d S r   )rD   rE   n_embdr   r\   fc_infc_outr   activation_functionactrI   rL   r   )rd   intermediate_sizerF   rU   re   r*   r+   rE   x  s   
zGPTJMLP.__init__r   Nr   c                 C   s,   |  |}| |}| |}| |}|S r   )r   r   r   r   )rd   r   r*   r*   r+   r     s
   



zGPTJMLP.forward)rS   r   r   rE   r    r   r   r   r*   r*   re   r+   r   w  s    "
r   c                       s   e Zd Zd fdd	Z						ddejdB dedB dejdB dejdB d	edB d
edB dejdB de	ej
 e	ej
e	ejdf f B dB fddZ  ZS )	GPTJBlockNc                    sZ   t    |jd ur|jnd|j }tj|j|jd| _t|j	 ||| _
t||| _d S )Nrg   eps)rD   rE   n_innerr   r   	LayerNormlayer_norm_epsilonln_1GPTJ_ATTENTION_CLASSES_attn_implementationattnr   mlp)rd   rF   rO   	inner_dimre   r*   r+   rE     s
   
zGPTJBlock.__init__Fr   r   r{   r3   r   r   r   r   .c              	   C   sH   |}|  |}| j|||||||d\}	}
| |}|	| | }||
fS )N)r   r   r{   r3   r   r   r   )r   r   r   )rd   r   r   r{   r3   r   r   r   residualattn_outputsr|   feed_forward_hidden_statesr*   r*   r+   r     s   



	zGPTJBlock.forwardr   r   )rS   r   r   rE   r    r   r	   r   r   r   r   r   r   r*   r*   re   r+   r     s2    
$	r   c                       s@   e Zd ZU eed< dZdZdgZdZdZ	dZ
 fddZ  ZS )GPTJPreTrainedModelrF   transformerTr   past_key_valuesc                    s6   t  | t|trt|jt|j|j	 d S d S r   )
rD   _init_weights
isinstancer@   initcopy_r2   r,   rH   rb   )rd   modulere   r*   r+   r     s   
z!GPTJPreTrainedModel._init_weights)rS   r   r   r   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_can_compile_fullgraphr   r   r*   r*   re   r+   r     s   
 r   c                       s   e Zd Z fddZdd Zdd Ze											ddejdB d	e	dB d
ej
dB dejdB dejdB dej
dB dedB dedB dedB dedB dejdB deeB fddZ  ZS )	GPTJModelc                    s   t     j| _ j| _t j| j| _t j	| _
t fddt jD | _tj| j jd| _d| _|   d S )Nc                    s   g | ]}t  |d qS ))rO   )r   ).0irF   r*   r+   
<listcomp>  s    z&GPTJModel.__init__.<locals>.<listcomp>r   F)rD   rE   r   rU   
vocab_sizer   	EmbeddingwterI   
embd_pdropdrop
ModuleListrangen_layerhr   r   ln_fgradient_checkpointing	post_initrd   rF   re   r   r+   rE     s    zGPTJModel.__init__c                 C   s   | j S r   r   )rd   r*   r*   r+   get_input_embeddings  s   zGPTJModel.get_input_embeddingsc                 C   s
   || _ d S r   r   )rd   new_embeddingsr*   r*   r+   set_input_embeddings  s   
zGPTJModel.set_input_embeddingsN	input_idsr   r{   token_type_idsr3   inputs_embedsr   r   output_hidden_statesreturn_dictr   r   c              
   K   s  |dur|n| j j}|	dur|	n| j j}	|dur|n| j j}|
dur$|
n| j j}
|du |duA r4td| jrC| jrC|rCt	d d}|du rL| 
|}|rX|du rXt| j d}|jd }|du rv|duri| nd}tj||| |jd}|du r|d}t| j |||||d	}|}|dur|d
|}| 
|}|| }| |}d
||d
f}|rdnd}|	rdnd}t| jD ]#\}}|	r||f }||||||||d}|d }|r||d f }q| |}||}|	r||f }|
stdd ||||fD S t||||dS )  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   r   r   r/   )rF   r   r{   r   r   r3   r6   r*   )r   r{   r3   r   r   r   c                 s   s    | ]	}|d ur|V  qd S r   r*   )r   vr*   r*   r+   	<genexpr>:  s    z$GPTJModel.forward.<locals>.<genexpr>)last_hidden_stater   r   
attentions)rF   r   r   r   use_return_dictrX   r   r   rP   rQ   r   r
   r1   get_seq_lengthr    r!   r/   r   r   rj   r   ri   	enumerater   r   r   r   )rd   r   r   r{   r   r3   r   r   r   r   r   r   r   
seq_lengthpast_key_values_lengthcausal_maskr   token_type_embedsoutput_shapeall_self_attentionsall_hidden_statesr   blockoutputsr*   r*   r+   r     s   


	







zGPTJModel.forwardNNNNNNNNNNN)rS   r   r   rE   r   r   r   r    r   r	   r   r   r   r   r   r   r*   r*   re   r+   r     sR    	
r   zK
    The GPT-J Model transformer with a language modeling head on top.
    )custom_introc                       s   e Zd ZddiZ fddZe													ddejdB dedB d	ej	dB d
ejdB dejdB dej	dB dejdB de
dB de
dB de
dB de
dB dejdB deejB deeB fddZ  ZS )GPTJForCausalLMzlm_head.weightztransformer.wte.weightc                    s4   t  | t|| _t|j|j| _| 	  d S r   )
rD   rE   r   r   r   r\   r   r   lm_headr   r   re   r*   r+   rE   N  s   
zGPTJForCausalLM.__init__Nr   r   r   r{   r   r3   r   labelsr   r   r   r   r   logits_to_keepr   c                 K   s   |dur|n| j j}| j||||||||	|
||d}|d }t|tr)t| dn|}| |dd|ddf }d}|durM| jd||| j jd|}|sc|f|dd  }|dura|f| S |S t	|||j
|j|jdS )aG  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)
r   r{   r   r3   r   r   r   r   r   r   r   )logitsr  r   r   lossr  r   r   r   r*   )rF   r   r   r   intslicer  loss_functionr   r   r   r   r   )rd   r   r   r{   r   r3   r   r  r   r   r   r   r   r  r   transformer_outputsr   slice_indicesr  r  outputr*   r*   r+   r   V  s<   zGPTJForCausalLM.forward)NNNNNNNNNNNNr   )rS   r   r   _tied_weights_keysrE   r   r    r   r	   r   r   r  r   r   r   r   r   r*   r*   re   r+   r
  F  s\    	
r
  a  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       s   e Zd Z fddZe											ddejdB dedB dejdB dejdB dejdB d	ejdB d
ejdB de	dB de	dB de	dB de	dB de
eB fddZ  ZS )GPTJForSequenceClassificationc                    s@   t  | |j| _t|| _tj|j| jdd| _| 	  d S )NFrA   )
rD   rE   
num_labelsr   r   r   r\   r   scorer   r   re   r*   r+   rE     s
   
z&GPTJForSequenceClassification.__init__Nr   r   r{   r   r3   r   r  r   r   r   r   r   c                 K   sD  |dur|n| j j}| j||||||||	|
|d
}|d }| |}|dur,|jd }n|jd }| j jdu r?|dkr?td| j jdu rHd}n1|durm|| j jk|jt	j
}t	j|jd |jt	j
d}|| d}nd}t| jj d |t	j||jd	|f }d}|dur||j}| j jdu r| jdkrd
| j _n| jdkr|jt	jks|jt	jkrd| j _nd| j _| j jd
krt }| jdkr|| | }n+|||}n%| j jdkrt }||d| j|d}n| j jdkrt }|||}|s|f|dd  }|dur|f| S |S t|||j|j|jdS )a!  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)	r   r{   r   r3   r   r   r   r   r   r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r6   )r/   r   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r   
regressionsingle_label_classificationmulti_label_classificationr  )rF   r   r   r  r1   pad_token_idrX   r.   r/   r    int32r!   argmaxrP   rQ   rR   rS   problem_typer  r   longr  r   squeezer   rj   r   r   r   r   r   )rd   r   r   r{   r   r3   r   r  r   r   r   r   r   r  r   r  
batch_sizelast_non_pad_tokennon_pad_masktoken_indicespooled_logitsr  loss_fctr  r*   r*   r+   r     sz   



"


z%GPTJForSequenceClassification.forwardr  )rS   r   r   rE   r   r    r   r	   r   r   r   r   r   r   r*   r*   re   r+   r    sN    		
r  c                       s   e Zd Z fddZe										ddejdB dejdB dejdB dejdB dejdB d	ejdB d
ejdB dedB dedB dedB de	e
B fddZ  ZS )GPTJForQuestionAnsweringc                    s<   t  | |j| _t|| _t|j|j| _| 	  d S r   )
rD   rE   r  r   r   r   r\   rT   
qa_outputsr   r   re   r*   r+   rE     s
   
z!GPTJForQuestionAnswering.__init__Nr   r{   r   r3   r   start_positionsend_positionsr   r   r   r   c              
   K   sV  |
dur|
n| j j}
| j|||||||	|
d}|d }| |}|jddd\}}|d }|d }d}|dur|durt| dkrR|d	|j
}t| dkrc|d	|j
}|d}|d|}|d|}t|d}|||}|||}|| d }|
s||f|dd  }|dur|f| S |S t||||j|jd	S )
r   N)r{   r   r3   r   r   r   r   r   r   r6   r   )ignore_indexr   )r  start_logits
end_logitsr   r   )rF   r   r   r+  r   r#  rq   rk   ri   r.   r/   clampr   r   r   r   )rd   r   r{   r   r3   r   r,  r-  r   r   r   r   r  sequence_outputr  r/  r0  
total_lossignored_indexr)  
start_lossend_lossr  r*   r*   r+   r   !  sN   




z GPTJForQuestionAnswering.forward)
NNNNNNNNNN)rS   r   r   rE   r   r    r   r   r   r   r   r   r   r*   r*   re   r+   r*    sH    		
r*  )r
  r*  r  r   r   );r   rY   r    r   torch.nnr   r   r    r   r   activationsr   cache_utilsr	   r
   
generationr   masking_utilsr   modeling_flash_attention_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   configuration_gptjr   r   
get_loggerrS   rP   r  r   r,   r4   r<   r?   Moduler@   r   r   r   r   r   r   r
  r  r*  __all__r*   r*   r*   r+   <module>   s`   
" % 	# 	MpQ