o ۷i~@sjdZddlmZmZddlZddlZddlmZddlm Z m Z m Z ddl m Z mZddlmZdd lmZmZmZdd lmZdd lmZmZmZdd lmZmZd dlmZe e!Z"ddZ#ddZ$d'ddZ%Gdddej&Z'ddZ(Gdddej&Z)eGdddeZ*eGddde*Z+edd Gd!d"d"e*eZ,ed#d Gd$d%d%e*Z-gd&Z.dS)(zPyTorch CTRL model.)OptionalUnionN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)Cache DynamicCache)GenerationMixin)BaseModelOutputWithPastCausalLMOutputWithPastSequenceClassifierOutput)PreTrainedModel)Conv1D find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging) CTRLConfigcCs$dtdd|d|}||S)Nri')torchpow)posi d_model_size angle_ratesr\/home/ubuntu/vllm_env/lib/python3.10/site-packages/transformers/models/ctrl/modeling_ctrl.py angle_defn(sr cCsttj|tjd|dtj|tjd|d|}t|dddddf}t|dddddf}tj||gdd}|S)Ndtyperrrdim) r rarangeint64to unsqueezesincoscat)positionrr" angle_radssinescosines pos_encodingrrrpositional_encoding-sr2c Cst||dddd}|jd}|t|}|dur7|d|d} } ||| | | d| fd7}|dur?||}tj|dd} |durN| |} t| |} | | fS) Nrrrrr#gr$)rmatmulpermuteshapenpsqrtsizesoftmax) qkvmaskattention_mask head_mask matmul_qkdkscaled_attention_logitsndnsattention_weightsoutputrrrscaled_dot_product_attention<s   rHcsDeZdZd fdd ZddZddZ      d d d ZZS) MultiHeadAttentionNcspt||_||_||_t||j|_t|||_ t|||_ t|||_ t|||_ t |_dSN)super__init__ num_headsr layer_idxintdepthrLinearWqWkWvdenseset pruned_heads)selfrrMrN __class__rrrLWs  zMultiHeadAttention.__init__cCs|j|j}t|dkrdSt||j||j\}}t|j||_t|j||_t|j||_t|j |dd|_ |jt||_||j|_|j ||_dS)Nrrr$) rrMlenrrWrrRrSrTrUunion)rXheadsattention_head_sizeindexrrr prune_headsfs   zMultiHeadAttention.prune_headscCs"||d|j|j}|gdS)Nr#rrrr)reshaperMrPr5)rXx batch_sizerrrsplit_into_headswsz#MultiHeadAttention.split_into_headsFc Cs|jd} ||}||}||}||| }||| }||| }|dur7||||jd| i\}}t||||||} | dgd} | d}| | d|j }| |} | |fS)Nrcache_positionrarr#) r6rRrSrTreupdaterNrHr5rbrrU)rXr=r<r;r> layer_pastr?r@ use_cacheoutput_attentionsrfrdrGscaled_attentionattnoriginal_size_attentionrrrforward{s       zMultiHeadAttention.forwardrJNNNFFN)__name__ __module__ __qualname__rLr`rern __classcell__rrrYrrIVs rIcCs"tt||tt||SrJ)r SequentialrQReLU)rdffrrrpoint_wise_feed_forward_networks"rwcs4eZdZdfdd Z      d ddZZS) EncoderLayer皙?Ncsbtt|||d|_t|||_tj|dd|_tj|dd|_ t ||_ t ||_ dS)NrNgư>eps) rKrLrImulti_head_attentionrwffnr LayerNorm layernorm1 layernorm2Dropoutdropout1dropout2)rXrrMrvraterNrYrrrLs   zEncoderLayer.__init__Fc  Cs~||} |j| | | |||||||d } | d} || } || } || } || }||}| |} | f| dd}|S)Nrhr?r@rirjrfrr)rr}rrr~r)rXrcr>rhr?r@rirjrfnormed attn_outputs attn_outputout1out2 ffn_outputoutputsrrrrns,     zEncoderLayer.forward)ryNro)rprqrrrLrnrsrrrYrrxsrxc@s"eZdZUeed<dZddZdS)CTRLPreTrainedModelconfig transformercCst|tjtfr"|jjjd|jjd|j dur |j j dSdSt|tj rE|jjjd|jjd|j durC|jj|j  dSdSt|tj rZ|j j |jjddSdS)zInitialize the weights.g)meanstdN?) isinstancerrQrweightdatanormal_rinitializer_rangebiaszero_ Embedding padding_idxrfill_)rXmodulerrr _init_weightss     z!CTRLPreTrainedModel._init_weightsN)rprqrrr__annotations__base_model_prefixrrrrrrs  rcseZdZfddZddZddZddZe dd ee j d ee d ee j d ee j dee j dee j dee j dee dee dee dee dee jdeee jeffddZZS) CTRLModelcstj|_j|_tj|jtj |_ t j j|_t j|_t fddtjD|_t jjjd|_|dS)Nc s&g|]}tjjjj|dqS)rz)rxn_embdn_headrv resid_pdrop).0rrrr sz&CTRLModel.__init__..r{)rKrLrrn_layer num_layersr2 n_positionsrfloatr1rr vocab_sizewr embd_pdropdropout ModuleListrangehrlayer_norm_epsilon layernorm post_initrXrrYrrrLs   zCTRLModel.__init__cCs|jSrJr)rXrrrget_input_embeddingsszCTRLModel.get_input_embeddingscCs ||_dSrJr)rXnew_embeddingsrrrset_input_embeddingss zCTRLModel.set_input_embeddingscCs(|D] \}}|j|j|qdS)zv Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} N)itemsrr}r`)rXheads_to_prunelayerr]rrr _prune_headsszCTRLModel._prune_headsN input_idspast_key_valuesr?token_type_ids position_idsr@ inputs_embedsrirjoutput_hidden_states return_dictrfreturnc  KsD| dur| n|jj} |dur|n|jj}| dur| n|jj} | dur$| n|jj} |dur4|dur4td|durP||||}|d|d}|j d}n|durb|dd}|j d}ntd|durm|j n|j }|r||dur|t |jd}|rt |t rtdt |}|dur|nd}|durtj||d|tj|d}|d}|dur|dkrtd ||d}|d d }|j|jd }d |t|jj}|||jj}|dur|d|d}||}|t|j9}nd}|dur||}|d}t t!||||d |}|t|j9}|j"||_"|j"|ddf}|||}|#|}| rEdnd}| rLdnd}t$|j%D])\}}| r_||f}||||||||| | d}|d}| r{||d f7}qS|&|}| r||f}| st dd||||fDSt'||||dS)aE input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) Example: ```python >>> from transformers import AutoTokenizer, CTRLModel >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 5, 1280] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer#rz5You have to specify either input_ids or inputs_embedsrzPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `DynamicCache` instead, e.g. `past_key_values=DynamicCache.from_legacy_cache(past_key_values)`.)r"devicez$batch_size has to be defined and > 0rrr!rrrcss|] }|dur|VqdSrJr)rr=rrr sz$CTRLModel.forward..)last_hidden_stater hidden_states attentions)(rrjriruse_return_dict ValueError%warn_if_padding_and_no_attention_maskr9viewr6rr rtuplelogger warning_oncefrom_legacy_cacheget_seq_lengthrr&longr)r(r"finfomin get_head_maskrrr7r8rtriuonesr1r enumeraterrr )rXrrr?rrr@rrirjrrrfkwargs input_shaperdr past_lengthtoken_type_embedsseq_lenr> pos_embedsrall_hidden_statesall_attentionsrrrrrrrns1           "       zCTRLModel.forward NNNNNNNNNNNN)rprqrrrLrrrrrr LongTensorr FloatTensorboolTensorrrr rnrsrrrYrrsZ      rz The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introc seZdZdgZfddZe             ddeejdee deej deejd eejd eej d eej d eejd ee dee dee dee deej de eej effddZdddZZS)CTRLLMHeadModelzlm_head.weightcs8t|t||_tj|j|jdd|_| dS)NTr) rKrLrrrrQrrlm_headrrrYrrrLs   zCTRLLMHeadModel.__init__Nrrr?rrr@rlabelsrirjrrrfrcKs| dur| n|jj} |j|||||||| | | | | d }|d}||}d}|dur8|j||fd|jji|}| sN|f|dd}|durL|f|S|St|||j|j|j dS)a  input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` Example: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLLMHeadModel >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLLMHeadModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Wikipedia The llama is", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> sequence_ids = model.generate(inputs["input_ids"]) >>> sequences = tokenizer.batch_decode(sequence_ids) >>> sequences ['Wikipedia The llama is a member of the family Bovidae. It is native to the Andes of Peru,'] >>> outputs = model(**inputs, labels=inputs["input_ids"]) >>> round(outputs.loss.item(), 2) 9.21 >>> list(outputs.logits.shape) [1, 5, 246534] ```N) rr?rrr@rrirjrrrfrrr)losslogitsrrr) rrrr loss_functionrr rrr)rXrrr?rrr@rrrirjrrrfrtransformer_outputsr lm_logitsrrGrrrrnsJ< zCTRLLMHeadModel.forwardc Ks|dur#|}|jd|kr|}n|jdd}|dd|df}|||d}|dd|D]\}} ||vrGtd|d| ||<q3|S)Nr)rrrirz Warning: z is not a recognized input.)rr6poprprint) rXrrrirrremove_prefix_length model_inputskeyvaluerrrprepare_inputs_for_generation-s  z-CTRLLMHeadModel.prepare_inputs_for_generation) NNNNNNNNNNNNNNN)rprqrr_tied_weights_keysrLrrrrr rrrrrr rnrrsrrrYrrs^       era The CTRL Model transformer with a sequence classification head on top (linear layer). [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). cseZdZfddZe            ddeejdeedeej deejdeejd eej d eej d eejd ee d ee dee dee de e ej effddZZS)CTRLForSequenceClassificationcs@t||j|_t||_tj|j|jdd|_| dS)NFr) rKrL num_labelsrrrrQr classifierrrrYrrrLVs   z&CTRLForSequenceClassification.__init__Nrrr?rrr@rrrirjrrrc  CsJ| dur| n|jj} |j|||||||| | | | d } | d}||}|dur1|jdd\}}n |jdd\}}|jjdurH|dkrHtd|jjdurQd}n1|durv||jjk|jt j }t j |jd|jt j d}|| d}n d}t |jjd |t j ||jd |f}d}|dur|jjdur|jdkrd |j_n|jdkr|jt jks|jt jkrd |j_nd |j_|jjd krt}|jdkr|||}n,|||}n&|jjd krt}||d|j|d}n|jjd krt}|||}| s|f| dd}|dur|f|S|St||| j| jdS)a2 input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Example of single-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> import torch >>> torch.manual_seed(42) # doctest: +IGNORE_RESULT >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> labels = torch.tensor(1) >>> loss = model(**inputs, labels=labels).loss >>> round(loss.item(), 2) 0.93 ``` Example of multi-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained( ... "Salesforce/ctrl", problem_type="multi_label_classification" ... ) >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> num_labels = len(model.config.id2label) >>> labels = torch.nn.functional.one_hot(torch.tensor([predicted_class_id]), num_classes=num_labels).to( ... torch.float ... ) >>> loss = model(**inputs, labels=labels).loss >>> loss.backward() # doctest: +IGNORE_RESULT ```N) rr?rrr@rrirjrrrrrz=Cannot handle batch sizes > 1 if no padding token is defined.r#)rr"z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`)r regressionsingle_label_classificationmulti_label_classification)rrrr)rrrrr6 pad_token_idrr(rrint32r&argmaxrrrZrp problem_typerr"rrOrsqueezerrrrrr)rXrrr?rrr@rrrirjrrrrrrdsequence_lengthlast_non_pad_token non_pad_mask token_indices pooled_logitsrloss_fctrGrrrrn_sxh       "      z%CTRLForSequenceClassification.forwardr)rprqrrrLrrrrr rrrrrrrnrsrrrYrrJsT       r)rrrrr)/__doc__typingrrnumpyr7rrtorch.nnrrr cache_utilsr r generationr modeling_outputsr r rmodeling_utilsr pytorch_utilsrrrutilsrrconfiguration_ctrlr get_loggerrprr r2rHModulerIrwrxrrrr__all__rrrrsH      G2M  ?