o wi@sXdZddlZddlZddlmZddlmZmZddlZddlm Z ddl m Z m Z m Z ddlmZdd lmZdd lmZmZmZmZmZmZmZdd lmZdd lmZmZmZm Z dd l!m"Z"m#Z#m$Z$ddl%m&Z&e$'e(Z)ddZ*Gddde j+Z,Gddde j+Z-Gddde-Z.e-e.dZ/Gddde j+Z0Gddde j+Z1Gddde j+Z2e#Gdd d eZ3ee#d!d"Gd#d$d$e"Z4e#Gd%d&d&e3Z5e#d'd"Gd(d)d)e3Z6Gd*d+d+e j+Z7Gd,d-d-e j+Z8e#Gd.d/d/e3Z9e#d0d"Gd1d2d2e3Z:e#Gd3d4d4e3Z;e#Gd5d6d6e3ZdS):zPyTorch ALBERT model.N) dataclass)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)#_prepare_4d_attention_mask_for_sdpa)BaseModelOutputBaseModelOutputWithPoolingMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indices"is_torch_greater_or_equal_than_2_2prune_linear_layer) ModelOutputauto_docstringlogging) AlbertConfigc Cs*zddl}ddl}ddl}Wn tytdwtj|}t d||j |}g}g} |D] \} } t d| d| |j || } | | | | q6t|| D]\} } t| q\t|| D]\} } | } | dd} | d d } | d d } | d d} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | dd} | d d!} | d"d#} | d$d%} t| dd&krd'| vsd(| vrd)| } d*| vr| d+d,} | d-d.} | d} d/| vs!d0| vs!d1| vs!d2| vs!d3| vr-t d4d| qj|}| D]}|d5|rA|d6|}n|g}|dd7ksR|dd8krXt|d.}nN|dd'ksf|dd9krlt|d:}n:|dd(kryt|d.}n-|dd;krt|d<}n z t||d}Wntyt d4d| Yq1wt|d=krt|d&}||}q1|d>dd?krt|d.}n |d7kr|| } z|j| jkrtd@|jdA| jdBWnty}z |j|j| jf7_d}~wwtdC| dD| t| |_qj|S)Ez'Load tf checkpoints in a pytorch model.rNzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape zmodule/ffn_1ffnzbert/zalbert/ attention_1 attentionz transform/ LayerNorm_1full_layer_layer_norm LayerNormzattention/LayerNormz transformer/zintermediate/dense/zffn/intermediate/output/dense/z ffn_output/z/output//z/self/z pooler/densepoolerzcls/predictions predictionszpredictions/attentionzembeddings/attention embeddings inner_group_zalbert_layers/group_zalbert_layer_groups/r output_biasoutput_weightsz classifier/seq_relationshipzseq_relationship/output_zsop_classifier/classifier/weightsweightadam_madam_vAdamWeightDecayOptimizerAdamWeightDecayOptimizer_1 global_stepz Skipping z [A-Za-z]+_\d+z_(\d+)kernelgammabetabiassquad classifieri _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight z from )renumpy tensorflow ImportErrorloggererrorospathabspathinfotrainlist_variables load_variableappendzipprintreplacelensplitjoin fullmatchgetattrAttributeErrorint transposeshape ValueErrorargstorch from_numpydata)modelconfigtf_checkpoint_pathr=nptftf_path init_varsnamesarraysnamerVarray original_namepointerm_name scope_namesnumermg/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/albert/modeling_albert.pyload_tf_weights_in_albert3s                          "                 roc sneZdZdZdeffdd Z     ddeejdeejd eejd eej d e d ej f d dZ Z S)AlbertEmbeddingszQ Construct the embeddings from word, position and token_type embeddings. r]csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_|jdt|jdddt|dd|_|jd tj|jtjd dddS) N) padding_idxeps position_ids)rF) persistentposition_embedding_typeabsolutetoken_type_idsdtype)super__init__r Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddingsr$layer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrYarangeexpandrRrwzerosrtsizelongselfr] __class__rmrnr}s  zAlbertEmbeddings.__init__Nr input_idsryrt inputs_embedspast_key_values_lengthreturnc Cs|dur |}n|dd}|d}|dur&|jdd|||f}|durPt|drE|jddd|f}||d|} | }n tj|tj|jjd}|durY| |}| |} || } |j dkrp| |} | | 7} | | } || } | S)Nrurryrr{devicerx)rrthasattrryrrYrrrrrrwrr$r) rrryrtrr input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedrr(rrmrmrnforwards,        zAlbertEmbeddings.forward)NNNNr)__name__ __module__ __qualname____doc__rr}rrY LongTensor FloatTensorrTTensorr __classcell__rmrmrrnrps*rpcseZdZdeffdd ZdejdejfddZdee dd fd d Z dd ejde ej de ej de deeejeejejfff ddZZS)AlbertAttentionr]cs4t|j|jdkrt|dstd|jd|j|j|_|j|_|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_t |j|_t |j|j|_tj|j|jd|_t|_t|dd|_|jdks|jd kr|j|_td |jd |j|_dSdS) NrrzThe hidden size (z6) is not a multiple of the number of attention heads (rrrwrx relative_keyrelative_key_queryr;r)r|r} hidden_sizenum_attention_headsrrWattention_head_size all_head_sizerLinearquerykeyvaluerattention_probs_dropout_probattention_dropoutroutput_dropoutdenser$rset pruned_headsrRrwrr~distance_embeddingrrrmrnr}s0  zAlbertAttention.__init__xrcCs6|dd|j|jf}||}|ddddS)Nrurr;rr )rrrviewpermute)rr new_x_shapermrmrntranspose_for_scoress z$AlbertAttention.transpose_for_scoresheadsNcCst|dkrdSt||j|j|j\}}t|j||_t|j||_t|j||_t|j |dd|_ |jt||_|j|j|_ |j ||_dS)Nrrdim) rNrrrrrrrrrrunion)rrindexrmrmrn prune_headss zAlbertAttention.prune_headsF hidden_statesattention_mask head_maskoutput_attentionscCs||}||}||}||}||} ||} t|| dd} | t|j } |dur8| |} |j dksB|j dkr| d} tj | tj |jddd} tj | tj |jddd}| |}|||jd}|j|jd}|j dkrtd||}| |} n|j dkrtd||}td | |}| ||} tjj| dd }||}|dur||}t|| }|d dd }||}||}|||}|r||fS|fS) Nrurrrrrzzbhld,lrd->bhlrzbhrd,lrd->bhlrrr;)rrrrrYmatmulrUmathsqrtrrwrrrrrrrtor{einsumr functionalsoftmaxrflattenrrr$)rrrrrmixed_query_layermixed_key_layermixed_value_layer query_layer key_layer value_layerattention_scoresrposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layerprojected_context_layerprojected_context_layer_dropoutlayernormed_context_layerrmrmrnr*sD               zAlbertAttention.forwardNNF)rrrrr}rYrrlistrTrrrboolrtuplerrrmrmrrnrs$rcsneZdZfddZ   d dejdeejdeejded e e eje ejejfff fd d Z Z S) AlbertSdpaAttentioncs t||j|_t |_dSN)r|r}r dropout_probrrequire_contiguous_qkvrrrmrnr}fs  zAlbertSdpaAttention.__init__NFrrrrrcs|jdks|rtdtj|||dS|\}}}|||}|||} || |} |j rM|j j dkrM|durM| }| } | } tjjj|| | ||jr[|jnddd} | dd } | |||j} || } || } ||| }|fS) NrxaAlbertSdpaAttention is used but `torch.nn.functional.scaled_dot_product_attention` does not support non-absolute `position_embedding_type` or `output_attentions=True` . Falling back to the eager attention implementation, but specifying the eager implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rcudaF)rrr attn_mask dropout_p is_causalrr;)rwrAwarningr|rrrrrrrrtype contiguousrYrrscaled_dot_product_attentiontrainingrrUreshaperrrr$)rrrrr batch_sizeseq_len_rrrattention_outputrrrrrmrnrks6   zAlbertSdpaAttention.forwardr) rrrr}rYrrrrrrrrrmrmrrnres  r)eagersdpacs~eZdZdeffdd Z    ddejdeejdeejd e d e d e ejejff d d Z dejd ejfddZ Z S) AlbertLayerr]cst||_|j|_d|_tj|j|jd|_ t |j ||_ t |j|j|_t |j|j|_t|j|_t|j|_dS)Nrrr)r|r}r]chunk_size_feed_forward seq_len_dimrr$rrr#ALBERT_ATTENTION_CLASSES_attn_implementationr!rintermediate_sizer ffn_outputr hidden_act activationrrrrrrmrnr}s  zAlbertLayer.__init__NFrrrroutput_hidden_statesrcCsL|||||}t|j|j|j|d}|||d}|f|ddS)Nrr)r!rff_chunkrrr#)rrrrrrrrrmrmrnrszAlbertLayer.forwardrcCs"||}||}||}|Sr)rrr)rrrrmrmrnr s   zAlbertLayer.ff_chunkNNFF)rrrrr}rYrrrrrrr rrmrmrrnrs( rcsteZdZdeffdd Z    ddejdeejdeejd e d e d e e eje ejfd ff d dZ Z S)AlbertLayerGroupr]cs.ttfddtjD|_dS)Ncg|]}tqSrm)r.0rr]rmrn z-AlbertLayerGroup.__init__..)r|r}r ModuleListrangeinner_group_num albert_layersrrrrnr}s $zAlbertLayerGroup.__init__NFrrrrrr.c Cs|d}d}t|jD]!\}} | |||||} | d}|r#|| df}|r*||f}q |f} |r5| |f} |r<| |f} | S)Nrmrr) enumerater) rrrrrrlayer_hidden_stateslayer_attentions layer_index albert_layer layer_outputoutputsrmrmrnrs    zAlbertLayerGroup.forwardr )rrrrr}rYrrrrrrrrrmrmrrnr s&r csjeZdZdeffdd Z     ddejdeejd eejd e d e d e d e e e ffddZ ZS)AlbertTransformerr]csFt|_tjj|_tfddt j D|_ dS)Ncr rm)r r rrmrnrrz.AlbertTransformer.__init__..) r|r}r]rrrrembedding_hidden_mapping_inrrnum_hidden_groupsalbert_layer_groupsrrrrnr}s $zAlbertTransformer.__init__NFTrrrrr return_dictrc Cs||}|r |fnd}|rdnd}|durdg|jjn|}t|jjD]@} t|jj|jj} t| |jj|jj} |j| |||| | | d| ||} | d}|r^|| d}|re||f}q%|sttdd|||fDSt|||dS)Nrmrrrucss|] }|dur|VqdSrrm)rvrmrmrn sz,AlbertTransformer.forward..)last_hidden_stater attentions) rr]num_hidden_layersrrTrr rr ) rrrrrrr!all_hidden_statesall_attentionsilayers_per_group group_idxlayer_group_outputrmrmrnrs2    zAlbertTransformer.forward)NNFFT)rrrrr}rYrrrrrr rrrrmrmrrnrs,  rc@s$eZdZeZeZdZdZddZ dS)AlbertPreTrainedModelalbertTcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSt|tre|jj dSdS)zInitialize the weights.r)meanstdN?) isinstancerrr/r[normal_r]initializer_ranger8zero_r~rqr$fill_ AlbertMLMHead)rmodulermrmrn _init_weights)s       z#AlbertPreTrainedModel._init_weightsN) rrrr config_classroload_tf_weightsbase_model_prefix_supports_sdpar9rmrmrmrnr-"s  r-z2 Output type of [`AlbertForPreTraining`]. ) custom_introc@steZdZUdZdZeejed<dZ eejed<dZ eejed<dZ ee ejed<dZ ee ejed<dS)AlbertForPreTrainingOutputa loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). sop_logits (`torch.FloatTensor` of shape `(batch_size, 2)`): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). Nlossprediction_logits sop_logitsrr%)rrrrr@rrYr__annotations__rArBrrr%rmrmrmrnr?<s  r?cseZdZeZdZddedeffdd Zdej fdd Z d ej dd fd d Z de e ee fdd fddZe ddeejdeejdeejdeejdeejdeejdeedeedeedeeeffddZZS) AlbertModelr.Tr]add_pooling_layercspt|||_t||_t||_|r$t|j |j |_ t |_ nd|_ d|_ |j |_|j|_|dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N)r|r}r]rpr(rencoderrrrr&Tanhpooler_activationrattn_implementationrw post_init)rr]rErrmrnr}Zs     zAlbertModel.__init__rcC|jjSrr(rrrmrmrnget_input_embeddingsqz AlbertModel.get_input_embeddingsrNcC ||j_dSrrL)rrrmrmrnset_input_embeddingst z AlbertModel.set_input_embeddingsheads_to_prunecCsT|D]#\}}t||jj}t|||jj}|jj|j|j|qdS)a Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} ALBERT has a different architecture in that its layers are shared across groups, which then has inner groups. If an ALBERT model has 12 hidden layers and 2 hidden groups, with two inner groups, there is a total of 4 different layers. These layers are flattened: the indices [0,1] correspond to the two inner groups of the first hidden layer, while [2,3] correspond to the two inner groups of the second hidden layer. Any layer with in index other than [0,1,2,3] will result in an error. See base class PreTrainedModel for more information about head pruning N) itemsrTr]rrFr rr!r)rrSlayerrr+inner_group_idxrmrmrn _prune_headsws  zAlbertModel._prune_headsrrryrtrrrrr!c Cs|dur|n|jj}|dur|n|jj}| dur| n|jj} |dur*|dur*td|dur9||||} n|durF|dd} ntd| \} } |durU|jn|j} |durctj | | d}|durt |j dr|j j ddd| f}| | | }|}n tj| tj| d}|j ||||d}|jdko|jd ko|duo| }|rt||j| d }n|d d }|j|jd }d|t|jj}|||jj}|j|||||| d}|d}|jdur|||dddfnd}| s||f|d dSt|||j|jdS)NzDYou cannot specify both input_ids and inputs_embeds at the same timeruz5You have to specify either input_ids or inputs_embeds)rryr)rtryrrrx)tgt_lenrr;rzr1)rrrr!r)r$ pooler_outputrr%) r]rruse_return_dictrW%warn_if_padding_and_no_attention_maskrrrYonesrr(ryrrrrIrwr r{ unsqueezerfinfomin get_head_maskr&rFr&rHr rr%)rrrryrtrrrrr!rrrrrrembedding_outputuse_sdpa_attention_maskextended_attention_maskencoder_outputssequence_output pooled_outputrmrmrnrst       *zAlbertModel.forward)T) NNNNNNNNN)rrrrr:r<rr}rr~rNrQdictrTrrWrrrYrrrr rrrrmrmrrnrDUsL   rDz Albert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `sentence order prediction (classification)` head. cseZdZddgZdeffdd ZdejfddZd ejdd fd d Z dej fd dZ e dde ejde ejde ejde ejde ejde ejde ejde ejde ede ede edeeeffddZZS)AlbertForPreTrainingpredictions.decoder.biaspredictions.decoder.weightr]cs6t|t||_t||_t||_|dSr) r|r}rDr.r7r' AlbertSOPHeadsop_classifierrJrrrmrnr}s     zAlbertForPreTraining.__init__rcCrKrr'decoderrMrmrmrnget_output_embeddingsrOz*AlbertForPreTraining.get_output_embeddingsnew_embeddingsNcCrPrrmrrprmrmrnset_output_embeddingsrRz*AlbertForPreTraining.set_output_embeddingscC |jjjSrr.r(rrMrmrmrnrN z)AlbertForPreTraining.get_input_embeddingsrrryrtrrlabelssentence_order_labelrrr!c  Cs| dur| n|jj} |j||||||| | | d } | dd\} }|| }||}d}|durU|durUt}||d|jj|d}||dd|d}||}| sl||f| dd}|durj|f|S|St|||| j | j dS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` sentence_order_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`. `0` indicates original order (sequence A, then sequence B), `1` indicates switched order (sequence B, then sequence A). Example: ```python >>> from transformers import AutoTokenizer, AlbertForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2") >>> model = AlbertForPreTraining.from_pretrained("albert/albert-base-v2") >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) >>> # Batch size 1 >>> outputs = model(input_ids) >>> prediction_logits = outputs.prediction_logits >>> sop_logits = outputs.sop_logits ```Nrryrtrrrrr!r;ru)r@rArBrr%) r]rZr.r'rlrrrr?rr%)rrrryrtrrrvrwrrr!rrerfprediction_scores sop_scores total_lossloss_fctmasked_lm_losssentence_order_lossoutputrmrmrnrs>)  zAlbertForPreTraining.forward NNNNNNNNNNN)rrr_tied_weights_keysrr}rrrorrr~rNrrrYrrrrr?rrrrmrmrrnrhsV      rhcsBeZdZdeffdd ZdejdejfddZd d d ZZ S) r7r]cspttj|j|jd|_tt|j |_ t |j |j|_ t |j|j |_t|j|_|j |j_ dS)Nrr)r|r}rr$rr ParameterrYrrr8rrrrnr rrrrrmrnr}Ms  zAlbertMLMHead.__init__rrcCs0||}||}||}||}|}|Sr)rrr$rn)rrryrmrmrnrWs    zAlbertMLMHead.forwardNcCs,|jjjjdkr|j|j_dS|jj|_dS)Nmeta)rnr8rrrMrmrmrn _tie_weightsaszAlbertMLMHead._tie_weights)rN) rrrrr}rYrrrrrmrmrrnr7Ls  r7cs8eZdZdeffdd ZdejdejfddZZS)rkr]cs.tt|j|_t|j|j|_ dSr) r|r}rrclassifier_dropout_probrrr num_labelsr:rrrmrnr}ks zAlbertSOPHead.__init__rfrcCs||}||}|Sr)rr:)rrfdropout_pooled_outputlogitsrmrmrnrqs  zAlbertSOPHead.forward) rrrrr}rYrrrrmrmrrnrkjsrkcseZdZddgZfddZdejfddZdejdd fd d Zdej fd d Z e dde e jde e jde e jde e jde e jde e jde e jde ede ede edeeeffddZZS)AlbertForMaskedLMrirjcs0t|t|dd|_t||_|dSNF)rE)r|r}rDr.r7r'rJrrrmrnr}{s   zAlbertForMaskedLM.__init__rcCrKrrmrMrmrmrnrorOz'AlbertForMaskedLM.get_output_embeddingsrpNcCs||j_|j|j_dSr)r'rnr8rqrmrmrnrrsz'AlbertForMaskedLM.set_output_embeddingscCrsrrtrMrmrmrnrNruz&AlbertForMaskedLM.get_input_embeddingsrrryrtrrrvrrr!c  Cs| dur| n|jj} |j|||||||| | d } | d} || } d}|dur8t}|| d|jj|d}| sN| f| dd}|durL|f|S|St|| | j| j dS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> import torch >>> from transformers import AutoTokenizer, AlbertForMaskedLM >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2") >>> model = AlbertForMaskedLM.from_pretrained("albert/albert-base-v2") >>> # add mask_token >>> inputs = tokenizer("The capital of [MASK] is Paris.", return_tensors="pt") >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> # retrieve index of [MASK] >>> mask_token_index = (inputs.input_ids == tokenizer.mask_token_id)[0].nonzero(as_tuple=True)[0] >>> predicted_token_id = logits[0, mask_token_index].argmax(axis=-1) >>> tokenizer.decode(predicted_token_id) 'france' ``` ```python >>> labels = tokenizer("The capital of France is Paris.", return_tensors="pt")["input_ids"] >>> labels = torch.where(inputs.input_ids == tokenizer.mask_token_id, labels, -100) >>> outputs = model(**inputs, labels=labels) >>> round(outputs.loss.item(), 2) 0.81 ``` N rrryrtrrrrr!rrur;r@rrr%) r]rZr.r'rrrrrr%)rrrryrtrrrvrrr!rsequence_outputsryr}r|rrmrmrnrs61 zAlbertForMaskedLM.forward NNNNNNNNNN)rrrrr}rrrorrr~rNrrrYrrrrrrrrrmrmrrnrwsP      rz Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ceZdZdeffdd Ze          ddeejdeej deejdeejd eej d eej d eejd ee d ee dee de e e ffddZZS)AlbertForSequenceClassificationr]csRt||j|_||_t||_t|j|_ t |j |jj|_ | dSr)r|r}rr]rDr.rrrrrrr:rJrrrmrnr}s   z(AlbertForSequenceClassification.__init__Nrrryrtrrrvrrr!rc  Csr| dur| n|jj} |j|||||||| | d } | d} || } || } d}|dur|jjdurV|jdkr 1` a classification loss is computed (Cross-Entropy). Nrr regressionsingle_label_classificationmulti_label_classificationrur;r)r]rZr.rr: problem_typerr{rYrrTrsqueezerrrrrr%)rrrryrtrrrvrrr!rrfrr@r|rrmrmrnrsV     "       z'AlbertForSequenceClassification.forwardr)rrrrr}rrrYrrrrrrrrrmrmrrnrsH     rcr)AlbertForTokenClassificationr]csdt||j|_t|dd|_|jdur|jn|j}t||_ t |j |j j|_ |dSr)r|r}rrDr.rrrrrrrr]r:rJ)rr]rrrmrnr}@s    z%AlbertForTokenClassification.__init__Nrrryrtrrrvrrr!rc  Cs| dur| n|jj} |j|||||||| | d } | d} || } || } d}|dursH    rcseZdZdeffdd Ze           ddeejdeej deejdeejd eej d eej d eejd eejd ee dee dee de e e ffddZZS)AlbertForQuestionAnsweringr]cs@t||j|_t|dd|_t|j|j|_| dSr) r|r}rrDr.rrr qa_outputsrJrrrmrnr}s  z#AlbertForQuestionAnswering.__init__Nrrryrtrrstart_positions end_positionsrrr!rc  CsH| dur| n|jj} |j||||||| | | d } | d} || }|jddd\}}|d}|d}d}|dur|durt|dkrO|d}t|dkr\|d}|d}| d|}| d|}t |d}|||}|||}||d}| s||f| dd}|dur|f|S|St |||| j | j dS) Nrrrrur) ignore_indexr;)r@ start_logits end_logitsrr%)r]rZr.rrOrrrNrclamprrrr%)rrrryrtrrrrrrr!rrerrrr{ ignored_indexr| start_lossend_lossrrmrmrnrsP          z"AlbertForQuestionAnswering.forwardrrrrrr}rrrYrrrrr?rrrrmrmrrnrsN      rcr)AlbertForMultipleChoicer]cs@t|t||_t|j|_t|j d|_ | dS)Nr) r|r}rDr.rrrrrrr:rJrrrmrnr}s   z AlbertForMultipleChoice.__init__Nrrryrtrrrvrrr!rc  Csn| dur| n|jj} |dur|jdn|jd} |dur%|d|dnd}|dur4|d|dnd}|durC|d|dnd}|durR|d|dnd}|dure|d|d|dnd}|j|||||||| | d } | d} || } || }|d| }d}|durt}|||}| s|f| dd}|dur|f|S|St ||| j | j dS)a[ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above) Nrrurrxr;r) r]rZrVrrr.rr:rrrr%)rrrryrtrrrvrrr! num_choicesrrfrreshaped_logitsr@r|rrmrmrnrsL,    zAlbertForMultipleChoice.forwardrrrmrmrrnrsH     r) ror-rDrhrrrrr)?rrrC dataclassesrtypingrrrYrtorch.nnrrr activationsr modeling_attn_mask_utilsr modeling_outputsr r rrrrrmodeling_utilsr pytorch_utilsrrrrutilsrrrconfiguration_albertr get_loggerrrAroModulerprrrrr rr-r?rDrhr7rkrrrrr__all__rmrmrmrnsn    $   ~Dp7)#5 f iWGM f