o wi$@s@dZddlZddlZddlmZmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZdd lmZmZmZmZmZmZdd lmZmZmZmZdd lmZmZd d l m!Z!e"e#Z$Gdddej%Z&d9ddZ'Gdddej%Z(Gdddej%Z)Gdddej%Z*Gdddej%Z+Gdddej%Z,Gdddej%Z-Gdddej%Z.Gd d!d!ej%Z/Gd"d#d#ej%Z0d$e0iZ1Gd%d&d&ej%Z2Gd'd(d(eZ3Gd)d*d*ej%Z4eGd+d,d,eZ5eGd-d.d.e5Z6eGd/d0d0e5Z7ed1d2Gd3d4d4e5Z8ed5d2Gd6d7d7e5Z9gd8Z:dS):zPyTorch MarkupLM model.N)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModelapply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging)MarkupLMConfigcs*eZdZdZfddZdddZZS)XPathEmbeddingszConstruct the embeddings from xpath tags and subscripts. We drop tree-id in this version, as its info can be covered by xpath. cstj|_tj|jj|_tj |_ t |_ tj|jdj|_ tdjj|_tfddt|jD|_tfddt|jD|_dS)Ncg|] }tjjqS)r Embeddingmax_xpath_tag_unit_embeddingsxpath_unit_hidden_size.0_configrk/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/transformers/models/markuplm/modeling_markuplm.py Dz,XPathEmbeddings.__init__..crr)rrmax_xpath_subs_unit_embeddingsrr r#rr%r&Kr')super__init__ max_depthrLinearr hidden_sizexpath_unitseq2_embeddingsDropouthidden_dropout_probdropoutReLU activationxpath_unitseq2_inner inner2emb ModuleListrangexpath_tag_sub_embeddingsxpath_subs_sub_embeddingsselfr$ __class__r#r%r*7s"     zXPathEmbeddings.__init__Nc Csg}g}t|jD](}||j||dddd|f||j||dddd|fq tj|dd}tj|dd}||}||| | |}|S)Ndim) r7r+appendr8r9torchcatr5r1r3r4)r;xpath_tags_seqxpath_subs_seqxpath_tags_embeddingsxpath_subs_embeddingsixpath_embeddingsrrr%forwardQs&(zXPathEmbeddings.forward)NN)__name__ __module__ __qualname____doc__r*rJ __classcell__rrr<r%r1s rcCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr?)neintrBcumsumtype_aslong) input_ids padding_idxpast_key_values_lengthmaskincremental_indicesrrr%"create_position_ids_from_input_idsds  rZcs@eZdZdZfddZddZ       d dd ZZS) MarkupLMEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cst||_tj|j|j|jd|_t|j |j|_ |j |_ t ||_ t|j|j|_tj|j|jd|_t|j|_|jdt|j ddd|j|_tj|j |j|jd|_ dS)N)rVeps position_ids)rr>F) persistent)r)r*r$rr vocab_sizer- pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsr+rrItype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsr/r0r1register_bufferrBarangeexpandrVr:r<rr%r*ws    zMarkupLMEmbeddings.__init__cCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr>rdtypedevicer)sizerBrjrVrTrn unsqueezerk)r; inputs_embeds input_shapesequence_lengthr^rrr%&create_position_ids_from_inputs_embedss  z9MarkupLMEmbeddings.create_position_ids_from_inputs_embedsNrcCs<|dur |}n|dd}|dur|jn|j} |dur0|dur+t||j|}n||}|dur=tj|tj| d}|durF||}|dur_|j j tj t t ||jgtj| d}|durx|j jtj t t ||jgtj| d}|} ||} ||} |||} | | | | }||}||}|S)Nr>rl)rornrZrVrtrBzerosrTrbr$ tag_pad_idonestuplelistr+ subs_pad_idrdrfrIrgr1)r;rUrDrEtoken_type_idsr^rqrWrrrnwords_embeddingsrdrfrI embeddingsrrr%rJs8          zMarkupLMEmbeddings.forward)NNNNNNr)rKrLrMrNr*rtrJrOrrr<r%r[ts r[c8eZdZfddZdejdejdejfddZZS)MarkupLMSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNr\) r)r*rr,r-densergrhr/r0r1r:r<rr%r* zMarkupLMSelfOutput.__init__ hidden_states input_tensorreturncC&||}||}|||}|SNrr1rgr;rrrrr%rJ  zMarkupLMSelfOutput.forwardrKrLrMr*rBTensorrJrOrrr<r%r $rc2eZdZfddZdejdejfddZZS)MarkupLMIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) r)r*rr,r-intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr:r<rr%r*s   zMarkupLMIntermediate.__init__rrcC||}||}|Sr)rrr;rrrr%rJ  zMarkupLMIntermediate.forwardrrrr<r%rs rcr~)MarkupLMOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) r)r*rr,rr-rrgrhr/r0r1r:r<rr%r*rzMarkupLMOutput.__init__rrrcCrrrrrrr%rJrzMarkupLMOutput.forwardrrrr<r%rrrcr)MarkupLMPoolercs*tt|j|j|_t|_dSr)r)r*rr,r-rTanhr3r:r<rr%r*s zMarkupLMPooler.__init__rrcCs(|dddf}||}||}|S)Nr)rr3)r;rfirst_token_tensor pooled_outputrrr%rJ s  zMarkupLMPooler.forwardrrrr<r%rs rcr)MarkupLMPredictionHeadTransformcsVtt|j|j|_t|jtrt |j|_ n|j|_ tj |j|j d|_ dSr) r)r*rr,r-rrrrr transform_act_fnrgrhr:r<rr%r*s  z(MarkupLMPredictionHeadTransform.__init__rrcCs"||}||}||}|Sr)rrrgrrrr%rJs   z'MarkupLMPredictionHeadTransform.forwardrrrr<r%rs  rcs,eZdZfddZddZddZZS)MarkupLMLMPredictionHeadcsLtt||_tj|j|jdd|_t t |j|_ |j |j_ dS)NF)bias) r)r*r transformrr,r-r`decoder ParameterrBrurr:r<rr%r*(s  z!MarkupLMLMPredictionHead.__init__cCs|j|j_dSr)rrr;rrr% _tie_weights5sz%MarkupLMLMPredictionHead._tie_weightscCrr)rrrrrr%rJ8rz MarkupLMLMPredictionHead.forward)rKrLrMr*rrJrOrrr<r%r's  rcr)MarkupLMOnlyMLMHeadcstt||_dSr)r)r*r predictionsr:r<rr%r*@s zMarkupLMOnlyMLMHead.__init__sequence_outputrcCs||}|Sr)r)r;rprediction_scoresrrr%rJDs zMarkupLMOnlyMLMHead.forwardrrrr<r%r?s rcseZdZdfdd ZdejdejfddZ      dd ejd eejd eejd eejd eejdee e ejdee de ejfddZ Z S)MarkupLMSelfAttentionNcst|j|jdkrt|dstd|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|p\t|dd|_|jdksh|jd kry|j|_t d |jd |j|_|j|_dS) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsolute relative_keyrelative_key_queryr)r)r*r-num_attention_headshasattr ValueErrorrQattention_head_size all_head_sizerr,querykeyvaluer/attention_probs_dropout_probr1getattrrrcrdistance_embedding is_decoderr;r$rr<rr%r*Ks*   zMarkupLMSelfAttention.__init__xrcCs6|dd|j|jf}||}|ddddS)Nr>rrrr)rorrviewpermute)r;r new_x_shaperrr%transpose_for_scoreses z*MarkupLMSelfAttention.transpose_for_scoresFrattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionscCs||}|du} | r|dur|d} |d} |}nP| r/|||} |||} |}n;|durZ|||} |||} tj|d| gdd} tj|d| gdd} n|||} |||} ||} |du} |jrz| | f}t| | dd}|j dks|j dkr | j d| j d}}| rtj |dtj |j d dd}ntj|tj |j d dd}tj|tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}||}n|j dkr td | |}td | |}|||}|t|j}|dur||}tjj|dd}||}|dur0||}t|| }|dddd }|dd|jf}||}|rX||fn|f}|jrd||f}|S)Nrrrr?r>rrrlrmzbhld,lrd->bhlrzbhrd,lrd->bhlrr) rrrrrBrCrmatmul transposershapetensorrTrnrrjrrctormeinsummathsqrtrr functionalsoftmaxr1r contiguousror)r;rrrrrrrmixed_query_layeris_cross_attention key_layer value_layer query_layer use_cacheattention_scores query_length key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapeoutputsrrr%rJjsn             zMarkupLMSelfAttention.forwardrNNNNNF) rKrLrMr*rBrrr FloatTensorrxboolrJrOrrr<r%rJs4 reagercseZdZdfdd ZddZ      ddejdeejd eejd eejd eejd ee e ejd ee de ejfddZ Z S)MarkupLMAttentionNcs4tt|j||d|_t||_t|_dS)Nr) r)r*MARKUPLM_SELF_ATTENTION_CLASSES_attn_implementationr;routputset pruned_headsrr<rr%r*s   zMarkupLMAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr?)lenrr;rrrrrrrrrrunion)r;headsindexrrr% prune_headss zMarkupLMAttention.prune_headsFrrrrrrrrc Cs<||||||||}||d|} | f|dd} | S)Nrr)r;r) r;rrrrrrr self_outputsattention_outputrrrr%rJs  zMarkupLMAttention.forwardrr) rKrLrMr*rrBrrrrxrrJrOrrr<r%rs4 rcseZdZfddZ      ddejdeejdeejdeejd eejd eeeejd ee d eejfd dZ ddZ Z S) MarkupLMLayercsrt|j|_d|_t||_|j|_|j|_|jr-|js&t|dt|dd|_ t ||_ t ||_ dS)Nrz> should be used as a decoder model if cross attention is addedrr)r)r*chunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionrcrossattentionr intermediaterrr:r<rr%r* s   zMarkupLMLayer.__init__NFrrrrrrrrc Cs|dur |ddnd}|j|||||d} | d} |jr(| dd} | d} n| dd} d} |jro|durot|dsDtd|d|durN|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nr)rrrrr>rz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrrrrrfeed_forward_chunkrr)r;rrrrrrrself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputs layer_outputrrr%rJsP         zMarkupLMLayer.forwardcCs||}|||}|Sr)rr)r;rintermediate_outputr rrr%rZs  z MarkupLMLayer.feed_forward_chunkr) rKrLrMr*rBrrrrxrrJrrOrrr<r%r s4  ArcseZdZfddZ         ddejdeejdeejd eejd eejd eeeejd ee d ee dee dee de eeje ffddZ Z S)MarkupLMEncodercs:t|_tfddtjD|_d|_dS)Ncsg|]}tqSr)rr r#rr%r&esz,MarkupLMEncoder.__init__..F) r)r*r$rr6r7num_hidden_layerslayergradient_checkpointingr:r<r#r%r*bs   zMarkupLMEncoder.__init__NFTrrrrrpast_key_valuesrroutput_hidden_states return_dictrc  Cs8| rdnd} |r dnd} |r|jjrdnd} |jr%|jr%|r%tdd}|r)dnd}t|jD]K\}}| r;| |f} |durC||nd}|durM||nd}||||||||d}|d}|rg||df7}|r{| |df} |jjr{| |df} q0| r| |f} | std d ||| | | fDSt ||| | | d S) NrzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rrrrr>rrcss|] }|dur|VqdSrr)r!vrrr% sz*MarkupLMEncoder.forward..)last_hidden_staterr attentionscross_attentions) r$rrtraininglogger warning_once enumeraterrxr )r;rrrrrrrrrrall_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cacherH layer_modulelayer_head_maskr layer_outputsrrr%rJhsd       zMarkupLMEncoder.forward) NNNNNNFFT)rKrLrMr*rBrrrrxrrr rJrOrrr<r%r asD      r csDeZdZeZdZddZedee e e j fffdd Z ZS)MarkupLMPreTrainedModelmarkuplmcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSt|tre|jj dSdS)zInitialize the weightsg)meanstdN?)rrr,weightdatanormal_r$initializer_rangerzero_rrVrgfill_r)r;modulerrr% _init_weightss       z%MarkupLMPreTrainedModel._init_weightspretrained_model_name_or_pathcstj|g|Ri|Sr)r)from_pretrained)clsr1 model_argskwargsr<rr%r2sz'MarkupLMPreTrainedModel.from_pretrained)rKrLrMr config_classbase_model_prefixr0 classmethodrrrosPathLiker2rOrrr<r%r$s *r$cseZdZdfdd ZddZddZdd Ze dd ee j d ee j d ee j dee j dee j dee j dee j dee j dee dee dee de eeffddZddZZS) MarkupLMModelTcsDt|||_t||_t||_|rt|nd|_| dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) r)r*r$r[r}r encoderrpooler post_init)r;r$add_pooling_layerr<rr%r*s    zMarkupLMModel.__init__cCs|jjSrr}rbrrrr%get_input_embeddingssz"MarkupLMModel.get_input_embeddingscCs ||j_dSrr@)r;rrrr%set_input_embeddingss z"MarkupLMModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsr<rrr)r;heads_to_prunerrrrr% _prune_headsszMarkupLMModel._prune_headsNrUrDrErr{r^rrqrrrrc Cs| dur| n|jj} | dur| n|jj} | dur| n|jj} |dur*|dur*td|dur9||||} n|durF|dd} ntd|durQ|jn|j} |dur_tj | | d}|durltj | tj | d}| d d}|j |jd }d |d }|dur|dkr| d  d  d d}||jjdddd}n|dkr| d d d}|j t|jd }ndg|jj}|j||||||d }|j|||| | | d}|d }|jdur||nd}| s||f|ddSt|||j|j|jdS)a xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Tag IDs for each token in the input sequence, padded up to config.max_depth. xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Subscript IDs for each token in the input sequence, padded up to config.max_depth. Examples: ```python >>> from transformers import AutoProcessor, MarkupLMModel >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = MarkupLMModel.from_pretrained("microsoft/markuplm-base") >>> html_string = " Page Title " >>> encoding = processor(html_string, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 4, 768] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer>z5You have to specify either input_ids or inputs_embeds)rnrlrrrr(gr)rUrDrEr^r{rq)rrrr)r pooler_outputrrr)r$rruse_return_dictr%warn_if_padding_and_no_attention_maskrornrBrwrurTrprrmr@rkrnext parametersr}r<r=r rrr)r;rUrDrErr{r^rrqrrrrrrnextended_attention_maskembedding_outputencoder_outputsrrrrr%rJsn&     zMarkupLMModel.forwardcs.d}|D]}|tfdd|Df7}q|S)Nrc3s$|] }|d|jVqdS)rN) index_selectrrn)r! past_statebeam_idxrr%r`s"z/MarkupLMModel._reorder_cache..)rx)r;rrQreordered_past layer_pastrrPr%_reorder_cache\s zMarkupLMModel._reorder_cache)T) NNNNNNNNNNN)rKrLrMr*rArBrErrrB LongTensorrrrrxr rJrTrOrrr<r%r;sV     kr;c seZdZfddZe             ddeejdeejdeejdeejdeejd eejd eejd eejd eejd eejdeedeedeede e eje ffddZ Z S)MarkupLMForQuestionAnsweringcs@t||j|_t|dd|_t|j|j|_| dSNF)r?) r)r* num_labelsr;r%rr,r- qa_outputsr>r:r<rr%r*hs  z%MarkupLMForQuestionAnswering.__init__NrUrDrErr{r^rrqstart_positions end_positionsrrrrc CsL| dur| n|jj} |j||||||||| | | d }|d}||}|jddd\}}|d}|d}d}| dur| durt| dkrQ| d} t| dkr^| d} |d}| d|| d|t |d}||| }||| }||d}| s||f|dd}|dur|f|S|St ||||j |j d S) ae xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Tag IDs for each token in the input sequence, padded up to config.max_depth. xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Subscript IDs for each token in the input sequence, padded up to config.max_depth. Examples: ```python >>> from transformers import AutoProcessor, MarkupLMForQuestionAnswering >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> model = MarkupLMForQuestionAnswering.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> html_string = " My name is Niels " >>> question = "What's his name?" >>> encoding = processor(html_string, questions=question, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> answer_start_index = outputs.start_logits.argmax() >>> answer_end_index = outputs.end_logits.argmax() >>> predict_answer_tokens = encoding.input_ids[0, answer_start_index : answer_end_index + 1] >>> processor.decode(predict_answer_tokens).strip() 'Niels' ```N rDrErr{r^rrqrrrrrr>r?) ignore_indexr)loss start_logits end_logitsrr)r$rGr%rYsplitsqueezerrroclamp_rrrr)r;rUrDrErr{r^rrqrZr[rrrrrlogitsr_r` total_loss ignored_indexloss_fct start_lossend_lossrrrr%rJrsT/          z$MarkupLMForQuestionAnswering.forward) NNNNNNNNNNNNN)rKrLrMr*rrrBrrrrxrrJrOrrr<r%rVesZ       rVzC MarkupLM Model with a `token_classification` head on top. ) custom_introceZdZfddZe            ddeejdeejdeejdeejdeejd eejd eejd eejd eejd eedeedeede e eje ffddZ Z S)MarkupLMForTokenClassificationcsbt||j|_t|dd|_|jdur|jn|j}t||_ t |j |j|_ | dSrW)r)r*rXr;r%classifier_dropoutr0rr/r1r,r- classifierr>r;r$rmr<rr%r*s   z'MarkupLMForTokenClassification.__init__NrUrDrErr{r^rrqlabelsrrrrc  Cs| dur| n|jj} |j||||||||| | | d } | d}||}d}| dur:t}||d|jj| d}| sP|f| dd}|durN|f|S|St||| j| j dS)a xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Tag IDs for each token in the input sequence, padded up to config.max_depth. xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Subscript IDs for each token in the input sequence, padded up to config.max_depth. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Examples: ```python >>> from transformers import AutoProcessor, AutoModelForTokenClassification >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> processor.parse_html = False >>> model = AutoModelForTokenClassification.from_pretrained("microsoft/markuplm-base", num_labels=7) >>> nodes = ["hello", "world"] >>> xpaths = ["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span"] >>> node_labels = [1, 2] >>> encoding = processor(nodes=nodes, xpaths=xpaths, node_labels=node_labels, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```Nr\rr>rr^rdrr) r$rGr%rnrrrXrrr)r;rUrDrErr{r^rrqrprrrrrrr^rgrrrr%rJs@- z&MarkupLMForTokenClassification.forward NNNNNNNNNNNN)rKrLrMr*rrrBrrrrxr rJrOrrr<r%rlsT      rlz MarkupLM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. crk)!MarkupLMForSequenceClassificationcsdt||j|_||_t||_|jdur|jn|j}t ||_ t |j |j|_ |dSr)r)r*rXr$r;r%rmr0rr/r1r,r-rnr>ror<rr%r*Gs    z*MarkupLMForSequenceClassification.__init__NrUrDrErr{r^rrqrprrrrc  Csv| dur| n|jj} |j||||||||| | | d } | d}||}||}d}| dur|jjdurX|jdkr>d|j_n|jdkrT| jtj ksO| jtj krTd|j_nd|j_|jjdkrvt }|jdkrp|| | }n+||| }n%|jjdkrt }||d|j| d}n|jjdkrt}||| }| s|f| dd}|dur|f|S|St||| j| jd S) a xpath_tags_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Tag IDs for each token in the input sequence, padded up to config.max_depth. xpath_subs_seq (`torch.LongTensor` of shape `(batch_size, sequence_length, config.max_depth)`, *optional*): Subscript IDs for each token in the input sequence, padded up to config.max_depth. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> from transformers import AutoProcessor, AutoModelForSequenceClassification >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = AutoModelForSequenceClassification.from_pretrained("microsoft/markuplm-base", num_labels=7) >>> html_string = " Page Title " >>> encoding = processor(html_string, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```Nr\r regressionsingle_label_classificationmulti_label_classificationr>rrq)r$rGr%r1rn problem_typerXrmrBrTrQrrbrrrrrr)r;rUrDrErr{r^rrqrprrrrrrdr^rgrrrr%rJVsZ,     "       z)MarkupLMForSequenceClassification.forwardrr)rKrLrMr*rrrBrrrrxrrJrOrrr<r%rs?sT      rs)rVrsrlr;r$)r);rNrr9typingrrrBtorch.utils.checkpointrtorch.nnrrr activationsr modeling_layersr modeling_outputsr r r rrrmodeling_utilsrrrrutilsrrconfiguration_markuplmr get_loggerrKrModulerrZr[rrrrrrrrrrrr r$r;rVrlrs__all__rrrr%s`       3c 4WQpd t