o ߥi/A@s6ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl mZddlmZddlmZddlmZddlmZddlmZmZddlmZmZdd lmZmZm Z dd l!m"Z"e"Z#d gZ$Gd d d e%Z&GdddeZ'ej(e j)ej*dGdd d eZ+   dddZ,dS)N) roi_align)Models) TorchModel)MODELS)FPNTrans)LayoutRobertaModelLayoutRobertaPreTrainedModel)TransformerDecoderTransformerDecoderLayer)ModeKeys ModelFileTasks) get_loggerVLDocForDocVLEmbeddingc@seZdZddZdS)GeoVLDocModelOutputscCs"||_||_||_||_||_dS)N text_featurestext_mm_featuresblock_vis_featuresblock_vis_mm_featuresimage_mm_features)selfrrrrrr]/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/multi_modal/vldoc/model.py__init__"s   zGeoVLDocModelOutputs.__init__N)__name__ __module__ __qualname__rrrrrr s rcsZeZdZd fdd ZdefddZ                  d dd ZZS) GeoVLDocModelFcst|||_||_t|jddr'|jjddkr!t||_n t||_nt||_t|jj dd|_ t ddg|_ t d|jj|_t|jj|jj|jjdd }t|d|_t|jj|jj|jjdd }t|d|_|dS) N architecturesrrF)img_size inner_vitT) self_attn)superrconfighard_negtive_samplinggetattrrr text_encoderr image_sizevisual_encodernnAdaptiveAvgPool2dpoolLinear hidden_size vis_linearr num_attention_headsintermediate_sizer cross_modal_textcross_modal_visual init_weights)rr&r'cross_modal_text_layercross_modal_visual_layer __class__rrr8s<      zGeoVLDocModel.__init__ ckpt_pathcCsFtj|dd}i}|D]\}}|dd}|||<q ||dS)Ncpu map_locationgeo_vl_doc_model.)torchloaditemsreplaceload_state_dict)rr; state_dictstate_dict_newkvrrrfrom_pretrainedZs   zGeoVLDocModel.from_pretrainedNc+ Ks|j\}}|dur |n|jj}||d<|jjddkr/|j|f|||| | | |||d |}n|j|f|||| | | |||d |}|dd\}}|j\}}tjd||jd|d ||}|||f}|j\}}}| |}tjd||jd|d || d}t ||fd||d j |d jd } |d jtjkrt|d  tj| tjd|d dd d }!|!j |d jd }!nt|d | tjd|d dd d }!|!dd||d}!||!}!|!| d}!||d dd}"||" d}"t |"|!fd}#t|df |j}$t |$|fd}%d|dk}&d|%dk}'|j|dd|#dd|&|'d}(|j|#dd|dd|'|&d})|(dd}(|)dd})|)ddddf}*t||(|!|*|)dS)N line_bboxrr) bboxattention_masktoken_type_ids position_ids head_mask inputs_embedsoutput_attentionsoutput_hidden_states return_dict)devicer"feat_ms)dtypeg@@) spatial_scaler#)tgtmemorytgt_key_padding_maskmemory_key_padding_maskr)shaper&use_return_dictrr)rAarangerVreshapeexpandr+ unsqueezecattorZfloat16rfloat32sizesqueezer1r.onesr4 transposer5r)+r input_idsimagerLbbox_4p_normalizedrMfirst_token_idxesfirst_token_idxes_maskrNrOrPrQencoder_hidden_statesencoder_attention_maskpast_key_values use_cacherRrSrTkwargs batch_sizeseq_lenoutputssequence_output pooled_output_ num_first B_batch_dim feature_bbox block_num visual_out batch_idxsbatch_idx_with_bboxblk_vis_featuresfull_img_featuresvis_inps glb_feat_attnvis_masknew_attention_mask new_vis_mask text_mm_feat vis_mm_featrrrrforwardbs                     zGeoVLDocModel.forward)FNNNNNNNNNNNNNNNNNN)rrrrstrrJr __classcell__rrr9rr6s," r) module_namecsTeZdZdZdeffdd Z                  dddZZS) rz Generate multi-modal document embeddings in segment-level and token-level. Args: model_dir: the path in model hub, e.g., 'damo/multi-modal_convnext-roberta-base_vldoc-embedding' model_dirc sFtj|d|i|ddlm}tj|d}td |tj |s(J| ||_ t |j |_tj|tj}tj |sDJ|j|td |ddlm}tj|tj}|||_tjrwd ttjd dnd |_tjr|j|jtd ttjd ddS|jtd dS) Nrr)LayoutRobertaConfigz config.jsonzLoading config file from {}zLoading model from {})VLDocXLMTokenizerzcuda:{} LOCAL_RANKr<z%Use GPU {} for finetuning & inferencez"Use CPU for finetuning & inference)r%r;modelscope.models.multi_modal.vldoc.modeling_layout_robertarospathjoinloggerinfoformatexistsfrom_json_filer&r doc_modelr TORCH_MODEL_FILErJ0modelscope.models.multi_modal.vldoc.tokenizationrTOKENIZER_FOLDER tokenizerrAcuda is_availableintenvirongetrVrgfloat) rrargsrwrmodel_cfg_path model_pathrtokenizer_pathr9rrr s6        zVLDocForDocVLEmbedding.__init__NcKs|jdid|d|d|d|d|d|d|d|d | d | d | d | d | d|d|d|d|d||}t|j|jdS)a Args: - input_ids: :math:`(B, T, E)`, the input tokens, where B is the batch size, T is the max token size, E is the embedding dimension. - image: :math:`(B, C, H, W)`, normalized images. - bbox: :math:`(B, T, 4)`, segment boxes denoted by top-left and bottom-right vertexes whose values are normalized to [0, 1000). - bbox_4p_normalized: :math:`(B, T, 8)`, word boxes denoted by 4 vertexes, whose values are normalized to [0, 1). - attention_mask: :math:`(B, T)`, mask for input tokens, where 0 means masked. - first_token_idxes: :math:`(B, S)`, indexes of the corresponding first tokens of all segments, where S is the max segment size. - first_token_idxes_mask: :math:`(B, S)`, mask for segments, where 0 means masked. Optional: - line_rank_id: :math:`(B, T)`, orders of segments. - line_rank_inner_id: :math:`(B, T)`, BIE-like tags. To be more specific, please refer to the class `TextLayoutSerializer` in `modelscope/models/multi_modal/vldoc/processing.py`. rnrorLrprMrqrrrNrOrPrQrsrtrurvrRrSrT) img_embeddingtext_embeddingNr)rdictrr)rrnrorLrprMrqrrrNrOrPrQrsrtrurvrRrSrTrw vldoc_outputsrrrr*sT )     zVLDocForDocVLEmbedding.forwardr)rrr__doc__rrrrrrr9rrs,"robertac sdur tj|ddg}g}t}|dkrFtt|D]&}||} d} | dr7| dd} t | } | rE| ||| | qt ||D] \} }  | | <qKgggt dddurn_dfdd d} t|d std d Drd } || dtdkrtd|jjtdkrtd|jjtdkrtd|jjd|S)Nr<r=rzroberta.zgeo_vl_doc_model.text_encoder. _metadatar@c shdurin |ddi}|||d|jD]\}}|dur1|||dq dS)NrWT.)r_load_from_state_dict_modulesrC)moduleprefixlocal_metadatanamechild error_msgsrBmetadata missing_keysrFunexpected_keysrrrBs  z$init_pretrained_weight..loadgeo_vl_doc_modelcss|]}|dVqdS)r?N) startswith).0srrr s  z)init_pretrained_weight..r?)rrz7Weights of {} not initialized from pretrained model: {}z0Weights from pretrained model not used in {}: {}z*Error(s) in loading state_dict for {}: {}z )r@)rArBlistkeysrangelenrrDcopydeepcopyappendzippopr(rhasattranyrrrr:r RuntimeErrorr) modelpretrained_model_pathrF cache_dir init_backboneold_keysnew_keysstate_dict_keysikeynew_keyold_key start_prefixrrrinit_pretrained_weightnsf          r)NNr)-rloggingmathrresysjsonrAtorch.distributed distributeddisttorch.nnr,torchvision.opsrmodelscope.metainformodelscope.modelsrmodelscope.models.builderr2modelscope.models.multi_modal.vldoc.conv_fpn_transrrrr5modelscope.models.multi_modal.vldoc.transformer_localr r modelscope.utils.constantr r r modelscope.utils.loggerrr__all__objectrrregister_moduledocument_vl_embeddingvldocrrrrrrs<        Jq