o
    ߥi?q                     @   s
  d dl Zd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZ d dlmZ d dlmZ e Z ddgZ!dddZ"G dd deeZ#G dd de#Z$G dd deZ%dS )    N)ABC)OrderedDict)AnyDictOptional)nn)
TorchModel)build_backbone
build_head)
OutputKeys)load_task_model_checkpoint)Config
ConfigDict)DEFAULT_MODEL_REVISION	ModelFile)func_receive_dict_inputs)
get_loggerEncoderModelSingleBackboneTaskModelBase   c                 C   s   |dkr|   S g }|  }|r|d}g }dd }| j D ]\}}t||d }||d}|d| d |  q || }	|   d }
|	rbt|dkrW|sW|
|d 7 }
n|
d	d	|	 d 7 }
|
d
7 }
|
S )Nr   
c                    sP   |  d}t|dkr| S |d} fdd|D }d|}|d | }|S )Nr   r   r   c                    s   g | ]} d  | qS )  ).0line	numSpacesr   `/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/nlp/task_models/task_model.py
<listcomp>+   s    z-_repr.<locals>._addindent.<locals>.<listcomp>)splitlenpopjoin)s_r   sfirstr   r   r   
_addindent%   s   


z_repr.<locals>._addindentr      (z): z
  ))		_get_name
extra_reprr   _modulesitems_reprappendr    r"   )modulesdepthextra_linesr+   child_linesr&   keymodulemod_strlinesmain_strr   r   r   r.      s(   

r.   c                       s   e Zd ZdZdZdZdZdef fddZdd Z	e
dd	 Zd
eeef deeef fddZ		dddZdd Z			dddZ  ZS )BaseTaskModelz( Base task model interface for nlp

    N	model_dirc                    s(   t  j|g|R i | t|| _d S N)super__init__r   configselfr:   argskwargs	__class__r   r   r=   M   s   zBaseTaskModel.__init__c                 C      d}t | |S Nr   r.   r@   r1   r   r   r   __repr__Q      
zBaseTaskModel.__repr__c                 K   s0   | d}| di |}|jdd|i| |S )Nr:   model_local_dirr   )getload_checkpoint)clsrB   r:   modelr   r   r   _instantiateV   s   
zBaseTaskModel._instantiateinputreturnc                 C   s   d S r;   r   )r@   rQ   r   r   r   forward]   s   zBaseTaskModel.forwardc                 K   sX   t j|tj}tj|dd}|durt| | j||ddd\}}}	}
|||	|
dS )ak  
        Load model checkpoint file and feed the parameters into the model.
        Args:
            model_local_dir: The actual checkpoint dir on local disk.
            default_dtype: Set the default float type by 'torch.set_default_dtype'
            load_state_fn: An optional load_state_fn used to load state_dict into the model.

        Returns:

        cpu)map_locationNT)load_state_fnignore_mismatched_sizes
_fast_init)missing_keysunexpected_keysmismatched_keys
error_msgs)	ospathr"   r   TORCH_MODEL_BIN_FILEtorchloadset_default_dtype_load_checkpoint)r@   rK   default_dtyperV   rB   	ckpt_file
state_dictrY   rZ   r[   r\   r   r   r   rM   `   s"   
zBaseTaskModel.load_checkpointc                    s  |   }| jt }| D ]\}}|s&|ds&||dd|g< q|||< q|}dd | D }	t| }
dd  |	} fdd|	D }	tdkrht	fd	d
|	D }t	fdd
|
D }nd}d}| op|}|ou| }|rfdd|
D fdd|
D }
n|rfdd|
D }
tt
|
t
|	 }tt
|	t
|
 }| jd ur| jD ]fdd|D }q| jd ur| jD ]fdd|D }q|r| j|||d}|D ]}| | qd}| }tdkrt| s|rd }tdkr!t| r!|s!t| }t	fdd
|	D r!tdfdd}fdd}|||||||}||||}t|dkrTd|}td| jj d| t|dkrttd| jj d| d| jj d | jj d!	 ntd"| jj d# t|dkrtd$| jj d%| d& nt|dkrtd'| jj d(| jj d) t|dkrd*d+d |D }td$| jj d,| d& ||||fS )-Nhead.c                 S   s   g | ]}|qS r   r   r   kr   r   r   r      s    z2BaseTaskModel._load_checkpoint.<locals>.<listcomp>c                 S   s,   d| v r
|  ddS d| v r|  ddS | S )Nbetabiasgammaweight)replace)r4   r   r   r   _fix_key   s
   z0BaseTaskModel._load_checkpoint.<locals>._fix_keyc                    s   g | ]} |qS r   r   r   r4   )rp   r   r   r      s    r   c                 3       | ]}|  V  qd S r;   
startswithr   r$   prefixr   r   	<genexpr>   s    z1BaseTaskModel._load_checkpoint.<locals>.<genexpr>c                 3   rr   r;   rs   ru   rv   r   r   rx      s    

Fc                    s   g | ]	}|  s|qS r   rs   ru   rv   r   r   r      s
    
c                    s2   g | ]}|  rd |d dd n|qS )rh   r   N)rt   r"   r   ru   rv   r   r   r      s    $c                    s   g | ]	}d   |gqS )rh   )r"   ru   rv   r   r   r      s    c                        g | ]}t  |d u r|qS r;   researchri   patr   r   r          c                    ry   r;   rz   ri   r}   r   r   r      r   )rw   
add_prefixremove_prefix c                 3   s    | ]}| v V  qd S r;   r   rq   )expected_keys_not_prefixedr   r   rx      s    zjThe state dictionary of the model you are trying to load is corrupted. Are you sure it was properly saved?c                    s   g }|rF|D ]?}|}|r  d| }n|r"d |ddd  }||v rE|| j}	| | j}
|
|	krE||| | j|| jf | |= q|S )Nrh   r   )r"   r   shaper/   )rf   model_state_dictloaded_keysadd_prefix_to_modelremove_prefix_from_modelrW   r[   checkpoint_key	model_keymodel_shapecheckpoint_shaperv   r   r   _find_mismatched_keys   s*   

z=BaseTaskModel._load_checkpoint.<locals>._find_mismatched_keysc                    s   g }g }  D ]$}d }d|v r|dd}d|v r |dd}|r,|| || qt||D ]\}}||< q2tdd  d urO_g  d ur`| |d  d  S ddtj	f fd	d
| |d  S )Nrm   rn   rk   rl   	_metadata)rw   local_metadatar\   r   r5   c                    sn   d u ri n	 |d d i }||dg g  f}| j|  | j D ]\}}|d ur4||| d  q#d S )NTrh   )rL   _load_from_state_dictr,   r-   )r5   rw   r   rA   namechildr\   ra   metadatarf   r   r   ra   "  s   
zQBaseTaskModel._load_checkpoint.<locals>._load_state_dict_into_model.<locals>.loadrv   )r   )
keysro   r/   zipr!   getattrcopyr   r   Module)model_to_loadrf   start_prefixold_keysnew_keysr4   new_keyold_key)rV   r   r   _load_state_dict_into_model   s>   


zCBaseTaskModel._load_checkpoint.<locals>._load_state_dict_into_modelz
	z#Error(s) in loading state_dict for z:
	zESome weights of the model checkpoint were not used when initializing z: z,
- This IS expected if you are initializing z from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForTokenClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing z from the checkpoint of a model that you expect to be exactly identical (initializing a BertForTokenClassification model from a BertForTokenClassification model).z9All model checkpoint weights were used when initializing z.
zSome weights of zK were not initialized from the model checkpoint and are newly initialized: zo
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.zAll the weights of z were initialized from the model checkpoint If your task is similar to the task the model of the checkpoint was trained on, you can already use z* for predictions without further training.r   c              	   S   s*   g | ]\}}}d | d| d| dqS )z- z: found shape z in the checkpoint and z in the model instantiatedr   )r   r4   shape1shape2r   r   r   r   ]  s    zl were not initialized from the model checkpoint and are newly initialized because the shapes did not match:
)rf   _backbone_prefixr   r-   rt   r"   r   listr    anyset_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpectedretrieve_modules_from_names_init_weightshasattrr   
ValueErrorRuntimeErrorrD   __name__loggerwarninginfo)r@   rf   rV   rW   rX   r   new_state_dictr   r5   r   expected_keysoriginal_loaded_keyshas_prefix_moduleexpects_prefix_moduler   r   rY   rZ   uninitialized_modulesr   r   r   r   r[   r\   	error_msgmismatched_warningr   )rp   r   rV   r~   rw   r   rc      s   











 
1


zBaseTaskModel._load_checkpointFc           	      C   s   t dd |D }|t dd |D }g }|  D ]4\}}|r5||r2d|ddd  n|}n|rFt|dkrDd||gn|}||v rO|| q|S )Nc                 S   s$   g | ]}d  |d dd qS )rh   Nr   )r"   r   rq   r   r   r   r   n  s   $ z=BaseTaskModel.retrieve_modules_from_names.<locals>.<listcomp>c                 S   s0   g | ]}|d    rd|ddd qS )r   rh   N)isdigitr"   r   rq   r   r   r   r   s  s    
rh   r   r   )r   unionnamed_modulesrt   r"   r   r    r/   )	r@   namesrw   r   r   module_keysretrieved_modulesr   r5   r   r   r   r   i  s*   
z)BaseTaskModel.retrieve_modules_from_names)NN)NFF)r   
__module____qualname____doc__r   r   r   strr=   rI   classmethodrP   r   r   rS   rM   rc   r   __classcell__r   r   rC   r   r9   B   s&    
"
$ hr9   c                       s   e Zd ZdZdZdZdef fddZdd Zd	d
 Z	e
 fddZe
 fddZdeeef deeef fddZdd Zdd Z  ZS )r   zI
    This is the base class of any single backbone nlp task classes.
    bertrg   r:   c                    sL   t  j|g|R i | | jdd | _| jd usJ | jdd | _d S )Nbackbonerg   )r<   r=   r>   rL   backbone_cfghead_cfgr?   rC   r   r   r=     s   z$SingleBackboneTaskModelBase.__init__c                 C   s.   d|v r	|d | _ t|}t| |d | d S )Nrw   )r   r	   setattrr@   cfgr   r   r   r   r	     s   
z*SingleBackboneTaskModelBase.build_backbonec                 C   sB   |d u rt dd|v r|d | _t|| jd}t| | j| |S )N?Head config is missing, check if this was a backbone-only modelrw   	task_name)r   _head_prefixr
   	group_keyr   r@   r   rg   r   r   r   r
     s   
z&SingleBackboneTaskModelBase.build_headc                    "   d| j krt| | j S t dS )Nr   )r   r   r<   __getattr__r@   rC   r   r   r        
z$SingleBackboneTaskModelBase.backbonec                    r   Nrg   )r   r   r<   r   r   rC   r   r   rg     r   z SingleBackboneTaskModelBase.headrQ   rR   c                 C   2   t | jjr| j|}|S | jjdi |}|S z3default forward method is the backbone-only forwardNr   )r   r   rS   r@   rQ   outputsr   r   r   rS     
   z#SingleBackboneTaskModelBase.forwardc                 C   s   | j ||}|S r;   )rg   compute_loss)r@   r   labelslossr   r   r   r     s   z(SingleBackboneTaskModelBase.compute_lossc                 C   s@   d }d }t | jdr| j|}t | jdr| j|}||fS )Nextract_sequence_outputsextract_pooled_outputs)r   r   r   r   )r@   r   sequence_outputpooled_outputr   r   r   extract_backbone_outputs  s   z4SingleBackboneTaskModelBase.extract_backbone_outputs)r   r   r   r   r   r   r   r=   r	   r
   propertyr   rg   r   r   rS   r   r   r   r   r   rC   r   r     s    "c                
       s   e Zd ZdZdZdZdZdZdZdZ	dZ
dZdef fddZd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Ze fddZe fddZdeeef deeef fdd Z										d+d!d"Zed#d$ Zeeddfd%ed&e e d'e!d(ef fd)d*Z"  Z#S ),r   zA
    This is the base class of any encoder nlp task classes.
    Nencoderrg   Fr:   c                    s^   t  j|g|R i | t|| _|  }|  }| | |jd ur)| | | 	  d S r;   )
r<   r=   r   r>   parse_encoder_cfgparse_head_cfgbuild_encodertyper
   	post_init)r@   r:   rA   rB   r   r   rC   r   r   r=     s   



zEncoderModel.__init__c                 C   sL   zt | jd}|D ]
}| jd|  q	W d S  ty%   td Y d S w )Nr   zhead.z+head has no _keys_to_ignore_on_load_missing)r   rg   r   r/   	Exceptionr   r   )r@   #head_keys_to_ignore_on_load_missingir   r   r   r     s   zEncoderModel.post_initc                 C   rE   rF   rG   rH   r   r   r   rI     rJ   zEncoderModel.__repr__c                 C   s4   t j| jtj}d }t j|rt|}|	 S r;   )
r]   r^   r"   r:   r   CONFIGexistsr   	from_filer   )r@   transformer_config_filetransformer_configr   r   r   _get_transformer_config  s   

z$EncoderModel._get_transformer_configc                 C   s   d|vr
d|vr
dS dS )N
model_typer   TFr   )r@   r   r   r   r   _use_transformer_config  s   z$EncoderModel._use_transformer_configc                 C   s   | j dd }|d u r*| j  }d|v rd|vr|j|_n| |r*|  }|j|_d|vs1| jr5| j|_|jd u r>t	d| j
|_
|S )Nr   r   r   zOMissing encoder type, please explicit define encoder type in configuration.json)r>   rL   r   r   r   r   r   override_base_model_typebase_model_typeKeyErrorr:   )r@   encoder_cfgr   r   r   r     s    



zEncoderModel.parse_encoder_cfgc                 C   sh   | j dd }|d u r*| j  }d|v rd|vr|j|_n| |r*|  }| j|_d|vr2| j|_|S )Nrg   	head_typer   )r>   rL   r   r   r   r   r   )r@   r   r   r   r   r     s   


zEncoderModel.parse_head_cfgc                 C   sZ   t |}d|v r|d | _nd|v r|d | _nt|dr$| js$|j| _t| | j| d S )Nrw   base_model_prefix)r	   r   r   override_base_model_prefixr   r   r   r   r   r   &  s   
zEncoderModel.build_encoderc                 C   s0   |d u rt dt|| jd}t| | j| d S )Nr   r   )r   r
   r   r   head_prefixr   r   r   r   r
   1  s   zEncoderModel.build_headc                    r   )Nr   )r   r   r<   r   r   rC   r   r   r   9  r   zEncoderModel.encoderc                    r   r   )r   r   r<   r   r   rC   r   r   rg   ?  r   zEncoderModel.headrQ   rR   c                 K   r   r   )r   r   rS   r   r   r   r   extract_featureE  r   zEncoderModel.extract_featurec                 O   sX   t j|v r|du r|t jd}| j||||||||	|
d	}| jj|||fi |}|S )a  
        Args:
        input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using :class:`~modelscope.models.nlp.structbert.SbertTokenizer`. See
            :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for
            details.

        attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0,
            1]``:

            - 0 corresponds to a `sentence A` token,
            - 1 corresponds to a `sentence B` token.

        position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0,
            config.max_position_embeddings - 1]``.

        head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
            Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert :obj:`input_ids` indices into associated
            vectors than the model's internal embedding lookup matrix.
        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):
            Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ...,
            config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
         output_attentions (:obj:`bool`, `optional`):
            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
            tensors for more detail.
        output_hidden_states (:obj:`bool`, `optional`):
            Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
            more detail.
        return_dict (:obj:`bool`, `optional`):
            Whether or not to return a :class:`~transformers.ModelOutput` instead of a plain tuple.
        *args:
            In Torch 1.11 onnx has a bug in the _slow_forward method, could only keep *args solving the problem
        **kwargs:
            Accept additional kwargs in the children class

        Returns:
            Returns `modelscope.outputs.ModelOutput`

        Examples:
            >>> from modelscope.models import Model
            >>> from modelscope.preprocessors import Preprocessor
            >>> model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
            >>> preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
            >>> print(model(**preprocessor(('This is a test', 'This is also a test'))))
        N)		input_idsattention_masktoken_type_idsposition_ids	head_maskinputs_embedsoutput_attentionsoutput_hidden_statesreturn_dict)r   LABELr!   r  rg   rS   )r@   r  r  r  r  r  r  r   r  r	  r
  rA   rB   featurer   r   r   r   rS   M  s   MzEncoderModel.forwardc                 K   s4   | d}| di |}td||d|}|d S )Nr:   )r   rK   rO   r   )rL   r   )rN   rB   r:   rO   model_load_handlerr   r   r   rP     s   
zEncoderModel._instantiatemodel_name_or_pathrevisioncfg_dictdevicec                    s<   | dd }tt| jd|||||d ur|n| jd|S )Ntask)r  r  r  devicr  r   )r!   r<   r   from_pretrainedr  )rN   r  r  r  r  rB   r  rC   r   r   r    s   zEncoderModel.from_pretrained)
NNNNNNNNNN)$r   r   r   r   r   r   r   r   r   r   r   r   r   r=   r   rI   r   r   r   r   r   r
   r   r   rg   r   r   r  rS   r   rP   r   r   r   r  r   r   r   rC   r   r     sd    
	"	
^
)r   )&os.pathr]   r{   abcr   collectionsr   typingr   r   r   r`   r   modelscope.models.baser   modelscope.models.builderr	   r
   modelscope.outputsr   modelscope.utils.checkpointr   modelscope.utils.configr   r   modelscope.utils.constantr   r   modelscope.utils.file_utilsr   modelscope.utils.loggerr   r   __all__r.   r9   r   r   r   r   r   r   <module>   s.   
)  GC