o
    ॵi                     @   s   d dl mZ d dlmZmZmZmZmZmZm	Z	 d dl
Z
d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ e ZeG dd dZej ej!dG dd deZ"dS )    )	dataclass)AnyCallableDictListOptionalTupleUnionN)nn)Dataset)Trainers)
TorchModel)	MsDataset)Preprocessor)TRAINERS)NlpEpochBasedTrainer)DEFAULT_MODEL_REVISION)
get_loggerc                   @   s>   e Zd ZdZdd Zdeeeef  deeef fddZ	dS )	GroupCollatorz
    Wrapper that does conversion from List[Tuple[encode_qry, encode_psg]] to
    List[qry], List[psg] and pass batch separately to the actual collator.
    Abstract out data detail for the model.
    c                 C   s>  g g g }}}g g }}g g g }}	}
d}|D ]v}t |dkr"qt |d dkrO|D ]!\}}}}}}|| || || || || q,qt |d dkrd}|D ]3\	}}}}}}}}}|| || || || || || |	| |
| q[q||||||||	|
|	 |S )NFr      	   T)lenappendupdate)selfgisinpsgis_input_idsgis_token_type_idsgis_rel_type_idsgis_absolute_position_idsgis_relative_position_idsgis_prov_idsgis_city_idsgis_dist_idschina_versiondocgeom_id	geom_typerel_typeabsolute_positionrelative_positionlxlyprov_idcity_iddist_id r0   h/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/trainers/multi_modal/mgeo_ranking_trainer.pyget_gis   sD   












zGroupCollator.get_gisfeaturesreturnc                 C   s  t |d trt|g }|d  }dd |D }|D ]}| D ]\}}|| | q!qi }g }g }	|D ]}d|kr|d d }
| |
|d }
|
jd urh||
j|
j	|
j
|
j|
j|
j|
j|
j|
jd	 n||
j|
j	|
j
|
j|
j|
jd |	tdg|
jj q6d	|kr|d
 d }
| |
|d	 }
|
jd ur||
j|
j	|
j
|
j|
j|
j|
j|
j|
jd	 n||
j|
j	|
j
|
j|
j|
jd |	tdg|
jj q6d|v sd|v rtj|| dd||< q6|ds(dd || D }tjjj|}t| dkr| ||< q6| \}}}||||  ||< q6t|dkr4||d< t|	dkr?|	|d< |S )Nr   c                 S   s   i | ]}|t  qS r0   )list).0kr0   r0   r1   
<dictcomp>C       z*GroupCollator.__call__.<locals>.<dictcomp>sentence1_gisgis1)		input_idsattention_masktoken_type_idsrel_type_idsabsolute_position_idsrelative_position_idsprov_idscity_idsdist_ids)r<   r=   r>   r?   r@   rA      sentence2_gisgis2qidlabels)dimr   c                 S   s   g | ]}|  qS r0   )t)r6   itr0   r0   r1   
<listcomp>   r9   z*GroupCollator.__call__.<locals>.<listcomp>   gis_listgis_tp)
isinstancer5   sumkeysitemsr   r2   rB   r<   r=   r>   r?   r@   rA   rC   rD   torch
LongTensortodevicecat
startswithr
   utilsrnnpad_sequencer   sizerK   view)r   r3   rS   batcheler7   vmerged_batchrO   rP   r   k_tpadlb1b2r0   r0   r1   __call__?   s   




zGroupCollator.__call__N)
__name__
__module____qualname____doc__r2   r   r   strr   ri   r0   r0   r0   r1   r      s    *!r   )module_namec                       s   e Zd Zdddddddddef
deeeeje	f  dee	 dee
 dee
 dee
 deeeef  d	eeeef  d
ee deejjejjjf dee	 f fddZ  ZS )MGeoRankingTrainerN)NNmodelcfg_filecfg_modify_fnarg_parse_fndata_collatortrain_dataseteval_datasetpreprocessor
optimizersmodel_revisionc                    s:   |d u rt  }t jd|||||||	|||
d
| d S )N)
rq   rr   rs   rt   ru   rx   ry   rv   rw   rz   r0   )r   super__init__)r   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   kwargs	__class__r0   r1   r|      s    
zMGeoRankingTrainer.__init__)rj   rk   rl   r   r   r	   r   r
   Modulern   r   r   r   r   r   rU   optim	Optimizerlr_scheduler_LRSchedulerr|   __classcell__r0   r0   r~   r1   rp      sD    	
rp   )#dataclassesr   typingr   r   r   r   r   r   r	   rU   r
   torch.utils.datar   modelscope.metainfor   modelscope.models.baser    modelscope.msdatasets.ms_datasetr   modelscope.preprocessors.baser   modelscope.trainers.builderr   modelscope.trainers.nlp_trainerr   modelscope.utils.constantr   modelscope.utils.loggerr   loggerr   register_modulemgeo_ranking_trainerrp   r0   r0   r0   r1   <module>   s&   $ 