o
    ߥi^                     @   s   d dl mZ d dlmZmZ d dlZd dlZd dlm	Z	 d dl
m	  mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ dd	lmZ dd
lmZmZ ejejejdG dd deZ dS )    N)AnyDict)Models)
TorchModel)MODELS)
OutputKeys)	ModelFileTasks   )build_backbone)FPNSegmentorLinearClassifier)module_namec                       sT   e Zd ZdZdef fddZdeeef fddZdd	 Z	d
d Z
dd Z  ZS )VisionMiddlewareModela  
    The implementation of 'ViM: Vision Middleware for Unified Downstream Transferring'.
        This model is dynamically initialized with the following parts:

        - backbone: the upstream pre-trained backbone model (CLIP in this code)
        - ViM: the zoo of middlestream trained ViM modules
        - ViM-aggregation: the specific aggregation weights for downstream tasks
    	model_dirc                    s  t t|   t|tj}tj|dd}|d }|d | _	|d }t
|d |d| _| j  |d }t|}	t|	D ] }
| jjj|
 j||
 d	  | jjj|
 j||
 d
  q<|d }|d }|d D ]/}t|	D ](}
| jjj|
 j||| |
 d | | jjj|
 j||| |
 d | qoqit | _i | _|d D ]S}|d }|drt| jj|| d jd d| j|< n|drt | j|< ntd|| j|  ||  | j|   ||d ! v r|d | | j|< qdS )a  
        Initialize a ViM-based Model.

        Args:
            model_dir: model id or path, where model_dir/pytorch_model.pt contains:

                - 'meta_info': basic information of ViM, e.g. task_list
                - 'backbone_weights': parameters of backbone [upstream]
                - 'ViM_weights': parameters of ViM [midstream]
                - 'ViM_agg_weights': parameters of ViM-aggregation [downstream]

        cpu)map_location	meta_info	task_listbackbone_weightsbackbone_arch)arch
pretrainedViM_weightsvim_att_weightsvim_mlp_weightsViM_agg_weightsViM_agg_algovim_att_aggvim_mlp_agghead_weightsclszclassifier.biasr   )in_channelsnum_classessegzTask type [{}] is not supported	label_mapN)"superr   __init__ospjoinr   TORCH_MODEL_FILEtorchloadr   r   backboneevallenrangetransformer	resblocksvim_attregister_ViMvim_mlpregister_tasknn
ModuleDictheads
label_maps
startswithr   
output_dimshaper   NotImplementedErrorformatload_state_dictkeys)selfr   argskwargs
model_path
model_dictr   r   vim_weights
num_layerslayer_iagg_weightsagg_algo	task_namer    	__class__ `/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/cv/vision_middleware/model.pyr'       s   






zVisionMiddlewareModel.__init__returnc                 C   s   |  | ||||S )N)postprocessforward)rB   inputsrL   rO   rO   rP   __call__j   s   zVisionMiddlewareModel.__call__c                 C   s@   || j vrtd| j  d| | j||d}| j| |}|S )z
        Dynamic Forward Function of ViM.

        Args:
            x: the input images (B, 3, H, W)
            task_name: specified task for forwarding
        ztask_name should in z
, but got )rL   )r   r>   r-   r9   )rB   rT   rL   featuresoutputsrO   rO   rP   rS   n   s   
zVisionMiddlewareModel.forwardc                    s   |  \}}}}dv r{tj|dd}t|||fddd}|d   }tj|dd}	tt	t
|	d }
g g }}|
D ]$}|	|k}||   || ||   |     qA fd	d
|
D }tj|tj|tj|iS td)z
        Post-process of ViM, based on task_name.

        Args:
            inputs: batched input image (B, 3, H, W)
            outputs: batched output (format based on task_name)
            task_name (str): task name
        r$   r
   )dimNbilinearTr   c                    s   g | ]	} j  | qS rO   )r:   ).0labelrB   rL   rO   rP   
<listcomp>   s    z5VisionMiddlewareModel.postprocess.<locals>.<listcomp>z9Only segmentation task is currently supported in pipeline)sizeFsoftmaxinterpolatedetachr   r+   argmaxsortedlistsetreshapenumpyappendlongfloatsumitemr   MASKSLABELSSCORESr>   )rB   rW   rT   rL   _r"   
img_height	img_widthr$   predlabelsmasksscoresr\   masklabel_namesrO   r]   rP   rR      s8   


z!VisionMiddlewareModel.postprocessc                 C   s   | j S )z?
        Get the supported tasks of current ViM model.
        )r   )rB   rO   rO   rP   	get_tasks   s   zVisionMiddlewareModel.get_tasks)__name__
__module____qualname____doc__strr'   r   r   rU   rS   rR   r{   __classcell__rO   rO   rM   rP   r      s    	J*r   )!os.pathpathr(   typingr   r   jsonr+   torch.nnr7   torch.nn.functional
functionalr`   modelscope.metainfor   'modelscope.models.base.base_torch_modelr   modelscope.models.builderr   modelscope.outputsr   modelscope.utils.constantr   r	   r-   r   headr   r   register_moduleimage_segmentationvision_middlewarer   rO   rO   rO   rP   <module>   s"   