o
    ॵi$                     @   s<  d dl mZ d dlZd dlZd dlmZmZmZ d dl	Z	d dl
Zd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZ d dl m!Z! e! Z"dd Z#G dd dZ$dd Z%G dd deZ&ej'ej(ej)dG dd de&Z*ej'ej+ej,dG dd de&Z-dS )    N)AnyDictOptional)Image)BertTokenizer)snapshot_download)	Pipelines)Pipeline)	PIPELINES)
load_image)DEFAULT_MODEL_REVISION
FrameworksInvokeTasks)
get_loggerc                 C   s   d | | d S )Nz{:.2f}msi  )format)endbegin r   e/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/pipelines/multi_modal/gridvlp_pipeline.pycost   s   r   c                   @   sp   e Zd ZdZejg dejdddejejf Zejg dejdddejejf Z	e
dZe
dZdZdS )Configgp?)g
ףp=
?gv/?gCl?dtypeN)gZd;O?gy&1?g?      )__name__
__module____qualname__SCALEnprequirefloat32newaxisMEANSTDintRESIZE_HEIGHTRESIZE_WIDTH	CROP_SIZEr   r   r   r   r      s    r   c                 C   s6  |  d} | j\}}||kr|tjks||kr|tjkr| } ||k r8tj}ttj| | }| ||ftj} ntj}ttj| | }| ||ftj} | j\}}tt|tj	 d }tt|tj	 d }| 
|||tj	 |tj	 f} tj| tjd}tj|dtjd}|tj9 }|tj8 }|tj }|S )NRGBg       @r   )   r      )convertsizer   r(   r&   resizer   BILINEARroundr)   cropr    arrayr"   r!   	transposer   r$   r%   )imgwhowohcrop_top	crop_left_imgr   r   r   pre_processor,   s0   





r=   c                   @   s6   e Zd ZdZdefddZd
deeef fddZd	S )GridVlpPipelinez> Pipeline for gridvlp, including classification and embedding.model_name_or_pathc                 K   s   t d|  t|r|}ndtjtjf }t|t|d}|| _	t d|  t
jt|d| _tj
| _d| _d| _d| _t d	|  t|| _d
S )z Pipeline for gridvlp, including classification and embedding.
        Args:
            model: path to local model directory.
        z load checkpoint from modelscope z%s/%s)
user_agentzload model from zpytorch_model.ptcpuTFzload tokenizer from N)loggerinfoospexistsr   KEYPIPELINEr   r   local_model_dirtorchjitloadjoinmodelr   	frameworkdevice_name_model_prepare_auto_collater   from_pretrained	tokenizer)selfr?   kwargsrH   
invoked_byr   r   r   __init__K   s(   

zGridVlpPipeline.__init__1   inputsc              
   C   s(  | dd}| dd}t }zt|}t }t|}t }	W n  tyB   tjdtjd}t }t }	t	t
  Y nw |d u sO| sO| sVt	d d}| j|dd|d	}t }
t	d
| d|  t	dt|| dt|	| dt|
|	  ||d |d |d d}|S )Nimage text)   r   r   r   ztext is empty!
max_lengthT)padding
truncationr^   zexample. text: z image: zpreprocess. Img_Download:z
, Img_Pre:z
, Txt_Pre:	input_idsattention_masktoken_type_ids)rZ   ra   
input_masksegment_ids)gettimer   r=   	Exceptionr    zerosr"   rB   rC   	traceback	print_excisspacestriprS   r   )rT   rY   max_seq_lengthrZ   r\   s1r5   s2
image_datas3s4
input_dictr   r   r   
preprocessi   sD   
&zGridVlpPipeline.preprocessN)rX   )	r   r   r   __doc__strrW   r   r   ru   r   r   r   r   r>   H   s    r>   )module_namec                       sj   e Zd ZdZdef fddZdeeef deeef fddZdeeef deeef fd	d
Z	  Z
S )GridVlpClassificationPipelineu   Pipeline for gridvlp classification, including cate classfication and
    brand classification.

    Example:

    ```python
    >>> from modelscope.pipelines.multi_modal.gridvlp_pipeline import     GridVlpClassificationPipeline

    >>> pipeline = GridVlpClassificationPipeline('rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-cate')
    >>> output = pipeline({'text': '女装快干弹力轻型短裤448575',        'image':'https://yejiabo-public.oss-cn-zhangjiakou.aliyuncs.com/alinlp/clothes.png'})
    >>> output['text'][0]
    {'label': {'cate_name': '休闲裤', 'cate_path': '女装>>裤子>>休闲裤>>休闲裤'}, 'score': 0.4146, 'rank': 0}

    ```
    r?   c                    sD   t  j|fi | td| j  ttt	| jd| _
dS )z Pipeline for gridvlp classification, including cate classfication and
    brand classification.
        Args:
            model: path to local model directory.
        zload label mapping from zlabel_mapping.jsonN)superrW   rB   rC   rH   jsonrK   openrD   rL   label_mapping)rT   r?   rU   	__class__r   r   rW      s
   
z&GridVlpClassificationPipeline.__init__rY   returnc              
   C   s   t   }tjdtjd}| t|d d|dtj|d tjddtj|d tjddtj|d tjdd}|d  	 }t   }t
dt||  |S )	Nr,   r   rZ   r   ra   rd   re   forward. Infer:rg   rI   ri   r"   rM   tensor	unsqueezelongdetachnumpyrB   rC   r   )rT   rY   rs   
box_tensoroutputs5r   r   r   forward   s   z%GridVlpClassificationPipeline.forwardc           
      C   s   t   }|}t| }|| }g }ttdt| jD ]}| jt||  }||t	t
|| d|d qt   }	tdt|	|  d|iS )N
      )labelscorerankzpostprocess. Post: r\   )rg   r    argsortrangeminlenr}   rw   appendr1   floatrB   rC   r   )
rT   rY   r   r   indexout_sorttop_kir   s6r   r   r   postprocess   s   
z)GridVlpClassificationPipeline.postprocess)r   r   r   rv   rw   rW   r   r   r   r   __classcell__r   r   r~   r   ry      s
    "*ry   c                   @   sT   e Zd ZdZdeeef deeef fddZdeeef deeef fddZdS )	GridVlpEmbeddingPipelineum   Pipeline for gridvlp embedding. These only generate unified multi-modal
    embeddings and output it in `text_embedding` or `img_embedding`.

    Example:

    ```python
    >>> from modelscope.pipelines.multi_modal.gridvlp_pipeline import     GridVlpEmbeddingPipeline

    >>> pipeline = GridVlpEmbeddingPipeline('rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-embedding')
    >>> outputs = pipeline({'text': '女装快干弹力轻型短裤448575',        'image':'https://yejiabo-public.oss-cn-zhangjiakou.aliyuncs.com/alinlp/clothes.png'})
    >>> outputs["text_embedding"].shape
    (768,)

    ```
    rY   r   c              
   C   s   t   }tjdtjd}| t|d d|dtj|d tjddtj|d tjddtj|d tjdd}t   }|d  	 }t   }t
dt|| d	t||  |S )
Nr,   r   rZ   r   ra   rd   re   r   z, Post: r   )rT   rY   rs   r   r   r   r   r   r   r   r      s   "z GridVlpEmbeddingPipeline.forwardc                 C   s   ||d}|S )N)img_embeddingtext_embeddingr   )rT   rY   outputsr   r   r   r     s   z$GridVlpEmbeddingPipeline.postprocessN)	r   r   r   rv   r   rw   r   r   r   r   r   r   r   r      s    "&r   ).os.pathpathrD   rg   rj   typingr   r   r   r{   r   r    rI   PILr   transformersr    modelscope.hub.snapshot_downloadr   modelscope.metainfor   modelscope.pipelinesr	   modelscope.pipelines.builderr
   modelscope.preprocessors.imager   modelscope.utils.constantr   r   r   r   modelscope.utils.loggerr   rB   r   r   r=   r>   register_modulevisual_question_answering"gridvlp_multi_modal_classificationry   multi_modal_embeddinggridvlp_multi_modal_embeddingr   r   r   r   r   <module>   s>   RH