o
    
۾i                     @   s   d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ ddlmZmZmZ ddlmZmZmZm Z  ddl!m"Z"m#Z#m$Z$ ee%Z&G dd dej'Z(G dd deZ)ej*e)e edG dd deeeeZ+dS )    )IterableMappingN)BatchFeature)ModelConfig
VllmConfig)TokensPrompt)init_logger)ColumnParallelLinearRowParallelLinear)DispatchPooler)MULTIMODAL_REGISTRY)IntermediateTensors   )SupportsCrossEncodingSupportsMultiModalSupportsScoreTemplate)Qwen2VLDummyInputsBuilderQwen2VLForConditionalGenerationQwen2VLMultiModalProcessorQwen2VLProcessingInfo)AutoWeightsLoaderWeightsMappermaybe_prefixc                       s0   e Zd Zd	dddef fddZdd Z  ZS )
JinaVLScorer model_configr   prefixc                    sZ   t    |j }|j}t|j|j|d| dd| _t|j|j	|d| dd| _
d S )NTz.dense)params_dtypebiasr   z	.out_proj)super__init__	hf_configget_text_config
head_dtyper	   hidden_sizedenser
   
num_labelsout_proj)selfr   r   configr#   	__class__ V/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/jina_vl.pyr       s"   

zJinaVLScorer.__init__c                 K   s*   |  |\}}t|}| |\}}|S N)r%   torchrelur'   )r(   xkwargs_r,   r,   r-   forward1   s   
zJinaVLScorer.forward)r   )__name__
__module____qualname__strr    r4   __classcell__r,   r,   r*   r-   r      s    r   c                
       sJ   e Zd Zdedeeef deeef deeef def
 fddZ  ZS )JinaVLMultiModalProcessorpromptmm_data	mm_kwargs
tok_kwargsreturnc                    s,   |  D ]\}}|  qt ||||S r.   )itemsreverser   _call_hf_processor)r(   r;   r<   r=   r>   r3   valuer*   r,   r-   rB   9   s   
z,JinaVLMultiModalProcessor._call_hf_processor)	r5   r6   r7   r8   r   objectr   rB   r9   r,   r,   r*   r-   r:   8   s    


r:   )infodummy_inputsc                       s   e Zd ZdZeddddddddZd	d
dedef fddZe	dede
dedB fddZe	dedededB fddZe	deddfddZ		d&dejdB dejdedB dejdB d edejf fd!d"Zd#eeeejf  fd$d%Z  ZS )'JinaVLForSequenceClassificationTzscore.dense.zscore.out_proj.zlanguage_model.model.visual.zlanguage_model.lm_head.)zscore.0.zscore.2.zmodel.language_model.rH   zlm_head.zmodel.)orig_to_new_prefixr   r   vllm_configr   c                   sV   t  j|t|dd |jj}|d usJ t|jt|dd| _tj|| jd| _	d S )Nqwen2_vl)rK   r   scorerJ   )
classifier)
r   r    r   r   pooler_configr   rM   r   for_seq_clspooler)r(   rK   r   rO   r*   r,   r-   r    b   s   
z(JinaVLForSequenceClassification.__init__modalityir?   Nc                 C   s   | drdS td)Nimagez+<|vision_start|><|image_pad|><|vision_end|>z Only image modality is supported)
startswith
ValueError)clsrR   rS   r,   r,   r-   get_placeholder_strn   s   
z3JinaVLForSequenceClassification.get_placeholder_strquerydocumentc                 C   s   d| d| S )Nz**Document**:
z
**Query**:
r,   )rW   rY   rZ   r,   r,   r-   get_score_templateu   s   z2JinaVLForSequenceClassification.get_score_templater;   c                 C   s   |d  d d S )Nprompt_token_idsd   )append)rW   r;   r,   r,   r-   post_process_tokensy   s   z3JinaVLForSequenceClassification.post_process_tokens	input_ids	positionsintermediate_tensorsinputs_embedsr2   c                    s    t  jd||||d|}|S )N)r`   ra   rb   rc   r,   )r   r4   )r(   r`   ra   rb   rc   r2   hidden_statesr*   r,   r-   r4   ~   s   z'JinaVLForSequenceClassification.forwardweightsc                 C   s   t | }|j|| jdS )N)mapper)r   load_weightsweight_mapper)r(   re   loaderr,   r,   r-   rg      s   z,JinaVLForSequenceClassification.load_weights)NN)r5   r6   r7   is_pooling_modelr   rh   r   r8   r    classmethodintrX   r[   r   r_   r/   Tensorr   rD   r4   r   tuplerg   r9   r,   r,   r*   r-   rG   I   sD    $rG   ),collections.abcr   r   r/   torch.nnnntransformersr   vllm.configr   r   vllm.inputsr   vllm.loggerr   !vllm.model_executor.layers.linearr	   r
   !vllm.model_executor.layers.poolerr   vllm.multimodalr   vllm.sequencer   
interfacesr   r   r   rL   r   r   r   r   utilsr   r   r   r5   loggerModuler   r:   register_processorrG   r,   r,   r,   r-   <module>   s8   

