o
    
۾i}                     @   s  d dl Z d dlmZmZmZ d dlmZmZ d dlZ	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZmZmZ d d	lmZmZ d d
lm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9m:Z: d dl;m<Z< d dl=m>Z>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZEmFZFmGZGmHZH d dlImJZJ d dlKmLZL d dlMmNZNmOZO d dlPmQZQ e$eRZSdZTG dd  d e0ZUG d!d" d"e/ZVG d#d$ d$ejWZXG d%d& d&eXZYG d'd( d(ejWZZG d)d* d*eNZ[G d+d, d,Z\G d-d. d.Z]G d/d0 d0eEZ^G d1d2 d2eDe^ Z_G d3d4 d4eFe^ Z`G d5d6 d6ejWZae<jbe`e^e_d7G d8d9 d9ejWe5ZcdS ):    N)IterableMappingSequence)	AnnotatedLiteral)	rearrange)Image)OPENAI_CLIP_MEANOPENAI_CLIP_STD)
transforms)
BartConfigBatchFeaturePretrainedConfig
TensorType)CacheConfig
VllmConfig)
LoRAConfig)BaseDummyOptions)init_logger)
get_act_fn)ColumnParallelLinearRowParallelLinear)LogitsProcessor)QuantizationConfig)ParallelLMHeadVocabParallelEmbedding)default_weight_loader)MultiModalEmbeddingsSupportsMultiModal)
RadioModel)WhisperAttentionWhisperCrossAttention)MULTIMODAL_REGISTRY)MultiModalDataDictMultiModalFieldConfigMultiModalKwargsItems)MultiModalDataItems)BaseDummyInputsBuilderBaseProcessingInfoEncDecMultiModalProcessorPromptReplacementPromptUpdate)TokenizerLike)RadioConfig)TensorSchemaTensorShape)AttentionType)i   ip  c                       L   e Zd ZdZ	ddededef fddZdejd	ejf fd
dZ	  Z
S )BartScaledWordEmbeddingzj
    This module overrides VocabParallelEmbedding's
    forward by multiplying with embeddings scale.
          ?num_embeddingsembedding_dimembed_scalec                       t  || || _d S Nsuper__init__r6   selfr4   r5   r6   	__class__ ]/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/nemotron_parse.pyr;   L      
z BartScaledWordEmbedding.__init__	input_idsreturnc                    s   t  || j S r8   r:   forwardr6   r=   rC   r>   r@   rA   rF   R      zBartScaledWordEmbedding.forwardr3   __name__
__module____qualname____doc__intfloatr;   torchTensorrF   __classcell__r@   r@   r>   rA   r2   F   s    "r2   c                       r1   )BartParallelLMHeadz
    This module overrides ParallelLMHead's
    forward by dividing by embeddings scale,
    yielding effectively the inverse of
    BartScaledWordEmbedding
    r3   r4   r5   r6   c                    r7   r8   r9   r<   r>   r@   rA   r;   ^   rB   zBartParallelLMHead.__init__rC   rD   c                    s   t  || j S r8   rE   rG   r>   r@   rA   rF   d   rH   zBartParallelLMHead.forwardrI   rJ   r@   r@   r>   rA   rT   V   s    "rT   c                	       sb   e Zd Z			ddededB dedB def fddZ	dd	ej	d
ej	dB dej	fddZ
  ZS )BartDecoderLayerN configcache_configquant_configprefixc                    s   t    |j| _t| j|jtj||| dd| _t	|j
| _t| j| _	 t| j|j||| dd| _t| j| _| j}|j}d}t||||| dd| _t||||| dd| _t| j| _d S )	Nz
.self_attn)	embed_dim	num_heads	attn_typerX   rY   rZ   z.encoder_attnrX   rY   rZ   Tz.fc1)biasrY   rZ   z.fc2)r:   r;   d_modelr[   r    decoder_attention_headsr0   DECODER	self_attnr   activation_functionactivation_fnnn	LayerNormself_attn_layer_normr!   encoder_attnencoder_attn_layer_normencoder_ffn_dimr   fc1r   fc2final_layer_norm)r=   rW   rX   rY   rZ   ffn_hidden_sizeffn_intermediate_sizeffn_has_biasr>   r@   rA   r;   i   sN   
zBartDecoderLayer.__init__decoder_hidden_statesencoder_hidden_statesrD   c                 C   s   |}| j |d}|| }| |}|}| j||d}|| }| |}|}| |\}}| |}| |\}}|| }| |}|S )z
        Args:
            decoder_hidden_states: torch.Tensor of *decoder* input embeddings.
            encoder_hidden_states: torch.Tensor of *encoder* input embeddings.
        Returns:
            Decoder layer output torch.Tensor
        hidden_statesru   rs   )rc   rh   ri   rj   rl   re   rm   rn   r=   rr   rs   residualru   fc1_out_r@   r@   rA   rF      s$   



zBartDecoderLayer.forward)NNrV   r8   )rK   rL   rM   r   r   r   strr;   rQ   rR   rF   rS   r@   r@   r>   rA   rU   h   s*    :rU   c                   @   s0   e Zd Z	ddejdejdB dejfddZdS )MBartDecoderLayerNrr   rs   rD   c                 C   s   |}|  |}| j|d}|| }|}| |}| j||d}|| }|}| |}| |\}}| |}| |\}}|| }|S )Nrt   rv   )rh   rc   rj   ri   rn   rl   re   rm   rw   r@   r@   rA   rF      s$   



zMBartDecoderLayer.forwardr8   )rK   rL   rM   rQ   rR   rF   r@   r@   r@   rA   r|      s    r|   c                       s   e Zd ZdZ					ddededB dedB dedB dej	dB d	e
f fd
dZdddejdB dejdB dejdB dejfddZdeee
ejf  dee
 fddZ  ZS )MBartDecoderNoPosz
    Transformer decoder consisting of *config.decoder_layers* layers.
    Each layer is a [`BartDecoderLayer`]
    Args:
        config: BartConfig
        embed_tokens (nn.Embedding): output embedding
    NrV   rW   rX   rY   lora_configembed_tokensrZ   c                    s   t     | _| _|| _jrtjnd}t	j
j|d| _|d ur,|j| j_t fddtjD | _tj| _tj| _d S )Nr3   )r6   c              	      s&   g | ]}t   d | dqS )z.layers.rZ   )r|   ).0	layer_idxrX   rW   rZ   rY   r@   rA   
<listcomp>  s    z.MBartDecoderNoPos.__init__.<locals>.<listcomp>)r:   r;   rX   rY   r~   scale_embeddingmathsqrtr`   r2   
vocab_sizer   weightrf   
ModuleListrangedecoder_layerslayersrg   layernorm_embedding
layer_norm)r=   rW   rX   rY   r~   r   rZ   r6   r>   r   rA   r;      s"   
	

zMBartDecoderNoPos.__init__)inputs_embedsdecoder_input_idsrs   r   rD   c                K   sB   |du r	|  |}| |}| jD ]}|||d}q| |}|S )a>  
        Args:
            decoder_input_ids: Indices of *decoder* input sequence tokens in the
                vocabulary. Padding will be ignored by default should you provide it.
            encoder_hidden_states: Tensor of encoder output embeddings
        Returns:
            Decoder output torch.Tensor
        N)rr   rs   )r   r   r   r   )r=   r   rs   r   kwargsru   decoder_layerr@   r@   rA   rF   "  s   



zMBartDecoderNoPos.forwardweightsc                 C   s   g d}t |  }t }|D ]S\}}|drq|D ](\}}}	||vr%q|||}|dr5||vr5q|| }
|
j}||
||	  n|drN||vrNq|| }
t|
dt}||
| |	| q|S )N)).self_attn.qkv_projz.self_attn.q_projq)r   z.self_attn.k_projk)r   z.self_attn.v_projv).encoder_attn.kv_projz.encoder_attn.k_projr   )r   z.encoder_attn.v_projr   embed_positionsz.biasweight_loader)
dictnamed_parametersset
startswithreplaceendswithr   getattrr   add)r=   r   stacked_params_mappingparams_dictloaded_paramsnameloaded_weight
param_nameweight_nameshard_idparamr   r@   r@   rA   load_weightsB  s.   

zMBartDecoderNoPos.load_weights)NNNNrV   )rK   rL   rM   rN   r   r   r   r   rf   	Embeddingr{   r;   rQ   rR   rF   r   tupler   r   rS   r@   r@   r>   rA   r}      s>    *
, r}   c                   @   s:   e Zd ZU dZed ed< eeje	ddddf ed< d	S )
NemotronParsePixelInputszx
    Dimensions:
        - b: Batch size
        - c: Number of channels (3)
        - h: Height
        - w: Width
    pixel_valuestypeb   hwdataN)
rK   rL   rM   rN   r   __annotations__r   rQ   rR   r/   r@   r@   r@   rA   r   i  s   
  r   c                   @   s   e Zd ZdZefdefddZdd Zdej	dej	fd	d
Z
dej	dej	fddZdejeej B deeejf fddZdejeej B deeejf fddZdS )NemotronParseImageProcessorz'
    NemotronParse Image Processor
    
final_sizec                 K   s   t |ttfrt|dkrt|d t|d f| _nt |ttfr,t|t|f| _nt| _t	t
dddd| _t	tdddd| _|   d S )N   r      r   )
isinstancelistr   lenrO   r   rP   DEFAULT_FINAL_IMAGE_SIZErQ   rR   r	   reshape	norm_meanr
   norm_std_create_transforms)r=   r   r   r@   r@   rA   r;   {  s   z$NemotronParseImageProcessor.__init__c              
   C   s   zddl }W n ty } ztd|d}~ww t| jttfr2t| jd t| jd | _| _n	t| j | _| _ddl	}|
|j| j| j|jg dddg| _t
t g| _dS )zCreate transform objects.r   NzyThe package `albumentations` is required to use NemotronParse model. Please install it with `pip install albumentations`.r   )   r   r   r3   )
min_height	min_widthborder_modefillp)albumentationsImportErrorr   r   r   r   rO   target_heighttarget_widthcv2ComposePadIfNeededBORDER_CONSTANT	transformTToTensortorch_transform)r=   Aerrr   r@   r@   rA   r     s<   
z.NemotronParseImageProcessor._create_transformsimagerD   c           
      C   s|   |j dd \}}| j}| j}|| }|}|}||kr#|}t|| }||kr/|}t|| }ddl}	|	j|||f|	jdS )z[Resize image maintaining aspect ratio (exact replica of original
        LongestMaxSizeHW).Nr   r   )interpolation)shaper   r   rO   r   resizeINTER_LINEAR)
r=   r   heightwidthmax_size_heightmax_size_widthaspect_ratio
new_height	new_widthr   r@   r@   rA   _resize_with_aspect_ratio  s    z5NemotronParseImageProcessor._resize_with_aspect_ratioc           	      C   s   |j dd \}}| j| j}}td|| }td|| }|dkr(|dkr(|S t|j dkrAtj|d|fd|fdfddd}|S tj|d|fd|ffddd}|S )	zUPad image to target size with white padding (matches A.PadIfNeeded
        behavior).Nr   r   r   )r   r   constantr   )modeconstant_values)r   r   r   maxr   nppad)	r=   r   r   r   r   r   pad_hpad_wpaddedr@   r@   rA   _pad_to_size  s$   z(NemotronParseImageProcessor._pad_to_sizeimagesc           
      K   s   t |ts|g}g }|D ]}t |tjrt|}|| qg }|D ]4}| |}| jdur:| j|d}|d }n| |}| 	|}|j
d dkrR|ddd}|| q#t|}|| j | j }	d|	iS )	z
        Preprocess an image or batch of images for the NemotronParse model.

        Args:
            images: Input image(s)
        N)r   r   r   r   r   r   )r   r   r   r   asarrayappendr   r   r   r   r   expandrQ   stackr   r   )
r=   r   r   processed_imagesr   r   processed_imagetransformedpixel_values_tensornormalized_valuesr@   r@   rA   
preprocess  s*   







z&NemotronParseImageProcessor.preprocessc                 K   s   | j |fi |S r8   )r   )r=   r   r   r@   r@   rA   __call__&  s   z$NemotronParseImageProcessor.__call__N)rK   rL   rM   rN   r   r   r;   r   r   ndarrayr   r   r   r   r   r{   rQ   rR   r   r   r@   r@   r@   rA   r   v  s&    
(
5r   c                	       sv   e Zd ZdZdededdf fddZddd	Z			dd
edB de	j	e
e	j	 B dB deeB dB defddZ  ZS )NemotronParseProcessorz!
    NemotronParse Processor
    rW   	tokenizerrD   Nc                    s(   t    || _|| _t|jd| _d S )N)r   )r:   r;   rW   r   r   
image_sizeimage_processor)r=   rW   r   r   r>   r@   rA   r;   1  s   
zNemotronParseProcessor.__init__c                 C   s    |d u rg }t |ts|g}|S r8   )r   r   )r=   
input_itemr@   r@   rA   _make_batch_input>  s
   
z(NemotronParseProcessor._make_batch_inputtextr   return_tensorsc                    sb    fdd||fD \}}t |dkri n |} j|fddi|}ti |||d}|S )Nc                    s   g | ]}  |qS r@   )r  )r   xr=   r@   rA   r   L  s    z3NemotronParseProcessor.__call__.<locals>.<listcomp>r   add_special_tokensF)r   tensor_type)r   r  r   r   )r=   r  r   r  r   image_inputstext_inputscombined_outputsr@   r  rA   r   E  s   
zNemotronParseProcessor.__call__r8   )NNN)rK   rL   rM   rN   r   r,   r;   r  r{   r   r   r   r   r   rS   r@   r@   r>   rA   r   ,  s,    
	
r   c                   @   s   e Zd Zdd ZdefddZedefddZde	e
edB f fd	d
ZdefddZdede	e
ef de	e
ef dB fddZdS )NemotronParseProcessingInfoc                 C   s
   | j  S r8   )ctxget_hf_configr  r@   r@   rA   r  X     
z)NemotronParseProcessingInfo.get_hf_configrD   c                 K   s"   | j jtf|  |  d|S )N)rW   r   )r  init_processorr   r  get_tokenizer)r=   r   r@   r@   rA   get_hf_processor[  s   z,NemotronParseProcessingInfo.get_hf_processorc                 C      dS )NTr@   r  r@   r@   rA   skip_prompt_length_checkc  s   z4NemotronParseProcessingInfo.skip_prompt_length_checkNc                 C   s   ddiS )Nr   r   r@   r  r@   r@   rA   get_supported_mm_limitsg  s   z3NemotronParseProcessingInfo.get_supported_mm_limitsc                 C   s6   |   }|j}|jj}|d | |d | d  d S )Nr   r      )r  r   encoder
patch_size)r=   rW   r   r  r@   r@   rA   get_num_image_tokensj  s    z0NemotronParseProcessingInfo.get_num_image_tokensseq_len	mm_countsc                 C   s   |   }d|iS )Nr   )r  )r=   r  r  image_tokensr@   r@   rA   get_mm_max_tokens_per_itemq  s   z6NemotronParseProcessingInfo.get_mm_max_tokens_per_item)rK   rL   rM   r  r   r  propertyboolr  r   r{   rO   r  r  r  r@   r@   r@   rA   r  W  s    
r  c                	   @   sX   e Zd Zdeeef defddZ	d
dedeeef deeef dB defdd	Z	dS )NemotronParseDummyInputsBuilderr  rD   c                 C   r  NrV   r@   )r=   r  r@   r@   rA   get_dummy_text}  s   z.NemotronParseDummyInputsBuilder.get_dummy_textNr  
mm_optionsc                 C   s0   | dd}| j j\}}d| j|||diS )Nr   r   )r   r   
num_images)getinfor  r   _get_dummy_images)r=   r  r  r$  r%  r   r   r@   r@   rA   get_dummy_mm_data  s   z1NemotronParseDummyInputsBuilder.get_dummy_mm_datar8   )
rK   rL   rM   r   r{   rO   r#  r   r#   r)  r@   r@   r@   rA   r!  z  s    
r!  c                
       s   e Zd Zdeee B dedeee B fddZdedeee	f deee	f deee	f de
f
 fd	d
Zde
deee	f deeef fddZdedeee	f dedee fddZ  ZS ) NemotronParseMultiModalProcessorpromptmm_itemsrD   c                 C   s   dgS )Nr   r@   )r=   r+  r,  r@   r@   rA   create_encoder_prompt  s   z6NemotronParseMultiModalProcessor.create_encoder_promptmm_data	mm_kwargs
tok_kwargsc                    s<   |rt  ||||}|S | j }|j}||ddd}|S )NFpt)r  r  )r:   _call_hf_processorr'  r  r   )r=   r+  r.  r/  r0  processed_outputshf_processorr   r>   r@   rA   r2    s   	
z3NemotronParseMultiModalProcessor._call_hf_processor	hf_inputshf_processor_mm_kwargsc                 C   s   t tddS )Nr   )r   )r   r$   batched)r=   r5  r6  r@   r@   rA   _get_mm_fields_config  s   z6NemotronParseMultiModalProcessor._get_mm_fields_configout_mm_kwargsc                 C   s"   | j  }tddgdg| dgS )Nr   r   )modalitytargetreplacement)r'  r  r*   )r=   r,  r6  r9  num_image_tokensr@   r@   rA   _get_prompt_updates  s   
z4NemotronParseMultiModalProcessor._get_prompt_updates)rK   rL   rM   r{   r   rO   r&   r-  r   objectr   r2  r$   r8  r%   r   r+   r>  rS   r@   r@   r>   rA   r*    sF    









r*  c                       s   e Zd ZdZ		ddededB def fddZ	dd	ededB d
efddZ	de
jd
e
jfddZdeeee
jf  fddZ  ZS )RadioWithNeckz2Vision encoder using RADIO model with custom neck.NrV   rW   rY   rZ   c                    s   t    |j| _| j||d| _d}td|d| _tj	|ddd| _
tj||ddd	d
d| _tj	|ddd| _td||| dd| _tj	|ddd| _d S )NrY   i   i   r   gư>T)epselementwise_affine)r   r  r   F)kernel_sizestridepaddingr_   i   z	.sum_proj)rY   rZ   )r:   r;   r  rW   get_vit_model_from_radio_configmodel_encoderrf   Conv1dconv1rg   layer_norm1Conv2dconv2layer_norm2r   sum_projlayer_norm3)r=   rW   rY   rZ   last_hidden_stater>   r@   rA   r;     s<   
zRadioWithNeck.__init__	hf_configrD   c                 C   sL   |j }|jd}|d u rtd| td||jd|j}t||dS )NmodelzUnsupported vit model type: )
model_namer   )rW   rY   r@   )r  argsr&  
ValueErrorr-   r   r   )r=   rR  rY   hf_config_visionrT  radio_configr@   r@   rA   rG    s   z-RadioWithNeck.get_vit_model_from_radio_configr   c                 K   s   |  |\}}| |dddddd}| |}| jj}t|d|jd | |jd | d}| |}t|d}| 	|}| 
| |d }tj||dfdd	}|S )
Nr   r   r   zb (h w) d -> b d h wr   r   r   zb d h w -> b (h w) ddim)rH  rJ  permuterK  rW   r  r   r   rM  rN  rP  rO  rQ   cat	unsqueeze)r=   r   r   summaryfeatureoutputr  r@   r@   rA   rF     s    



zRadioWithNeck.forwardr   c              	   C   s   g }dd t |   D }|D ]8\}}|dr,|d|ddd  |f q|| }t  t	|| W d    n1 sDw   Y  q| j
| d S )Nc                 S   s    i | ]\}}| d s||qS )rH  r   )r   r   r   r@   r@   rA   
<dictcomp>  s    z.RadioWithNeck.load_weights.<locals>.<dictcomp>rH  .r   )r   r   itemsr   r   joinsplitrQ   no_gradr   rH  r   )r=   r   model_encoder_weightsadaptor_dictr   r   r   r@   r@   rA   r     s   
$
zRadioWithNeck.load_weightsr"  r8   )rK   rL   rM   rN   r   r   r{   r;   r   rG  rQ   rR   rF   r   r   r   rS   r@   r@   r>   rA   r@    s*    +
$r@  )r'  dummy_inputsc                	       s   e Zd Zdddedef fddZededed	ed
B fddZde	d	e
d
B fddZde
d	ejfddZde	d	ed
B fddZ	
d dejd
B dejdeej d
B d	ejfddZdejd	ejd
B fddZdeeeejf  fddZ  ZS )!%NemotronParseForConditionalGenerationrV   r   vllm_configrZ   c                   s   t    |jj}|| _|j| _|j}|j}| 	|d t
||| dd| _W d    n1 s2w   Y  | | t|j||| dd| _W d    n1 sTw   Y  |jj| _t|jj|jj|d| _t| j|jj| _d S )Nr   z.encoder)rW   rY   rZ   z.decoderr^   rA  )r:   r;   model_configrR  rW   r  vision_configrX   rY   _mark_tower_modelr@  _mark_language_modelr}   decoderr   r   r`   lm_headr   logits_processor)r=   rn  rZ   rW   rX   rY   r>   r@   rA   r;   1  s4   





z.NemotronParseForConditionalGeneration.__init__r:  irD   Nc                 C   s   | drd S td)Nr   z Only image modality is supported)r   rV  )clsr:  rv  r@   r@   rA   get_placeholder_strO  s   
z9NemotronParseForConditionalGeneration.get_placeholder_strr   c                 K   s   | dd }| dd }|d u r|d u rd S |d ur"|d ur"td|d ur6| jj\}}td|||ddS |d ur<ttd)Nr   image_embedsz0Both pixel values and image embeds are provided.rZ  )r   r   resolve_bindingsz This line should be unreachable.)poprV  rW   r   r   NotImplementedErrorAssertionError)r=   r   r   ry  r   r   r@   r@   rA   _parse_and_validate_image_inputV  s$   	zENemotronParseForConditionalGeneration._parse_and_validate_image_inputimage_inputc                 C   s<   |d dksJ |d }t | j j}||}| |S )Nr   r   r   )nextr  
parametersdtypeto)r=   r  r   r  r@   r@   rA   _process_image_inputr  s
   

z:NemotronParseForConditionalGeneration._process_image_inputc                 K   s*   | j di |}|d u rd S | |}|S )Nr@   )r~  r  )r=   r   r  vision_embeddingsr@   r@   rA   embed_multimodal{  s
   
z6NemotronParseForConditionalGeneration.embed_multimodalrC   	positionsencoder_outputsc                 K   s(   d}|rt j|dd}| j||d}|S )a[  
        Args:
            input_ids: torch.Tensor of *decoder* input token ids.
            positions: torch.Tensor of *decoder* position indices.
            encoder_outputs: List of encoder output tensors (vision embeddings).
                During profiling, this may be None or empty.
        Returns:
            Output torch.Tensor
        Nr   r[  )r   rs   )rQ   r^  rs  )r=   rC   r  r  r   r   ru   r@   r@   rA   rF     s   z-NemotronParseForConditionalGeneration.forwardru   c                 C   s   |  | j|S r8   )ru  rt  )r=   ru   r@   r@   rA   compute_logits  s   z4NemotronParseForConditionalGeneration.compute_logitsr   c              	   C   s2  t | j }dtdtfdd}dtdtfdd}dtfdd}g }g }|D ]d\}}	||r@|d	|d	d
d  |	f q&||rV|d	|d	d
d  |	f q&||rd	|d	d
d  }
||
 }t	  t
||	 W d    n1 s~w   Y  q&td| q&| j| | j| d S )Nr   rD   c                 S   
   |  dS )Nr  rc  r   r@   r@   rA   
is_encoder  r  zFNemotronParseForConditionalGeneration.load_weights.<locals>.is_encoderc                 S   r  )Nrs  rc  r  r@   r@   rA   
is_decoder  r  zFNemotronParseForConditionalGeneration.load_weights.<locals>.is_decoderc                 S   r  )Nrt  rc  r  r@   r@   rA   
is_lm_head  r  zFNemotronParseForConditionalGeneration.load_weights.<locals>.is_lm_headre  r   zFound unexpected weight: %s)r   rt  r   r{   r   r   rg  rh  rQ   ri  r   loggerr'  r  r   rs  )r=   r   lm_head_dictr  r  r  encoder_weightsdecoder_weightsr   r   trimmed_namer   r@   r@   rA   r     s*   $$
z2NemotronParseForConditionalGeneration.load_weightsr8   )rK   rL   rM   r   r{   r;   classmethodrO   rx  r?  r   r~  rQ   rR   r  r   r  r   rF   r  r   r   r   rS   r@   r@   r>   rA   rm  +  s>    

	

$rm  )dr   collections.abcr   r   r   typingr   r   numpyr   rQ   torch.nnrf   einopsr   PILr   timm.data.constantsr	   r
   torchvisionr   r   transformersr   r   r   r   vllm.configr   r   vllm.config.lorar   vllm.config.multimodalr   vllm.loggerr   %vllm.model_executor.layers.activationr   !vllm.model_executor.layers.linearr   r   +vllm.model_executor.layers.logits_processorr   3vllm.model_executor.layers.quantization.base_configr   3vllm.model_executor.layers.vocab_parallel_embeddingr   r   -vllm.model_executor.model_loader.weight_utilsr   %vllm.model_executor.models.interfacesr   r    vllm.model_executor.models.radior   "vllm.model_executor.models.whisperr    r!   vllm.multimodalr"   vllm.multimodal.inputsr#   r$   r%   vllm.multimodal.parser&   vllm.multimodal.processingr'   r(   r)   r*   r+   vllm.tokenizersr,   %vllm.transformers_utils.configs.radior-   vllm.utils.tensor_schemar.   r/   vllm.v1.attention.backendr0   rK   r  r   r2   rT   ModulerU   r|   r}   r   r   r   r  r!  r*  r@  register_processorrm  r@   r@   r@   rA   <module>   sr   	e'u 7+
#
5e