o
    i\                     @   s^  d dl Z d dlZd dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZmZmZmZmZmZmZmZ d dlmZ d d	l m!Z! d d
l"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 e
rd dl3m4Z4m5Z5 ne6e7ef Z4e6e7ef Z5ee8Z9e6e:e7e;f e7dB f  Z<	 de$de;de7dB fddZ=de$de7dB de>e6e7ef  dB ddde7dB f
ddZ?dej@jAd e7de;fd!d"ZBdej@jAd e7d#e7de;fd$d%ZC	d\dej@jAd e7d#e7dB de;fd&d'ZDd(ej@jAd e7fd)d*ZEd(ej@jAfd+d,ZFd(ej@jAfd-d.ZGde7dej@jHdB fd/d0ZIed1d2de7d3edefd4d5ZJde7dB de>e6e7ef  dB de$dddef
d6d7ZKede7dB d8ed9efd:d;ZLde7dB de>e6e7ef  dB d8ede$dddefd<d=ZMG d>d? d?ejNjOZPde7dee7 fd@dAZQeeQZRedeSe7 fdBdCZT	Dd]de$de7dEe6e7ef dFe;de6e7ef f
dGdHZUdddDdIddde$dJe>e de>e6e7ef  dB de7dB dKe;de7e>eV B fdLdMZW	N	O	P	Q		Od^dRdSZXdPdQde>e7 fdTdUZYdVe7e>eV B dPdQdWe7dB de7e>eV B fdXdYZZG dZd[ d[e2Z[dS )_    N)defaultdictdeque)Set)	lru_cache)TYPE_CHECKINGAnycast)ModelConfig)ChatCompletionMessageParamChatTemplateContentFormatChatTemplateContentFormatOptionChatTemplateResolutionErrorConversationMessageload_chat_templateparse_chat_messagesparse_chat_messages_async)init_logger)cached_get_tokenizer)CachedHfTokenizerHfTokenizer)get_chat_template_fallback_path)cached_get_processor)supports_kw   )
DictPrompt)parse_dec_only_prompt)
ChatParams)BaseRenderer)MultiModalDataDictMultiModalUUIDDict	tokenizertrust_remote_codereturnc                C   s   | j |f}|tv rt| S ddlm}m}m} z%t| j |||f|d}t||r;t|dr;|j	 }d ur;|t|< |W S W n t
yN   tjd| j dd Y nw d t|< d S )Nr   )PreTrainedTokenizerPreTrainedTokenizerFastProcessorMixin)processor_clsr!   chat_templatez1Failed to load AutoProcessor chat template for %sTexc_info)name_or_path_PROCESSOR_CHAT_TEMPLATEStransformersr#   r$   r%   r   
isinstancehasattrr'   	Exceptionloggerdebug)r    r!   	cache_keyr#   r$   r%   	processorr'    r4   G/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/renderers/hf.py _try_get_processor_chat_template;   s<   


r6   r'   toolsmodel_configr	   c                C   s   |d ur|S |d u rt | |jd}|d ur|S z| j||dW S  ty1   tjd| jdd Y nw t|jj	| jd}|d urLt
d| j t|}|S td| j |S )	N)r!   )r7   z1Failed to load AutoTokenizer chat template for %sTr(   )
model_typetokenizer_name_or_pathzKLoading chat template fallback for %s as there isn't one defined on HF Hub.z)There is no chat template fallback for %s)r6   r!   get_chat_templater/   r0   r1   r*   r   	hf_configr9   	info_oncer   
debug_once)r    r'   r7   r8   pathr4   r4   r5   resolve_chat_templatef   sB   
r@   nodevarnamec                 C   s&   t | tjjr| jdko| j|kS dS )NloadF)r-   jinja2nodesNamectxname)rA   rB   r4   r4   r5   _is_var_access   s   rI   keyc                 C   s^   t | tjjrt| j|ot | jtjjo| jj|kS t | tjj	r-t| j|o,| j
|kS dS )NF)r-   rD   rE   GetitemrI   rA   argConstvalueGetattrattrrA   rB   rJ   r4   r4   r5   _is_attr_access   s   
rR   c                 C   s   t | tjjr| jd uot| j||S t | tjjr!t| j||S t | tjjr7t | jtjj	r7t| j||S |r?t
| ||S t| |S N)r-   rD   rE   FilterrA   _is_var_or_elems_accessTestrK   rL   SlicerR   rI   rQ   r4   r4   r5   rU      s   
rU   rootc                 c   s    | |fV  t |g}|rD| }| tjjD ]'}|j}|j}t||r?t	|tjj
s.J ||jfV  |j|kr?||j q|sd S d S rS   )r   popleftfind_allrD   rE   AssigntargetrA   rU   r-   rF   rH   append)rX   rB   related_varnamesrelated_varname
assign_astlhsrhsr4   r4   r5   _iter_nodes_assign_var_or_elems   s   



rc   c                 c   sp    dd t | dD }| tjjD ]#}|j}|j}|D ]}t||r4t|tjj	s,J ||j
fV   nqqd S )Nc                 S      g | ]\}}|qS r4   r4   .0_rB   r4   r4   r5   
<listcomp>       z4_iter_nodes_assign_messages_item.<locals>.<listcomp>messages)rc   rZ   rD   rE   Foriterr\   rU   r-   rF   rH   )rX   messages_varnamesloop_ast	loop_iterloop_targetrB   r4   r4   r5    _iter_nodes_assign_messages_item   s   
rq   c                 c   sp    dd t | D }| tjjD ]$}|j}|j}|D ]}t||dr4t|tjj	s,J ||j
fV   nqqd S )Nc                 S   rd   r4   r4   re   r4   r4   r5   rh      ri   z3_iter_nodes_assign_content_item.<locals>.<listcomp>content)rq   rZ   rD   rE   rk   rl   r\   rU   r-   rF   rH   )rX   message_varnamesrn   ro   rp   rB   r4   r4   r5   _iter_nodes_assign_content_item   s   rt   c                 C   sJ   dd l m  m} z|| }|j| W S  ty$   td Y d S w )Nr   z#Error when compiling Jinja template)	&transformers.utils.chat_template_utilsutilschat_template_utils_compile_jinja_templateenvironmentparser/   r0   	exception)r'   hf_chat_utilsjinja_compiledr4   r4   r5   _try_extract_ast   s   

r~       )maxsizedefaultc                C   sZ   t | }|d u r
|S z	tt| W dS  ty   Y dS  ty,   td | Y S w )Nstringz(Error when parsing AST of Jinja templateopenai)r~   nextrt   StopIterationr/   r0   r{   )r'   r   	jinja_astr4   r4   r5   _detect_content_format  s   
r   c                C   sJ   t || ||d}t|tr|nt| dd}|d u rd}|S t|dd}|S )Nr'   r7   r8   T)
is_literalr   )r   )r@   r-   strr   r   )r'   r7   r    r8   resolved_chat_template
jinja_textdetected_formatr4   r4   r5   %_resolve_chat_template_content_format  s"   	

r   given_formatr   c                 C   s6   t d| |dkr||krt d|| d S d S d S )NztDetected the chat template content format to be '%s'. You can set `--chat-template-content-format` to override this.autoa  You specified `--chat-template-content-format %s` which is different from the detected format '%s'. If our automatic detection is incorrect, please consider opening a GitHub issue so that we can improve it: https://github.com/vllm-project/vllm/issues/new/choose)r0   infowarning)r'   r   r   r4   r4   r5   !_log_chat_template_content_format8  s   r   c                C   s.   |dkr|S t | |||d}t| ||d |S )Nr   )r8   )r   r   )r   r   )r'   r7   r   r    r8   r   r4   r4   r5   $resolve_chat_template_content_formatP  s   r   c                   @   s,   e Zd ZdhZdejjdejjfddZ	dS )AssistantTracker
generationparserr"   c                 C   s@   t |jj}|jddd}| d}tj|g g |}||S )N)zname:endgenerationT)drop_needle_generation_support)	r   streamlinenoparse_statementscall_methodrD   rE   	CallBlock
set_lineno)selfr   r   bodycall
call_blockr4   r4   r5   rz   p  s
   

zAssistantTracker.parseN)
__name__
__module____qualname__tagsrD   r   ParserrE   Noderz   r4   r4   r4   r5   r   m  s    r   c                 C   s4   t jjddtt jjgd}|| }t j|}|S )NT)trim_blockslstrip_blocks
extensions)	rD   sandboxImmutableSandboxedEnvironmentr   extloopcontrolsrz   metafind_undeclared_variables)r'   envparsed_contenttemplate_varsr4   r4   r5   _resolve_chat_template_kwargsx  s   

r   c                  C   s0   ddl m}  t| j}tdd |j D S )Nr   )r#   c                 s   s,    | ]}|j tjjtjjfvr|jV  qd S rS   )kindinspect	ParameterVAR_KEYWORDVAR_POSITIONALrH   )rf   pr4   r4   r5   	<genexpr>  s    z4_get_hf_base_chat_template_params.<locals>.<genexpr>)r,   r#   r   	signatureapply_chat_template	frozenset
parametersvalues)r#   base_sigr4   r4   r5   !_get_hf_base_chat_template_params  s
   r   Tchat_template_kwargsraise_on_unexpectedc           	         sp   ddh}|r||  @  }rtd| fdd|D }t|}t }||B |B |   fdd| D S )Nr'   tokenizez4Found unexpected chat template kwargs from request: c                    s    h | ]}t  j|d dr|qS )F)allow_var_kwargs)r   r   )rf   k)r    r4   r5   	<setcomp>  s    z/resolve_chat_template_kwargs.<locals>.<setcomp>c                    s   i | ]\}}| v r||qS r4   r4   )rf   r   v)accept_varsr4   r5   
<dictcomp>  s    z0resolve_chat_template_kwargs.<locals>.<dictcomp>)keys
ValueError$_cached_resolve_chat_template_kwargsr   items)	r    r'   r   r   unexpected_varsunexpected_in_kwargsfn_kwr   hf_base_paramsr4   )r   r    r5   resolve_chat_template_kwargs  s    
r   )r7   r'   r   conversationr   c          	   
   K   s|   t |||| d}|d u rtdt|||d}z|jd||||d|W S  ty= } ztd tt||d }~ww )Nr   zAs of transformers v4.44, default chat template is no longer allowed, so you must provide a chat template if the tokenizer does not define one.)r    r'   r   )r   r7   r'   r   z@An error occurred in `transformers` while applying chat templater4   )	r@   r   r   r   r/   r0   r{   r   r   )	r8   r    r   r7   r'   r   kwargsresolved_kwargser4   r4   r5   safe_apply_chat_template  s>   
	r   mm_uuidsr   mm_datar   c                    sp   | d}|du r| S tdd |D sJ dtttttf  |} fdd|D }|r6t| } || d< | S )as  Rebuild mm_uuids after vision_chunk processing.

    When videos are split into chunks, the original UUIDs need to be updated
    to reflect the new UUIDs generated for each chunk.

    Args:
        mm_uuids: Original UUIDs dictionary
        mm_data: Processed multimodal data with vision_chunk items

    Returns:
        Updated UUIDs dictionary with chunk UUIDs
    vision_chunkNc                 s   s    | ]}t |tV  qd S rS   )r-   dictrf   itemr4   r4   r5   r     s    z0rebuild_mm_uuids_from_mm_data.<locals>.<genexpr>z+Expected all vision_chunk items to be dictsc                    s"   g | ]}| d   dur qS )uuidN)getr   uuid_valr4   r5   rh     s    z1rebuild_mm_uuids_from_mm_data.<locals>.<listcomp>)r   allr   listr   r   r   )r   r   vision_chunksvision_chunk_uuidsr4   r   r5   rebuild_mm_uuids_from_mm_data  s   

r   c                    s   |  d}|du rg S tt |D ]#}t|tsJ | ddkr4| dd}| dd} | | q fd	d
t  D }|S )a   Build video prompts from vision_chunk data.

    Collects prompts from video chunks and groups them by video_idx.

    Args:
        mm_data: Processed multimodal data with vision_chunk items

    Returns:
        List of video prompts, one per video.
    r   Ntypevideo_chunk	video_idxr   prompt c                    s   g | ]	}d   | qS )r   )join)rf   r   video_prompts_dictr4   r5   rh   *  s    z4build_video_prompts_from_mm_data.<locals>.<listcomp>)r   r   r   r-   r   r]   sortedr   )r   r   r   r   r   video_promptsr4   r   r5    build_video_prompts_from_mm_data  s   


r   
prompt_rawvideo_placeholderc                 C   sz   |r;t | tr;t|}| |}t|t|d kr.dtjt	||} | |d 7 } | S t
dt|d t| | S )Nr   r   zVNumber of video placeholders (%d) does not match number of videos (%d) in the request.)r-   r   r   splitlenr   	itertoolschainfrom_iterablezipr0   r   )r   r   r   r   prompt_raw_partsr4   r4   r5   &replace_vision_chunk_video_placeholder2  s   

r   c                       s   e Zd Zededeeef ddfddZdedeeef ddf fdd	Z	e
dedB fd
dZdefddZdee dedeee ef fddZdee dedeee ef fddZ  ZS )
HfRendererconfigtokenizer_kwargsr"   r   c                 C   s
   | ||S rS   r4   )clsr  r  r4   r4   r5   from_configM  s   
zHfRenderer.from_configNc                    sJ   t  | t|jdd| _|jrd }ntttddt	i|}|| _
d S )Nuse_unified_vision_chunkFtokenizer_clsr4   )super__init__getattrr<   r  skip_tokenizer_initr   r   r   r   
_tokenizer)r   r  r  r    	__class__r4   r5   r	  U  s   
zHfRenderer.__init__c                 C   s   | j S rS   )r  )r   r4   r4   r5   r    m  s   zHfRenderer.tokenizerc                 C   s   | j }|d u rtd|S )Nz7Tokenizer not available when `skip_tokenizer_init=True`)r    r   )r   r    r4   r4   r5   get_tokenizerq  s   zHfRenderer.get_tokenizerrj   paramsc              
   C   s   | j }|  }t||t|j|jd|j||dd\}}}t|||fi |	 }| j
rG|d urG|d urGt||}t|jdd }	t|||	}t|}
|d urS||
d< |d ur[||
d< ||
fS Nr7   )r'   r7   r   r    r8   )content_formatr   multi_modal_datamulti_modal_uuids)r  r  r   r   r'   r   r   chat_template_content_formatr   get_apply_chat_template_kwargsr  r   r
  r<   r   r   r   rj   r  r8   r    r   r   r   r   r   r   r4   r4   r5   render_messagesx  sN   


zHfRenderer.render_messagesc              
      s   | j }|  }t||t|j|jd|j||ddI d H \}}}t|||fi |	 }| j
rF|d urF|d urFt|jdd }	t|||	}t|}
|d urR||
d< |d urZ||
d< ||
fS r  )r  r  r   r   r'   r   r   r  r   r  r  r
  r<   r   r   r  r4   r4   r5   render_messages_async  sN   

z HfRenderer.render_messages_async)r   r   r   classmethodr	   r   r   r   r  r	  propertyr   r    r  r   r
   r   tupler   r   r  r  __classcell__r4   r4   r  r5   r  L  sB    


6r  rS   )T)r   r   r   r   r"   r   )\r   r   collectionsr   r   collections.abcr   	functoolsr   typingr   r   r   rD   
jinja2.extjinja2.metajinja2.nodesjinja2.parserjinja2.sandboxvllm.configr	   vllm.entrypoints.chat_utilsr
   r   r   r   r   r   r   r   vllm.loggerr   vllm.tokenizersr   vllm.tokenizers.hfr   r   &vllm.transformers_utils.chat_templatesr   !vllm.transformers_utils.processorr   vllm.utils.func_utilsr   inputsr   inputs.preprocessr   r  r   protocolr   vllm.multimodal.inputsr   r   r   r   r   r0   r  boolr+   r6   r   r@   rE   r   rI   rR   rU   rc   rq   rt   Templater~   r   r   r   r   r   	Extensionr   r   r   r   r   r   intr   r   r   r   r  r4   r4   r4   r5   <module>   sD  (

+
2





$
	
0
#
%


