o
    }oi}S                  
   @   s  d dl Z d dlmZ d dlmZmZ d dlmZ dZdZ	dZ
dZd	Zd
ZdZdZdZedd Zedd Zedd Zded< ded< ded< dZdZdZG dd deZe jG dd dZedddd d ejee d!d"Zed#dd$d d ejee d!d"Zed%d&d'd d ejd#d(Zed)d*d+d,ejd-d.Z ed/d0d1d d ej!d2e
d"Z"ed3d0d4d d ej#e	e
d"Z$ed5d0d4d d ej#e	e
d"Z%ed5d&d6d d ej&d7d(Z'ed%d8d d ej(d#d.Z)ed)d*d9d,ejd-d.Z*ed:d*d d ejd-d;d<Z+ed)d0d1d d ej!d2e
d"Z,ed:d0d d ej!d2e
d=d>Z-ed%d0d?d d ej.d%e
d"Z/e"Z0i d@e dAe d1e"dBe"dCe$dDe)dEe)dFe*d;e+dGe,d=e-dHe%dedIed$ed?e/dJeZ1e2dKkrWe3e04  dS dS )L    N)defaultdict)Enumauto)Listz<pad>z<extra_id_6>z<extra_id_7>z<unk><image>z<video>z<extra_id_0>z<extra_id_1>z<extra_id_2>c                   C      dS )Nz<extra_id_3> r   r   r   f/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/multimodal/data/neva/conversation.py<lambda>       r
   c                   C   r   )Nz<extra_id_4>r   r   r   r   r	   r
      r   c                   C   r   )Nz<extra_id_5>r   r   r   r   r	   r
      r   z<|reserved_special_token_3|>llama_3z<|reserved_special_token_4|>z<|reserved_special_token_5|>z<extra_id_8>z<extra_id_9>z<t{t}>c                   @   s@   e Zd ZdZe Ze Ze Ze Ze Z	e Z
e Ze ZdS )SeparatorStylezDifferent separator style.N)__name__
__module____qualname____doc__r   SINGLETWOPLAINLLAMA_2LLAMA_3MISTRALNVGPTYI34br   r   r   r	   r   +   s    
r   c                   @   s   e Zd ZU dZeed< ee ed< eee  ed< eed< ej	Z
eed< dZeed< d	Zeed
< dZeed< dZeed< dd Zdd ZdddZdd Zdd Zdd Zd	S )Conversationz,A class that keeps all conversation history.systemrolesmessagesoffset	sep_style###sepNsep2UnknownversionF	skip_nextc                 C   s  | j }t|dkrTt|d d tu rT| j  }|d  \}}|d dd }d| jv rL||f|d< |d| j	d df |d| j	d df n|d| f|d< | j
tjkr| j| j }|D ]#\}}|rt|tu rs|\}}}||d	 | | j 7 }qb||d
 7 }qb|S | j
tjkr| j| jg}| j|d  }t|D ]:\}	\}}|rt|tu r|\}}}||d	 | ||	d   7 }|	d dkr|	t|d kr|d7 }q||d
 7 }q|S | j
tjks| j
tjkr| j
tjkrdd }
ndd }
dd }d}| j
tjkr	|t7 }t|D ]q\}	\}}|	dkr+|sJ d|| j	d ks+J d|ryt|tu r:|\}}}|	dkrF|
| j| }|	d dkr\||}|| jd | 7 }q| j
tjkrp|d| d | j 7 }q||| j 7 }q|d7 }q|| j}|S | j
tjkr	 dd }
dd }dd }d|
| j | j }t|D ]M\}	\}}|	dkr|sJ d|| j	d ksJ dt|tu r|\}}}q|	d dkr|||| j 7 }q||||r| jnd 7 }q|S | j
tjkrt	 dd }
dd }dd }t| jdkr|
| jnd}t|D ]M\}	\}}|	dkr>|s2J d|| j	d ks>J dt|tu rL|\}}}q |	d dkr^|||| j 7 }q ||||rh| jnd 7 }q | }|S | j
tjkr| j| jg}| j}t|D ]&\}	\}}|rt|tu r|\}}}||||	d   7 }q|d7 }q|S | j
tjkr| j| j | j }|D ])\}}|rt|tu r|\}}}||d | d | j 7 }q||d 7 }q|S td| j
 )Nr      r    mmtagz<Image><image></Image>z	Received.z<image>
z: :    c                 S      d|  dS )Nz<<SYS>>
z
<</SYS>>

r   msgr   r   r	   r
   j       z)Conversation.get_prompt.<locals>.<lambda>c                 S   s   |  | rd S d S )N
r'   r   r-   r   r   r	   r
   l   s    c                 S   r,   )Nz[INST] z [/INST]r   r-   r   r   r	   r
   m   r/   z first message should not be nonez#first message should come from userc                 S   
   d|  S )Nz,<|start_header_id|>system<|end_header_id|>

r   r-   r   r   r	   r
         
 c                 S   r1   )Nz*<|start_header_id|>user<|end_header_id|>

r   r-   r   r   r	   r
      r2   c                 S   r1   )Nz/<|start_header_id|>assistant<|end_header_id|>

r   r-   r   r   r	   r
      r2   z<|begin_of_text|>c                 S   r,   )Nz<|im_start|>system

<|im_end|>r   r-   r   r   r	   r
      r/   c                 S   s   d|    dS )Nz<|im_start|>user
r3   )stripr-   r   r   r	   r
      s    c                 S   r,   )Nz<|im_start|>assistant
r3   r   r-   r   r   r	   r
      r/   r0   zInvalid style: )r   lentypetuplecopyreplacer4   r$   insertr   r   r   r   r   r!   r   r"   	enumerater   r   DEFAULT_BOS_TOKENlstripr   r   r   r   
ValueError)selfr   	init_roleinit_msgretrolemessage_sepsiwrap_sys	wrap_inst	wrap_userwrap_assistantr   r   r	   
get_promptF   s    


t
h



K
 0
 

zConversation.get_promptc                 C   s   | j ||g d S )N)r   append)r?   rC   rD   r   r   r	   append_message   s   zConversation.append_messagec                    st  g }t | j| jd  D ]\}\}}|d dkrt|tu rdd l}ddlm} ddlm	  |\}}}	|	dkrCd fdd	}
|
|}n|	d	krHn|	d
krR|
d}ntd|	 t|jt|j}}|| }d\}}tt|| ||}t|| }|j\}}||kr||}}n||}}|
||f}|r|| q| }|j|dd ||  }|| q|S )Nr*   r   BytesIOImagePadz   t   h   c                    s~   | j \}}||kr| S ||kr' | j||f|}|| d|| d f |S  | j||f|}|| || d df |S )Nr   r*   )sizenewmodepaste)pil_imgbackground_colorwidthheightresultrQ   r   r	   expand2square   s   
z.Conversation.get_images.<locals>.expand2squareCropResize)P  rd   zInvalid image_process_mode: i   i  JPEGformat)rT   )r;   r   r   r6   r7   base64iorP   PILrR   resizer>   maxrX   minintrM   save	b64encodegetvaluedecode)r?   
return_pilimagesrG   rC   r.   ri   rP   imageimage_process_modera   max_hwmin_hwaspect_ratiomax_lenmin_lenshortest_edgelongest_edgeWHbufferedimg_b64_strr   rQ   r	   
get_images   sD    




zConversation.get_imagesc                 C   s6  g }t | j| jd  D ]\}\}}|d dkrt|tu rdd l}ddlm} |\}}}t|j	t
|j	}	}
|	|
 }d\}}tt
|| ||
}t|| }|j	\}}||kr_||}}n||}}|||f}| }|j|dd ||  }d| d}|d	|}||d g q||d
 d
< q|S )Nr*   r   rO   re   rf   rg   z <img src="data:image/png;base64,z" alt="user upload image" />r   )r;   r   r   r6   r7   ri   rj   rP   rm   rX   rn   ro   rl   rp   rq   rr   rs   r9   rM   )r?   rB   rG   rC   r.   ri   rP   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   img_strr   r   r	   to_gradio_chatbot	  s2    


zConversation.to_gradio_chatbotc              
   C   s2   t | j| jdd | jD | j| j| j| j| jdS )Nc                 S   s   g | ]\}}||gqS r   r   .0xyr   r   r	   
<listcomp>,  s    z%Conversation.copy.<locals>.<listcomp>r   r   r   r   r   r!   r"   r$   )	r   r   r   r   r   r   r!   r"   r$   r?   r   r   r	   r8   (  s   zConversation.copyc                 C   sV   t |  dkr| j| jdd | jD | j| j| jdS | j| j| j| j| j| jdS )Nr   c                 S   s,   g | ]\}}|t |tu r|d  n|gqS )r   )r6   r7   r   r   r   r	   r   9  s   , z%Conversation.dict.<locals>.<listcomp>)r   r   r   r   r!   r"   )r5   r   r   r   r   r   r!   r"   r   r   r   r	   dict4  s   	zConversation.dict)F)r   r   r   r   str__annotations__r   ro   r   r   r   r!   r"   r$   r%   boolrL   rN   r   r   r8   r   r   r   r   r	   r   8   s$   
  
5r   zA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.

)User	Assistantnvgptr   zSystem
)r   r   r$   r   r   r   r!   r"   r0   nv_dpor'   )user	assistantz1.5)r   r   r$   r   r   r   r!   zA chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.)Humanr   ))r   zPWhat are the key differences between renewable and non-renewable energy sources?)r   ag  Renewable energy sources are those that can be replenished naturally in a relatively short amount of time, such as solar, wind, hydro, geothermal, and biomass. Non-renewable energy sources, on the other hand, are finite and will eventually be depleted, such as coal, oil, and natural gas. Here are some key differences between renewable and non-renewable energy sources:
1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable energy sources are finite and will eventually run out.
2. Environmental impact: Renewable energy sources have a much lower environmental impact than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, and other negative effects.
3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically have lower operational costs than non-renewable sources.
4. Reliability: Renewable energy sources are often more reliable and can be used in more remote locations than non-renewable sources.
5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different situations and needs, while non-renewable sources are more rigid and inflexible.
6. Sustainability: Renewable energy sources are more sustainable over the long term, while non-renewable sources are not, and their depletion can lead to economic and social instability.
r*   r    )r   r   r   r   r   r!   zA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.)USER	ASSISTANTv1r+   a  You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.llama_v2zYou are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.llama_v3z
<|eot_id|>)r'   r'   ))r   zHi!)r   z#Hi there! How can I help you today?a7  A chat between a curious user and an artificial intelligence assistant. The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.The visual content will be provided with the following format: <Image>visual content</Image>.v0_mmtag)r   r   r   r   r   r!   r$   v1_mmtagr   mistraldefaultv0	vicuna_v1llama_2plainv0_plainllava_v0llava_v1llava_llama_2
nv_steerlmyi_34b__main__)5dataclassescollectionsr   enumr   r   typingr   DEFAULT_PAD_TOKENr<   DEFAULT_EOS_TOKENDEFAULT_UNK_TOKENDEFAULT_IMAGE_TOKENDEFAULT_VIDEO_TOKENDEFAULT_SYSTEM_TOKENDEFAULT_SEPARATOR_TOKENDEFAULT_LABELS_TOKENDEFAULT_IMAGE_PATCH_TOKENDEFAULT_IM_START_TOKENDEFAULT_IM_END_TOKENDEFAULT_VID_START_TOKENDEFAULT_VID_END_TOKENTIME_TOKEN_TEMPLATEr   	dataclassr   r   
conv_nvgptconv_nv_dpor   conv_yi_34br   conv_vicuna_v0r   conv_vicuna_v1r   conv_llama_2conv_llava_llama_2r   conv_llava_llama_3r   conv_llava_plainconv_llava_v0conv_llava_v0_mmtagconv_llava_v1conv_llava_v1_mmtagr   conv_mistraldefault_conversationconv_templatesr   printrL   r   r   r   r	   <module>   s    
!	
	

