o
    پi	                  
   @   s   d dl mZmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
 zd dlZW n ey= Z zeZde_W Y dZ[ndZ[ww G dd	 d	eZdS )
    )MappingOptional)BaseBackend)get_chat_template_by_model_path)StreamExecutor)SglSamplingParamsN   c                       s   e Zd Zdddddejdfdee dee dee dee dee	eef  f
 fdd	Z
d
d ZdedefddZdedefddZ  ZS )LiteLLMNiX  organizationbase_urltimeoutmax_retriesdefault_headersc	           	         sD   t    tttrt|| _|pt|| _||||||d| _d S )N)api_keyr
   r   r   r   r   )	super__init__
isinstancelitellm	Exception
model_namer   chat_templateclient_params)	selfr   r   r   r
   r   r   r   r   	__class__ O/home/ubuntu/.local/lib/python3.10/site-packages/sglang/lang/backend/litellm.pyr      s   

zLiteLLM.__init__c                 C   s   | j S )N)r   )r   r   r   r   get_chat_template/   s   zLiteLLM.get_chat_templatessampling_paramsc                 C   sT   |j r|j }nd|jdg}tjd| j|d| j| }|jd jj	}|i fS )Nuserrolecontent)modelmessagesr   r   )
	messages_text_r   
completionr   r   to_litellm_kwargschoicesmessager#   )r   r   r   r%   retcompr   r   r   generate2   s   zLiteLLM.generatec                 c   sp    |j r|j }nd|jdg}tjd| j|dd| j| }|D ]}|jd jj	}|d ur5|i fV  q#d S )Nr    r!   T)r$   r%   streamr   r   )
r&   r'   r   r(   r   r   r)   r*   deltar#   )r   r   r   r%   r,   chunktextr   r   r   generate_streamF   s&   
zLiteLLM.generate_stream)__name__
__module____qualname__r   num_retriesr   strfloatintr   r   r   r   r   r.   r3   __classcell__r   r   r   r   r	      s<    	
r	   )typingr   r    sglang.lang.backend.base_backendr   sglang.lang.chat_templater   sglang.lang.interpreterr   sglang.lang.irr   r   ImportErrorer7   r	   r   r   r   r   <module>   s    