o پim@sddlZddlmZmZmZmZddlZddlmZddl m Z ddl m Z m Z ddlmZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZdd lmZmZm Z ddl!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*e+e,Z-GdddZ.dS)N)DictIterableListOptional) LoadConfig) get_layer_id)ParallelLMHeadVocabParallelEmbedding)BaseLoRABackend)get_backend_from_name)BaseLayerWithLoRAget_lora_layer) LoRAAdapter) LoRAConfig)LoRARef)LoRAMemoryPool)LoRATypeget_normalized_target_modulesget_target_module_name)LoRAUpdateOutput) ForwardBatch) ServerArgs)replace_submodule) AutoConfigc@seZdZ      dFdejjdededed ej d e d e d ed ede ede e e de eefddZdedefddZ dGdede defddZdedefddZd edefd!d"Zdedefd#d$Zd%ee e defd&d'Zefd(ee e d)ee e fd*d+Zd,efd-d.Zd/d0Z   dHde ede e e de eefd1d2ZdIde eefd3d4Z  dJde ede e e fd5d6Z!defd7d8Z"ded9e#e ej$ffd:d;Z% dIded9e#e ej$fdd?Z&d@dAZ'dBdCZ(dDdEZ)dS)K LoRAManagertritonrN base_modelbase_hf_configmax_loras_per_batch load_configdtype server_args lora_backendtp_sizetp_rank max_lora_ranktarget_modules lora_pathsc Cs||_||_||_||_||_t|jj|_||_| |_ d|_ |j |_ |j |_ td|dt|} | ||j|d|_|j| | | ddS)NzUsing z as backend of LoRA kernels.)rdevicer")r&r'r()rrrr r!next parametersr)r$r%lora_added_tokens_sizeenable_lora_overlap_loadinglora_eviction_policyeviction_policyloggerinfor r# init_state)selfrrrr r!r"r#r$r%r&r'r( backend_typer5P/home/ubuntu/.local/lib/python3.10/site-packages/sglang/srt/lora/lora_manager.py__init__3s0 zLoRAManager.__init__max_bs_in_cuda_graphnum_tokens_per_bscCs||_|jj||ddS)N)r8r9)r8r#init_cuda_graph_batch_info)r3r8r9r5r5r6r:bs  z&LoRAManager.init_cuda_graph_batch_infosuccess error_messagereturncCst||dd|jDdS)NcSsi|]}|j|jqSr5) lora_name lora_path).0lora_refr5r5r6 qsz9LoRAManager.create_lora_update_result..)r<r=loaded_adapters)r lora_refsvalues)r3r<r=r5r5r6create_lora_update_resultksz%LoRAManager.create_lora_update_resultrBc Cs|jdur |jdusJd|j|jvsJd|jdz(t|j}|||||j|j<||||j|j<|j t |j 7_ Wnt y`}z|j dt|dWYd}~Sd}~ww|j ddS) z Load a single LoRA adapter from the specified path. Args: lora_ref (LoRARef): The LoRARef object containing the LoRA name, path, and ID. N?LoRARef must have both lora_name and lora_path set for loading.LoRA adapter with ID Y is already loaded. This should have been verified before request is sent to the backend.Fr<r=Tr<)r?r@lora_idlorasrvalidate_new_adapterconfigsload_lora_weightsrEnum_pinned_lorasintpinned ExceptionrGstr)r3rB new_adapterer5r5r6load_lora_adapterws*       zLoRAManager.load_lora_adapter lora_configcCs|jdkr td|jD]'}|j|jkrtd|jd|j|jkr5t|jd|jd|jqt|dd}|oC| | }|rStd |jd |j d |j rg|j |j d kritd|jd dSdS)z Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible. rzQLoRA serving currently doesn't support adapters that add tokens to the vocabularyFailed to load LoRA adapter z because it is already loadedz is already loaded with name: z., but another copy is being loaded with name: memory_poolNz LoRA adapter z with rank z is incompatible with the current LoRA memory pool configuration. Please ensure that the LoRA adapter's rank is within the configured `--max-lora-rank` and that the target modules are included in `--lora-target-modules`.rz as a pinned adapter. It is not allowed to pin all slots in the LoRA memory pool to avoid starvation for unpinned adapters and base models. Please increase your `--max-loras-per-batch` or load it as unpinned LoRA adapters.)r, ValueErrorrErFr?r@r0warninggetattr can_supportrrTrRr)r3rZrBexisting_lora_refr\ incompatibler5r5r6rOs6      z LoRAManager.validate_new_adapterc Cs|j|j}|j|j}|dur|dusJd|jdz|j|j=|j|j=|j|j=|jt|j8_WntyU}z|j dt |dWYd}~Sd}~ww|j ddS)z Unload LoRA adapters by their names. This will remove the adapters from the memory pool and delete the corresponding LoRA modules. NrIzU is not loaded. This should have been verified before request is sent to the backend.FrKTrL) rPgetrMrErNrRrSrTrUrGrV)r3rBadapterrXr5r5r6unload_lora_adapters$     zLoRAManager.unload_lora_adapterlora_idscCst||jkr dS|jdkrdSd}|D]}|dur3|j|}|dus,Jd|d|t|j7}q||jksEJd|d|jd t||}|jj|j}||kS) zh Validate if the LoRA IDs in the batch can be loaded into the current LoRA memory pool. FrTNzLoRA ID z not found in lora_refs.z-Number of pinned LoRA adapters in the batch (z/) exceeds the total number of pinned adapters (z2). This indicates a bug in the LoRA loading logic.)lenrrRrErdrSrTr\)r3rgpinned_loras_in_batchrMrBrequired_slotsmem_pool_vacancyr5r5r6validate_lora_batchs*      zLoRAManager.validate_lora_batch new_loras running_lorascCsD||B}t||jks J|jj||j|j|j|j|j ddS)N)cur_uids lora_adapters lora_modulesrElora_embed_tokens_modulelora_lm_head_module) rhrr\prepare_lora_batchrNrqrEcopyembed_tokens_modulelm_head_module)r3rmrnror5r5r6fetch_new_lorass zLoRAManager.fetch_new_loras forward_batchc Cs|j}t|do||jko|j}dgt|j}dg|j}dg|j}t|jD]$\}}|j |||<|durO|j |} | j j |||<| j|||<q+|jj|||||ddS)Nr8r)ryweight_indices lora_ranksscalingsuse_cuda_graph) batch_sizehasattrr8 forward_mode is_cuda_graphrhrgr enumerater\ get_buffer_idrNconfigrascalingr#rt) r3rybsr}rzr{r|iuidlorar5r5r6rt s.     zLoRAManager.prepare_lora_batchc Cst|jD]*\}}|D]!\}}t||jj}||jj||tj d|jj||tj dq q|j durO|j |j dtj |j dtj |j dtj |j duri|j |j dtj |j dtj dSdS)zZ Update all LoRA modules to associate them with the latest memory buffer. ) target_modulelayer_id lora_typeN added_tokens embed_tokenslm_head)rrqitemsrr\r' set_lora_info get_tensorrLORA_ALORA_Brvget_embedding_tensorrw)r3r layer_modules module_namemodulerr5r5r6update_lora_info&s<  zLoRAManager.update_lora_infocCsP|s|dur |dusJd|||j||d|||dS)z Initialize the internal (mutable) state of the LoRAManager. When `lora_paths` is provided and not empty, it might be used for inferring LoRA shape info such as the target modules and max_lora_rank. NzWhen no initial --lora-paths is provided, you need to specify both --max-lora-rank and --lora-target-modules for LoRA initialization.)r&r')init_lora_adaptersinit_lora_shapesinit_lora_modulesinit_memory_poolr)r3r&r'r(r5r5r6r2Ks   zLoRAManager.init_statecCsVi|_i|_i|_d|_|r'|D]}||}|js&td|jd|jqdSdS)Nrr[z: ) rPrNrErRrYr< RuntimeErrorr?r=)r3r(rBresultr5r5r6res zLoRAManager.init_lora_adaptersc Cs|rt|nt|_|jD]u\}}t|jtr>|jdvr5|dur#q|j|j}t d|d|jdt d|jdt|jt sHt dt|j}|dur~| |js}||j}|j|j}t d|d t |d t |jd t |j|Bd q|j |q|dur||_ntd d|jDdd|_|jdurtdd|jDd}|dkrtd|d||_dSdS)zQInfer LoRA target modules and max_lora_rank from loaded adapters if not provided.)z all-linearallNzLoRA adapter 'z' uses target_modules='z' which cannot be resolved automatically. Please explicitly specify --lora-target-modules during server startup. You can specify 'all' to enable all supported module types.z*SGLang does not recognize target_modules='z'. Please use a list of module name suffixes in the adapter's PEFT config, or explicitly specify --lora-target-modules during server startup.aSGLang currently only supports inferring LoRA target modules when a list of suffixes is provided in `target_modules` field of PEFT config. Please explicitly specify `--lora-target-modules` during server startup. You can specify `all` to enable all support modules types. z' contains target modules z> that are not included in the specified --lora-target-modules zG. Please update --lora-target-modules to include all required modules: z/, or use 'all' to enable all supported modules.cSsg|]}|jqSr5)rarAxr5r5r6 sz0LoRAManager.init_lora_shapes..r)defaultcss |] }|jdkr|jVqdS)rN)r,rr5r5r6 s z/LoRAManager.init_lora_shapes..zself.lora_added_tokens_size=z from LoRA adapters.)rsetr'rPr isinstancerVrEr?r]listissubsetsortedupdater&maxrFr,r*r0r1) r3r&r'rMrr?adapter_target_modulesunsupported_modulesinferred_extra_vocab_sizer5r5r6rzsr            zLoRAManager.init_lora_shapescCsFt|j|j|j|j|j|j}||jr|||j |j<dS)zh Load the weights of a LoRA adapter to CPU memory and conducts post-loading validation. N) rrMrPrr r#initialize_weightsr-pin_weights_in_cpurN)r3rB lora_adapterr5r5r6rQs zLoRAManager.load_lora_weightstensorscCs:t|j|j|j|j|j|j}||||j|j<dS)zP Load the weights of a LoRA adapter from tensors to CPU memory. N)rrMrPrr r#initialize_weights_from_tensorsrN)r3rBrrr5r5r6load_lora_weights_from_tensorss  z*LoRAManager.load_lora_weights_from_tensors config_dictadded_tokens_configc Cs|jdur |jdusJd|j|jvsJd|jdz*t||}|||||j|j<|||||j |j<|j t |j 7_ Wnt yb}z|jdt|dWYd}~Sd}~ww|jddS) zJ Load a single LoRA adapter from tensors and config dict. NrHrIrJFrKTrL)r?r@rMrNr from_dictrOrPrrErRrSrTrUrGrV)r3rBrrrrWrXr5r5r6load_lora_adapter_from_tensorss*        z*LoRAManager.load_lora_adapter_from_tensorsc CsBt|j|j|j|j|j|j|j|j|j |j d |_ | dhdS)zH(Re)initialize the LoRA memory pool based on the current configurations.) rrr!r$r%r&r'rr/r,N) rrrr!r$r%r&r'rr/r,r\rx)r3r5r5r6rszLoRAManager.init_memory_poolcCst||j}t|j|||SN)r r#rr)r3rr lora_moduler5r5r6set_lora_module-s zLoRAManager.set_lora_modulec Csddt|jjD|_d|_d|_d|jvrYt|jdd}d}|j D] \}}| dr2|}nq%|durY|durY||urYt dt |j|j|jj|jd}|j|_||j_|j D]e\}}t|jddrp|j|spq^d|vrd|jvrt|trt|ts|||}||_q^d|vrd|jvrt|t rt|ts|||}||_q^|dd |jvrt|} ||||j| |<q^dS) NcSsg|]}iqSr5r5)rA_r5r5r6r4sz1LoRAManager.init_lora_modules..rrzslm_head is tied with embed_tokens. Creating a separate ParallelLMHead that shares the base weight for LoRA support.)num_embeddings embedding_dim params_dtypeorg_num_embeddingsshould_apply_lora.)rangernum_hidden_layersrqrvrwr'r_r named_modulesendswithr0r1rorg_vocab_sizerweightr!rrrr r rsplitr) r3rrnamemoduntied_lm_headrrrrr5r5r6r2sn        zLoRAManager.init_lora_modules)rrrNNN)r;)NNNr)NN)*__name__ __module__ __qualname__torchnnModulerrSrr!rrVrrrrr7r:boolrrGrYrrOrfrrlrxrrtrr2rrrQrTensorrrrrrr5r5r5r6r2s      /    "(    '     \    " r)/loggingtypingrrrrrsglang.srt.configs.load_configrsglang.srt.layers.utilsr*sglang.srt.layers.vocab_parallel_embeddingrr $sglang.srt.lora.backend.base_backendr %sglang.srt.lora.backend.lora_registryr sglang.srt.lora.layersr r sglang.srt.lora.lorarsglang.srt.lora.lora_configrsglang.srt.lora.lora_registryrsglang.srt.lora.mem_poolrsglang.srt.lora.utilsrrrsglang.srt.managers.io_structr,sglang.srt.model_executor.forward_batch_inforsglang.srt.server_argsrsglang.srt.utilsr&sglang.srt.utils.hf_transformers_utilsr getLoggerrr0rr5r5r5r6s*