o پi@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddlm Z ddl m Z ddl mZmZddlZddlZddlmZddlmZddlmZdd lmZdd lmZdd lmZmZdd lm Z m!Z!dd l"m#Z#m$Z$ddl%m&Z&m'Z'e$e(Z)dede*e+effddZ,d(dede-dBdefddZ.Gddde-e Z/Gddde-e Z0Gddde-e Z1ej2GdddZ3ej2GdddZ4da5d e6e-de3fd!d"Z7d#e3fd$d%Z8de3fd&d'Z9dS))z,The arguments of sglang-diffusion Inference.N)field)Enum)AnyOptional)envs)PipelineConfig)NunchakuSVDQuantArgs)NunchakuConfig) BYTES_PER_GB)AttentionBackendEnumcurrent_platform)is_port_availableis_valid_ipv6_address)configure_logger init_logger)FlexibleArgumentParser StoreBooleanobjreturncCs0z ddl}t||j|fWStyYdSw)zWReturn (is_tensor, torch_module_or_None) without importing torch at module import time.rN)FN)torch isinstanceTensor Exception)rrr]/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/runtime/server_args.py_is_torch_tensor/s  rkey_hintc Cs|dus t|ttttfr|St|tr|jSt|\}}|rzm| }|dkrt |j t|j d}z t| |d<Wn tyJYnwz t||d<Wn ty`Ynwzt||d<Wn tyxYnwddi|WSdt |j t|j d WStyYd Swt|ri}t|D]'}|jsq|j}d |vrqzt||} Wn tyYqwt| |d ||<q|St|tri} |D]#\} } zt| } Wn tyd } Ynw| dkrqt| | d | | <q| St|t ttfrdd|DSz,t|st|r9t|dd}t|dt|dd}|r6|d|WS|WSWn tyDYnwzt|WStyUYdSw)aRecursively convert objects to JSON-serializable forms for concise logging. Rules: - Drop any field/dict key named 'param_names_mapping'. - Render Enums using their value. - Render torch.Tensor as a compact summary; if key name is 'scaling_factor', include stats. - Dataclasses are expanded to dicts and sanitized recursively. - Callables/functions are rendered as their qualified name. - Fallback to str(...) for unknown types. Nscaling_factor)shapedtypeminmaxmeantensorT)r#rrz names_mappingrzparam_names_mappingcSsg|]}t|qSr)_sanitize_for_logging).0xrrr sz)_sanitize_for_logging.. __module__ __qualname____name__z .z) rstrintfloatboolrvaluerdetachcpulistrrr itemrr!r" dataclasses is_dataclassfieldsreprnamegetattrr'dictitemstuplesetinspect isroutineisclass)rr is_tensor torch_modtenstatsresultfr=r4 result_dictkvkey_strmoduleqnrrrr'9s               r'c@s@eZdZdZdZededdfddZedeefddZ d S) ExecutionModez Enumeration for different pipeline modes. Inherits from str to allow string comparison for backward compatibility. inferencer4rc CDz||WSty!td|dddd|Ddw)z%Convert string to ExecutionMode enum.zInvalid mode: . Must be one of: , cSg|]}|jqSrr4r(mrrrr*z-ExecutionMode.from_string..Nlower ValueErrorjoinclsr4rrr from_string zExecutionMode.from_stringcCdd|DS)2Get all available choices as strings for argparse.cSrWrrX)r(moderrrr*r[z)ExecutionMode.choices..rrarrrchoiceszExecutionMode.choicesN) r.r+r-__doc__ INFERENCE classmethodr0rbr7rhrrrrrRs rRc@sLeZdZdZdZdZdZdZede ddfdd Z ede e fd d Z d S) WorkloadTypez Enumeration for different workload types. Inherits from str to allow string comparison for backward compatibility. i2vt2vt2ii2ir4rc CrT)z$Convert string to WorkloadType enum.zInvalid workload type: rUrVcSrWrrXrYrrrr*r[z,WorkloadType.from_string..Nr\r`rrrrbrczWorkloadType.from_stringcCrd)recSrWrrX)r(workloadrrrr*r[z(WorkloadType.choices..rrgrrrrhrizWorkloadType.choicesN) r.r+r-rjI2VT2VT2II2Irlr0rbr7rhrrrrrms rmc@sHeZdZdZdZdZdZededdfddZ ede efd d Z d S) Backenda Enumeration for different model backends. - AUTO: Automatically select backend (prefer sglang native, fallback to diffusers) - SGLANG: Use sglang's native optimized implementation - DIFFUSERS: Use vanilla diffusers pipeline (supports all diffusers models) autosglang diffusersr4rc CrT)zConvert string to Backend enum.zInvalid backend: rUrVcSrWrrXrYrrrr*r[z'Backend.from_string..Nr\r`rrrrbrczBackend.from_stringcCrd)recSrWrrX)r(backendrrrr*r[z#Backend.choices..rrgrrrrhrizBackend.choicesN) r.r+r-rjAUTOSGLANG DIFFUSERSrlr0rbr7rhrrrrrws rwc @seZdZUeed<ejZeed<dZeed<dZ e j dBed<dZ ee eefBdBed<dZeeed<dZeed <dZedBed <d Zeed <dZeeed <dZeeed<dZeeed<dZeeed<d Zeed<d Zeed<dZeed<d Zeed<dZeeed<dZedBed<e e!ddZ"e!ed<dZ#edBed<dZ$edBed<dZ%eed<dZ&e'ed<dZ(edBed <dZ)e*edBed!<dZ+edBed"<dZ,edBed#<d$Z-e'ed%<dZ.edBed&<dZ/edBed'<dZ0edBed(<dZ1eed)<d*Z2eed+<dZ3eed,<dZ4eed-<dZ5eed.<dZ6e*eed/<dZ7edBed0<e e8ddZ9e8e:BdBed1<dZ;edBed2<d3Zeed7<d8Z?edBed9<d:Z@eed;<d<e e d?ZCe eefed@<e dAdBd?ZDe eefedC<dZEe'dBedD<dEZFeedF<eGdGefdHdIZHeGdGefdJdKZIdLdMZJdNdOZKdPdQZLdRdSZMdTdUZNdVdWZOdXdYZPdZd[ZQd\d]ZRd^d_ZSd`edGe eeffdadbZTdcddZUeVdeeWdGeWfdfdgZXdhdiZYeGdjdkZZ mdd6ednedoedGefdpdqZ[e\ ddre]j^dse*edBdGdfdtduZ_e\dve eefdGdfdwdxZ`eVdyedGe eeffdzd{Zae\dvedGdfd|d}ZbeVdre]j^dse*edGe eeffd~dZcddZdddZeddZfddZgdddZhdS) ServerArgs model_pathr{Nattention_backendattention_backend_configcache_dit_config nccl_portFtrust_remote_coderevisionnum_gpustp_size sp_degreeulysses_degree ring_degreedp_size dp_degreeenable_cfg_parallelhsdp_replicate_dimhsdp_shard_dimi dist_timeout)default_factoryr<pipeline_configpipeline_class_name lora_pathdefault lora_nickname? lora_scalevae_pathlora_target_modulesdit_cpu_offloaddit_layerwise_offloaddit_offload_prefetch_sizetext_encoder_cpu_offloadimage_encoder_cpu_offloadvae_cpu_offloaduse_fsdp_inferenceTpin_cpu_memory comfyui_modeenable_torch_compilewarmupwarmup_resolutionsdisable_autocastnunchaku_config master_port 127.0.0.1hosti0uportwebuii0 webui_portscheduler_portzoutputs/ output_pathprompt_file_path)r model_pathscCsddddddddS)NT) transformervae video_vae audio_vae video_dit audio_ditdual_tower_bridgerrrrrYszServerArgs. model_loadedboundary_ratioinfo log_levelrcCs |jdS)Nr)rselfrrr broker_portms zServerArgs.broker_portcCs|jdup |jduS)z If no server is running when a generation task begins, 'local_mode' will be enabled: a dedicated server will be launched N)rrrrrr is_local_modeqszServerArgs.is_local_modecCsD||||||||dS)z"set defaults and normalize values.N)_adjust_offload_adjust_quant_config_adjust_warmup_adjust_network_ports_adjust_parallelism_adjust_attention_backend_adjust_platform_specific_adjust_autocastrrrr_adjust_parametersxs zServerArgs._adjust_parameterscCs$||||dS)z6check consistency and raise errors for invalid configsN)_validate_pipeline_validate_offload_validate_parallelism_validate_cfg_parallelrrrr_validate_parameterss zServerArgs._validate_parameterscCs`|j}|dus t|trdS||jr|jsd|_dSt|jj|jj|jj|jjd|_dS)zvalidate and adjustN) precisionrank act_unsignedquantized_model_path) rrr validateenable_svdquantrquantization_precisionquantization_rankquantization_act_unsigned)rncfgrrrrs   zServerArgs._adjust_quant_configcCsttdkr1td|jdurd|_|jdurd|_|jdur%d|_|jdur/d|_dSdS|j j r`td|jdurDd|_|jdurLd|_|jdurTd|_|jdur^d|_dSdS|jdurhd|_|jdurpd|_|jdurxd|_|jdurd|_dSdS)Nz6Enabling all offloading for GPU with low device memoryTzODisabling some offloading (except dit, text_encoder) for image generation modelF) r get_device_total_memoryr loggerrrrrrr task_type is_image_genrrrrrsB                 zServerArgs._adjust_offloadcCs|jdvrd|_|jdurt|_nt|jtr#t||j|_|jdkrC|jdur6|jdvr6td|jdurCd|_t d|jdurT|j t j krV|dSdSdS)N)fa3fa4far)r sage_attnzVRing Attention is only supported for flash attention or sage attention backend for nowzRing Attention is currently only supported for flash attention or sage attention; attention_backend has been automatically set to flash attention)rraddictDictrr0_parse_attention_backend_configrr^rrr{rwr~_set_default_attention_backendrrrrrs*        z$ServerArgs._adjust_attention_backendcCs(|jdurd|_|jrtddSdS)NTzCWarmup enabled, the launch time is expected to be longer than usual)rrrrrrrrrs zServerArgs._adjust_warmupcCsl||j|_|j|jdkrtddnd}|||_|jdur%|jndtdd}||d|_dS)Nrrdi5u%) settle_portrrrandomrandintr)rinitial_scheduler_portinitial_master_portrrrrs  z ServerArgs._adjust_network_portscCs|jdurd|_|jdur|j|_|jdur4|j|j}|jr#|d9}|j|dkr1|j||_nd|_|jdurQ|jdurQ|jdkrQ|j|_t d|jd|jdurbd|_t d|j|jdurud|_t d|jdSdS)Nrrz+Automatically set ulysses_degree=sp_degree=z for best performancez,Ulysses degree not set, using default value z)Ring degree not set, using default value ) rrrrrrrrrrdebug)rnum_gpus_per_grouprrrrs6           zServerArgs._adjust_parallelismcCs|tr d|_d|_tjs6|jjj }d|vsd|vr:|jdur8t rtd?d9|jd@tdAd9|jdBtdCd9|jdDtdEd9|jdFtdGd9|jdHtdId9t||jdJt tjdKd|jdLt tjdMd|jdNttjdOd|jdPt tjdQd|jdRttjdSd|jdTt tj dSd|jdUttj!dVd|jdWttj"dXd|jdYttj#dZd|jd[ttj$d\dt%||jd]ttj&d^d|jd_tt'(tj)j*d`da|S)bNz --model-pathzTThe path of the model weights. This can be a local folder or a Hugging Face repo ID.)typehelpz --vae-pathzzCustom path to VAE model (e.g., for distilled autoencoder). If not specified, VAE will be loaded from the main model path.)r rr!z--attention-backendzThe attention backend to use. For SGLang-native pipelines, use values like fa, torch_sdpa, sage_attn, etc. For diffusers pipelines, use diffusers attention backend names such as flash, _flash_3_hub, sage, or xformers.z--attention-backend-configznConfiguration for the attention backend. Can be a JSON string, a path to a JSON/YAML file, or key=value pairs.z--diffusers-attention-backendr)r destrr!z--cache-dit-configzNPath to a Cache-DiT YAML/JSON config. Enables cache-dit for diffusers backend.z--trust-remote-codez1Trust remote code when loading HuggingFace models)actionrr!z --revisionzPThe specific model version to use (can be a branch name, tag name, or commit id)z --num-gpuszThe number of GPUs to use.z --tp-sizez= 1, it is treated as the absolute number of layers. 0.0 means prefetch 1 layer (lowest memory). Values above 0.5 might have peak memory close to no offload but worse performance.z--use-fsdp-inferencezwUse FSDP for inference by sharding the model weights. Latency is very low due to prefetch--enable if run out of memory.z--text-encoder-cpu-offloadz>Use CPU offload for text encoder. Enable if run out of memory.z--image-encoder-cpu-offloadz?Use CPU offload for image encoder. Enable if run out of memory.z--vae-cpu-offloadz5Use CPU offload for VAE. Enable if run out of memory.z--pin-cpu-memoryzPin memory for CPU offload. Only added as a temp workaround if it throws "CUDA error: invalid argument". Should be enabled in almost all casesz--disable-autocastzIDisable autocast for denoising loop and vae decoding in pipeline samplingz --master-portzSMaster port for distributed inference. If not set, a random free port will be used.z--scheduler-portzPort for the scheduler server.z--hostzHost for the HTTP API server.z--portzPort for the HTTP API server.z--webuiz'Whether to use webui for better displayz --webui-portz --output-pathz.Directory path to save generated images/videosz --lora-pathzYThe path to the LoRA adapter weights (can be local file path or HF hub id) to launch withz--lora-nicknamez0The nickname for the LoRA adapter to launch withz --lora-scalezRLoRA scale for merging (e.g., 0.125 for Hyper-SD). Same as lora_scale in Diffusersz --log-levelz!The logging level of all loggers.z --backendzThe model backend to use. 'auto' prefers sglang native and falls back to diffusers. 'sglang' uses native optimized implementation. 'diffusers' uses vanilla diffusers pipeline.)r rhrr!)+ add_argumentr0rrargparseSUPPRESSrrrrr1rrrrrrrrrrrrr2rr add_cli_argsrrrrrrrrrrrrrwrhr{r4)rrrrr*js      zServerArgs.add_cli_argscCs2t|jrd|jd|jSd|jd|jS)Nzhttp://[z]:zhttp://:)rrrrrrrurls zServerArgs.urlcCs,|j}|dus |dkr d}d|d|jS)z Internal endpoint for scheduler. Prefers the configured host but normalizes localhost -> 127.0.0.1 to avoid ZMQ issues. N localhostrztcp://r+)rr)rscheduler_hostrrrscheduler_endpointszServerArgs.scheduler_endpoint*rport_inc max_attemptscCsd}|}||kr7t|r|dkrtd|d|d|S|d7}|dkr+||7}ndtdd}||kstd |d |d ) z: Find an available port with retry logic. rzPort z was unavailable, using port z insteadr`iz$Failed to find available port after z attempts (started from port ))r rrrr RuntimeError)rrr1r2attempts original_portrrrrs& zServerArgs.settle_portargs unknown_argscCsF|durg}|||}|d}|r||}i||}||S)Nr)get_provided_argsgetload_config_file from_dict)rar9r: provided_args config_file config_argsrrr from_cli_argss     zServerArgs.from_cli_argskwargscCsddt|D}i}|D]2}|dkr(t|}tdt|||d<q|dkr6t|}||d<q||vr@||||<q|di|S)z-Create a ServerArgs object from a dictionary.cSrWr)r=)r(attrrrrr*r[z(ServerArgs.from_dict..rzUsing PipelineConfig: rNr) r9r;r from_kwargsrrr rr>)rarCattrsserver_args_kwargsrDrrrrrr>s     zServerArgs.from_dictr@cCs|dr!t|d }t|WdS1swYdS|drTzddl}Wn ty7tdwt|d }||WdS1sMwYdStd|)zLoad a config file.rrNrrzDPlease install PyYAML to use YAML config files. `pip install pyyaml`z Unsupported config file format: )r r rrr  ImportErrorr r^)r@rKr rrrr=s  $    $zServerArgs.load_config_filecKsd|vrt|dtrt|d|d<d|vr(t|dtr(t|d|d<d|vr= and divisible by sp_degree (r5z2) must be >= and divisible by hsdp_replicate_dim (z.) must be >= and divisible by hsdp_shard_dim (z ) must be divisible by dp_size (rz"--dp-size must be a natural numberzDP is not yet supportedrz)) must be divisible by (dp_size * tp_sizez * 2r,z) = z sp_degree (z+) must equal ring_degree * ulysses_degree (z * z = rrzucache-dit is enabled with hybrid parallelism (SP + TP). Proceeding anyway (SGLang integration may support this mode).)rrr^rrrrrrrrgetenvr]rrZ)rrhas_sphas_tprrrr[s`          z ServerArgs._validate_parallelismcCs |jr |jdkrtddSdS)NrzICFG Parallelism is enabled via `--enable-cfg-parallel`, but num_gpus == 1)rrr^rrrrrs z!ServerArgs._validate_cfg_parallelcCs.trtjj}||_td|dSdS)zGConfigure ROCm defaults when users do not specify an attention backend.z\Attention backend not specified. Using '%s' by default on ROCm to match SGLang SRT defaults.N) r is_rocmr AITERr=r]rrr)rdefault_backendrrrrs z)ServerArgs._set_default_attention_backend)r0rr)rN)ir.r+r-r0__annotations__rwr|r{rrrrrr?rrrr1rr3rrrrrrrrrrrrrrrrrrrr2rrr7rrrrrrrrrrrrrrrr rrrrrrrrrrrrpropertyrrrrrrrrrrrrrr staticmethodrr*r,r/rrlr( NamespacerBr>r=rEr;rrrrrrrrrrs                    "  &'/       +9rc@s`eZdZUeed<eed<eed<eed<dZedBed<e d dede ed dfd d Z dS) PortArgsscheduler_input_ipc_namer rpc_ipc_namemetrics_ipc_nameNrrdp_rankrcCs|jdur#|jtdd} t|rn|dkr|d7}n|d8}qn|j}tdtjdd j|dtjdd jdtjdd j|j d S) Nrr4Tr3r0+zipc://F)delete)rfrrgrhr) rrrrr retempfileNamedTemporaryFiler=r)rrirrrrfrom_server_argss$  zPortArgs.from_server_argsr) r.r+r-r0rar1rrcrrrnrrrrres rerMcCs(t}t|||}t|}|S)zJ Prepare the inference arguments from the command line arguments. )rrr* parse_argsrB)rMrraw_argsrrrrprepare_server_argss    rqrcCs|adS)z> Set the global sgl_diffusion config for each process N)_global_server_argsrrrrset_global_server_argssrscCstdurtdtS)Nz%Global sgl_diffusion args is not set.)rrr^rrrrget_global_server_argssrtr):rjr(r9rCrrXrrrLrlrenumrtypingrrrr sglang.multimodal_genr3sglang.multimodal_gen.configs.pipeline_configs.baser*sglang.multimodal_gen.configs.quantizationrIsglang.multimodal_gen.runtime.layers.quantization.configs.nunchaku_configr *sglang.multimodal_gen.runtime.loader.utilsr 'sglang.multimodal_gen.runtime.platformsr r *sglang.multimodal_gen.runtime.utils.commonr r1sglang.multimodal_gen.runtime.utils.logging_utilsrrsglang.multimodal_gen.utilsrrr.rrAr3rr0r'rRrmrw dataclassrrerrr7rqrsrtrrrrsZ        b@*