o پid@s,dZddlZddlZddlZddlZddlmZddlmZddl Z ddl Z ddl Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd l m!Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)e)e*Z+Gddde!Z,GdddeZ-e-Z.dS)z Diffusers backend pipeline wrapper. This module provides a wrapper that allows running any diffusers-supported model through sglang's infrastructure using vanilla diffusers pipelines. N)BytesIO)Any)DiffusionPipeline)Image)PipelineConfig)get_local_torch_device)ComposedPipelineBase)PipelineExecutor) SyncExecutor)Req) PipelineStage)AttentionBackendEnum) ServerArgs)maybe_download_model) init_loggerc seZdZdZdeffdd Zdededefdd Zd d d e d e de e e e ffddZdedejdBfddZdedejdBfddZde dejdBfddZdejdejfddZdejdejfddZdedede fddZde fdd ZdedejdBfd!d"ZZS)#DiffusersExecutionStagez7Pipeline stage that wraps diffusers pipeline execution.diffusers_pipecst||_dSN)super__init__r)selfr __class__n/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/runtime/pipelines/diffusers_pipeline.pyr/s  z DiffusersExecutionStage.__init__batch server_argsreturnc Cs|||}||\}}d|vrd|d<tVtjdd@tdz |jdi|}Wn(tyX}zdt |vrM| dd|jdi|}nWYd}~nd}~wwWdn1scwYWdn1srwY| ||_ |j dur| |j |_ |S)zExecute the diffusers pipeline. output_typeptT)recordalwaysNr)_build_pipeline_kwargs_filter_pipeline_kwargstorchno_gradwarningscatch_warnings simplefilterr TypeErrorstrpop_extract_outputoutput_postprocess_output)rrrkwargs_r-errrforward3s0      zDiffusersExecutionStage.forwardF)strictr/r3c Csz t|jj}Wnttfy|gfYSw|j}tdd|D}|r-|gfSt | dh}i}g}| D]\} } | |vrK| || <q>| | q>|rrt |jj} d| ddt|d} |rmt| t| ||fS)aFilter kwargs to those accepted by the pipeline's __call__. Args: kwargs: Arguments to filter strict: If True, raise ValueError on unsupported args; otherwise warn Returns: Tuple of (filtered_kwargs, ignored_keys) css|] }|jtjjkVqdSr)kindinspect Parameter VAR_KEYWORD).0prrr cs zBDiffusersExecutionStage._filter_pipeline_kwargs..rz Pipeline 'z' does not support: z, z". These arguments will be ignored.)r5 signaturer__call__ ValueErrorr) parametersanyvaluessetkeysitemsappendtype__name__joinsortedloggerwarning) rr/r3sigparamsaccepts_var_kwargsvalidfilteredignoredkv pipe_namemsgrrrr#Qs4      z/DiffusersExecutionStage._filter_pipeline_kwargsr-NcCsfdD])}t||s qt||}|durq||}|dur+td||j|j|SqtddS)z+Extract tensor output from pipeline result.)imagesframesvideosamplepred_original_sampleNz.Extracted output from '%s': shape=%s, dtype=%sz-Could not extract output from pipeline result)hasattrgetattr_convert_to_tensorrIdebugshapedtyperJ)rr-attrdataresultrrrr,s$    z'DiffusersExecutionStage._extract_outputracCst|tjr|St|tjr>t|}|dkr|d}|jdkr.| dddd}|S|jdkr<| ddddd}|St |d rIt |St|t rYt|dkrY||Sd S) z)Convert various data formats to a tensor.?o@rmodeN) isinstancer$Tensornpndarray from_numpyfloatmaxndimpermuterZTToTensorlistlen_convert_list_to_tensor)rratensorrrrr\s"        z*DiffusersExecutionStage._convert_to_tensorcCs0|d}t|trt|dkr|}|d}t|dr8dd|D}t|}t|dkr4|ddddS|dSt|tjrUt|}t|dkrQ|ddddS|dSt|tj rdd|D}|d d krqd d|D}|dj dkrd d|D}t|}t|dkr|ddddS|dSd S) z$Convert a list of items to a tensor.rrjcSsg|]}t|qSr)rtru)r8imgrrr szCDiffusersExecutionStage._convert_list_to_tensor..rgrhrfcSsg|] }t|qSr)r$rorp)r8arrrrrr{rccSsg|]}|dqS)rdrr8trrrr{scSsg|] }|dddqS)rhrrg)rsr~rrrr{r}N) rkrvrwrZr$stackrsrlrmrnrqrr)rrafirsttensorsstackedrrrrxs4         z/DiffusersExecutionStage._convert_list_to_tensorcCs|}t|st|r"tdtj|dddd}| | }}|dks7|dkr=|dd }| d d}| |}td |j|S) zDPost-process output tensor to ensure valid values and correct shape.z)Output contains invalid values, fixing...g?rcg)nanposinfneginfgg?rgrhrzFinal output tensor shape: %s)cpurpr$isnanr?isinfrIrJ nan_to_numminitemrqclamp_fix_output_shaper]r^)rr-min_valmax_valrrrr.s     z+DiffusersExecutionStage._postprocess_outputcCs|dkr|dddddS|dkr1|jddks#|jddvr%|S|ddddddS|dkr_|j\}}}|dkrL|dkrL|ddd}|jddkrZ|ddd}|dS|dkrr|dddddS|S)z~Fix tensor shape for downstream processing. Expected: (B, C, H, W) for images or (B, C, T, H, W) for videos. rirrhrgrfre)rgrfre)dimrsr^ unsqueezerepeat)rr-chwrrrrs      z)DiffusersExecutionStage._fix_output_shapecCs>i}|jdur |j|d<|jr|j|d<|jdur|j|d<|jdur(|j|d<|jdur2|j|d<|jdur<|j|d<|jdurF|j|d<|jdurU|jd krU|j|d <|jdur`|j|d <n|j duru| }t j |d  |j |d <||}|dur||d <|jd kr|j|d<|jr|jdi}|r|||S)z.Build kwargs dict for diffusers pipeline call.Npromptnegative_promptnum_inference_stepsguidance_scaletrue_cfg_scaleheightwidthrg num_frames generator)deviceimagenum_images_per_promptdiffusers_kwargs)rrrrrrrrrseed_get_pipeline_devicer$ Generator manual_seed_load_input_imagenum_outputs_per_promptextragetupdate)rrrr/rrrrrrr"s@                     z.DiffusersExecutionStage._build_pipeline_kwargsc CsZdD]!}t|j|d}|dur#z t|jWSty"Yqwqtjr+dSdS)z*Get the device the pipeline is running on.)unet transformervaeNcudar) r[rnextr>r StopIterationr$r is_available)rr` componentrrrr6s z,DiffusersExecutionStage._get_pipeline_devicec Cs|jdurt|jtjr|jS|jdurt|jtjr|jS|js#dSt|jtr/|jd|_z(|jdrNtj|jdd}| t t |j  dWSt |j dWStyr}ztd|j|WYd}~dSd}~ww)zLoad input image from batch.Nr)zhttp://zhttps://)timeoutRGBz Failed to load image from %s: %s)condition_imagerkr pixel_values image_pathrv startswithrequestsrraise_for_statusopenrcontentconvert ExceptionrIerror)rrresponser1rrrrAs.     z)DiffusersExecutionStage._load_input_image)rF __module__ __qualname____doc__rrr rr2dictbooltuplervr*r#rr$rlr,r\rxr.rr"rrr __classcell__rrrrr,s( .#3 rc@seZdZUdZdZdZgZeee d<   d5dede deedBde ee j jfdBd edBf d d Zdede d efd dZdede d dfddZdede d dfddZdede d efddZde d e jfddZddZ d6de de ee j jfdBd e eeffddZde fddZde fddZd7d d!Z d6d"ed#edBd dfd$d%Zed eefd&d'Z e !d(e"de d e"fd)d*Z#e$      d8ded+edBd,e jdBd-ee%BdBd.e&j'dBdeedBde ee j jfdBd dfd/d0Z(d6d1ed2ed efd3d4Z)dS)9DiffusersPipelinez Pipeline wrapper that uses vanilla diffusers pipelines. This allows running any diffusers-supported model through sglang's infrastructure without requiring native sglang implementation. F_required_config_modulesN model_pathrrequired_config_modulesloaded_modulesexecutorcCs`||_||_g|_i|_i|_i|_d|_|pt|d|_t d|| |||_ | dS)NF)rz"Loading diffusers pipeline from %s)rr_stages_stage_name_mappingmodules memory_usagespost_init_calledr rrIinfo_load_diffusers_pipeliner_detect_pipeline_type)rrrrrrrrrrjs  zDiffusersPipeline.__init__rc Cs|}t|}||_||}td|||j|jd}|j}|dur:t|dd}|dur:||d<tdt |j z t j |fi|}Wnt y} zRdt| vrtdzi|d|i} d | d <t j |fi| }Wn)ty} ztd t| } | r| d nd } td| d| | d} ~ wwWYd} ~ n=d} ~ wty} z-dt| vsdt| vrtd|| tj|d<t j |fi|}nWYd} ~ nd} ~ ww|t}|||||||||}td|jj |S)akLoad the diffusers pipeline. Optimizations applied: - device_map: Loads models directly to GPU, warming up CUDA caching allocator to avoid small tensor allocations during inference. - Parallel shard loading: When using device_map with accelerate, model shards are loaded in parallel for faster initialization. z(Loading diffusers pipeline with dtype=%s) torch_dtypetrust_remote_coderevisionNquantization_configzUsing quantization config: %szhas no attributezJPipeline class not found in diffusers, trying custom_pipeline from repo...custom_pipelineTrzhas no attribute (\w+)rgunknownzPipeline class 'z' not found in diffusers and no custom pipeline.py in repo. Try: pip install --upgrade diffusers (some pipelines require latest version). Original error: r_rpz1Failed with dtype=%s, falling back to float32: %srzLoaded diffusers pipeline: %s)rr _get_dtyperIrrrpipeline_configr[rErFrfrom_pretrainedAttributeErrorr*rresearchgroup RuntimeErrorlowerrJr$float32tor_apply_vae_optimizations_apply_attention_backend_apply_cache_ditr)rrroriginal_model_pathr_ load_kwargsconfig quant_configpiper1 custom_kwargse2match class_namerrrrs              z*DiffusersPipeline._load_diffusers_pipelinercCsn|j}|dur dSt|ddrt|dr|tdt|ddr3t|dr5|tddSdSdS) zFApply VAE memory optimizations (tiling, slicing) from pipeline config.N vae_slicingFenable_vae_slicingz*Enabled VAE slicing for lower memory usage vae_tilingenable_vae_tilingz*Enabled VAE tiling for large image support)rr[rZrrIrr)rrrrrrrrs     z*DiffusersPipeline._apply_vae_optimizationsc Cs|j}|dur|j}|durt|dd}|durdS|}ddtDddhB}||vr5td|dSdD]:}t||d}|durqt|d rqz||t d ||Wq7t yp}zt d |||WYd}~q7d}~wwq7dS) zApply attention backend setting from pipeline config or server_args. See: https://huggingface.co/docs/diffusers/main/en/optimization/attention_backends Available backends: flash, _flash_3_hub, sage, xformers, native, etc. Ndiffusers_attention_backendcSsh|]}|jqSr)namer)r8r1rrr sz=DiffusersPipeline._apply_attention_backend..fa3fa4zSkipping diffusers attention backend '%s' because it matches a SGLang backend name. Use diffusers backend names when running the diffusers backend.)rrset_attention_backendz Set attention backend '%s' on %sz.Failed to set attention backend '%s' on %s: %s) attention_backendrr[rr rIr]rZrrrrJ) rrrbackendrsglang_backendscomponent_namerr1rrrrsJ    z*DiffusersPipeline._apply_attention_backendc Cs|j}|s|Szddl}Wnty}ztd|d}~wwt|ds(tdz||}Wnty@}ztd|d}~wwz |j|fi|}Wn tyYt dwt d|S) z6Enable cache-dit for diffusers pipeline if configured.rNzVcache-dit is required for --cache-dit-config. Install it with `pip install cache-dit`. load_configszNcache-dit>=1.2.0 is required for --cache-dit-config. Please upgrade cache-dit.zdFailed to load cache-dit config. Provide a YAML/JSON path (or a dict supported by cache-dit>=1.2.0).z1Failed to enable cache-dit for diffusers pipelinez(Enabled cache-dit for diffusers pipeline) cache_dit_config cache_dit ImportErrorrrZrrr= enable_cacherI exceptionr)rrrrrr1 cache_optionsrrrrsF     z"DiffusersPipeline._apply_cache_ditcCsdtjrtjntj}t|dr0|jr0|jj}|dkr tj}|S|dkr)tj}|S|dkr0tj}|S)z? Determine the dtype to use for model loading. rfp16bf16fp32) r$ris_bf16_supportedbfloat16float16rZr dit_precisionr)rrr_rrrrr9szDiffusersPipeline._get_dtypecsN|jjjgd}tfdd|D|_td|jr"ddSddS)z-Detect if this is an image or video pipeline.)rWanimatcogvideowanhunyuanc3s|]}|vVqdSrr)r8indpipe_class_namerrr:Nsz:DiffusersPipeline._detect_pipeline_type..zDetected pipeline type: %srWrN)rrrFrr?is_video_pipelinerIr])rvideo_indicatorsrrrrJsz'DiffusersPipeline._detect_pipeline_typecCs d|jiS)z4Skip sglang's module loading - diffusers handles it.diffusers_pipeline)r)rrrrrr load_modulesTs zDiffusersPipeline.load_modulescCs|t|jddS)z;Create the execution stage wrapping the diffusers pipeline.diffusers_executionN) add_stagerrrrrrrcreate_pipeline_stages\s z(DiffusersPipeline.create_pipeline_stagescCsdS)zInitialize the pipeline.Nrrrrrinitialize_pipelinebsz%DiffusersPipeline.initialize_pipelinecCs,|jrdSd|_||j||jdS)zPost initialization hook.NT)rr!rr rrrr post_initfs  zDiffusersPipeline.post_initstage stage_namecCsD|dur ||}||jvrtd||j|||j|<|S)zAdd a stage to the pipeline.NzDuplicate stage name detected: )_infer_stage_namerr=rrD)rr$r%rrrrns    zDiffusersPipeline.add_stagecCs|jS)zList of stages in the pipeline.)rr"rrrstages{szDiffusersPipeline.stagesrcCs |js||j|j||S)z(Execute the pipeline on the given batch.)rr#rexecuter')rrrrrrr2szDiffusersPipeline.forwardrrrargsc Ks4||d<tjdi|} ||| ||d} | | S)z@Load a pipeline from a pretrained model using diffusers backend.r)rrNr)r from_kwargsr#) clsrrrrr)rrr/rrrrrrs z!DiffusersPipeline.from_pretrained module_name default_valuecCs|dkr|jS|j||S)zGet a module by name.r)rrr)rr,r-rrr get_moduleszDiffusersPipeline.get_module)NNNr)rN)NNNNNN)*rFrrr pipeline_namerrrvr*__annotations__rrr$nnModuler rrrrrrr_rrrr r!r#r rpropertyr'r%r r2 classmethodrargparse Namespacerr.rrrrr^s   R.&          r)/rr5r5rr&iortypingrnumpyrmrr$torchvision.transforms transformsrt diffusersrPILr3sglang.multimodal_gen.configs.pipeline_configs.baser)sglang.multimodal_gen.runtime.distributedrCsglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_baserHsglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executorr Dsglang.multimodal_gen.runtime.pipelines_core.executors.sync_executorr ;sglang.multimodal_gen.runtime.pipelines_core.schedule_batchr 3sglang.multimodal_gen.runtime.pipelines_core.stagesr 'sglang.multimodal_gen.runtime.platformsr )sglang.multimodal_gen.runtime.server_argsr6sglang.multimodal_gen.runtime.utils.hf_diffusers_utilsr1sglang.multimodal_gen.runtime.utils.logging_utilsrrFrIrr EntryClassrrrrs@                4K