o پii@sUddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZmZmZddlmZmZmZddlmZmZmZddlZddlZddlZddlmZddlm Z ddl!m"m#Z#ddl$m%Z%m&Z&e&e'Z(ed Z)ej*ej+ej,d Z-d Z.e/e0d <d Z1e/e0d<de/fddZ2ej3j4Z5da6dej3j7dBddfddZ8e8ej3_4dej3j7dBfddZ9Gdddej:Z;Gdddej<Z=de>e)de>e)fddZ?de@de@de@fd d!ZAd"e/defd#d$ZBd%d&ZCd'eDe/e/ffd(d)ZEd*ed+e/eFBe Bd,eGed-eDe/efdef d.d/ZHdeDe/effd0d1ZId]d2d3ZJde/fd4d5ZKGd6d7d7ZLd]d8d9ZMeGd:d;d;ZNe OZPdeNfdejRd?ejRd@ejRdBdAe dBfdBdCZSdejRfdDdEZT    d_dFeDe/efdBdGe@dBdHe@dBdIe@dBdeUeUeUejVdBf dJdKZWdLe@ddfdMdNZXedOdPdeYfdQdRZZedOdPdeYfdSdTZ[ Vd`deGeUejVeUejVffdWdXZ\dYdZZ]d[d\Z^dS)aN)Callable) dataclassfields is_dataclass) lru_cachepartialwraps)AnyTypeVarcast) RemotePdb)MixedPrecisionPolicy)SortedHelpFormatter init_loggerT)fp32fp16bf16"SGLANG_DIFFUSION_ATTENTION_BACKENDSTR_BACKEND_ENV_VAR!SGLANG_DIFFUSION_ATTENTION_CONFIGSTR_ATTN_CONFIG_ENV_VARreturncCs|tj}|rtd|t |Stjjdurd}ntjjdur!d}nttjdr0tjj dur0d}nt dtd|t |S) a We either use the library file specified by the `VLLM_NCCL_SO_PATH` environment variable, or we find the library file brought by PyTorch. After importing `torch`, `libnccl.so.2`, `librccl.so.1` or `libmccl.so.2` can be found by `ctypes` automatically. zEFound nccl from environment variable SGLANG_DIFFUSION_NCCL_SO_PATH=%sNz libnccl.so.2z librccl.so.1musaz libmccl.so.2z0NCCL only supports CUDA, ROCm and MUSA backends.zFound nccl from library %s) envsSGLANG_DIFFUSION_NCCL_SO_PATHloggerinfotorchversioncudahiphasattrr ValueErrorstr)so_filer&O/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/utils.pyfind_nccl_library3s    r(streamcCs|a|dur t|dSdSN)_current_streamprev_set_stream)r)r&r&r'_patched_set_streamTs r-cCs@ddlm}|s dStdur|rtjntjatS)a replace `torch.cuda.current_stream()` with `sglang.multimodal_gen.utils.current_stream()`. it turns out that `torch.cuda.current_stream()` is quite expensive, as it will construct a new stream object at each call. here we patch `torch.cuda.set_stream` to keep track of the current stream directly, so that we can avoid calling `torch.cuda.current_stream()`. the underlying hypothesis is that we do not call `torch._C._cuda_setStream` from C/C++ code. rcurrent_platformN) 'sglang.multimodal_gen.runtime.platformsr/ is_cuda_aliker+is_rocmrr Streamcurrent_streamr.r&r&r'r4^s  r4cs(eZdZdfdd ZdddZZS) StoreBooleanFNc stj||dd|||ddS)N?T)option_stringsdestnargsconstdefaultrequiredhelp)super__init__)selfr7r8r;r<r= __class__r&r'r?s zStoreBoolean.__init__cCs|dur t||jddSt|tr8|dkr!t||jddS|dkr0t||jddStd|dt||jt|dS)NTtruefalseFzInvalid boolean value: z. Expected 'true' or 'false'.)setattrr8 isinstancer$lowerr#bool)r@parser namespacevalues option_stringr&r&r'__call__s    zStoreBoolean.__call__)FFNr*)__name__ __module__ __qualname__r?rM __classcell__r&r&rAr'r5~s r5csjeZdZdZdfdd Z ddejffdd Zdee dee fd d Z d e dee fd d Z Z S)FlexibleArgumentParserz=ArgumentParser that allows both underscore and dash in names.rNcs&d|vrt|d<tj|i|dS)Nformatter_class)rr>r?r@argskwargsrAr&r'r?szFlexibleArgumentParser.__init__cs|dur tjdd}tdd|Dr||}g}|D]a}|dr[d|vrI|dd\}}d|tdddd}||d|q|d|tddddq|dry|dkryt|d kry|d||d dq||qt ||}t |_ d }|t|kr||}|drd|vr|dd d ddd}|j ||d7}n/|d ddd}|j ||dt|kr||dds|d 7}n |d7}n|d7}|t|ks|S) Ncss|]}|dVqdS)--configN) startswith).0argr&r&r' z4FlexibleArgumentParser.parse_args..--=_-z-Or)sysargvany_pull_args_from_configrYsplitlenreplaceappendr> parse_argsset _providedadd)r@rUrJprocessed_argsr[keyvalueirAr&r'rksD  $       "   z!FlexibleArgumentParser.parse_argsrUc Cs6d}d}t|D]\}}|dr|dkrtd|}|}q|dur$|S|d|}d|vr?|ddd}||dd}n|t|dkrKtd||d}||dd}||} |d d kr|dkrltd |d } |d} |dd} | | g| | |}|S|d } |dd} | g| | |}|S) aMethod to pull arguments specified in the config file into the command-line args variable. The arguments in config file will be inserted between the argument list. example: ```yaml port: 12323 tensor-parallel-size: 4 ``` ```python $: vllm {serve,chat,complete} "facebook/opt-12B" --config config.yaml -tp 2 $: args = [ "serve,chat,complete", "facebook/opt-12B", '--config', 'config.yaml', '-tp', '2' ] $: args = [ "serve,chat,complete", "facebook/opt-12B", '--port', '12323', '--tp-size', '4', '-tp', '2' ] ``` Please note how the config args are inserted after the sub command. this way the order of priorities is maintained when these are args parsed by super(). NrXz$More than one config file specified!r_rWzCNo config file specified! Please check your command-line arguments.rbrservezANo model_tag specified! Please check your command-line arguments.) enumeraterYr#rgrh_load_config_file) r@rUindex config_argrrr[args_before_config file_pathargs_after_config config_argscommand model_tagother_args_beforer&r&r'rfsV"        z-FlexibleArgumentParser._pull_args_from_configrzc s|dd}|dvrtd|gi}zt| }t|}Wdn1s)wYWntyC}ztd||d}~wwdd|jDd t d t t t fffd d d |S)a Loads a yaml file and returns the key value pairs as a flattened list with argparse like pattern ```yaml port: 12323 tensor-parallel-size: 4 vae_config: load_encoder: false load_decoder: true ``` returns: processed_args: list[str] = [ '--port': '12323', '--tp-size': '4', '--vae-config.load-encoder': 'false', '--vae-config.load-decoder': 'true' ] .rs)yamlymljsonzUConfig file must be of a yaml/yml/json type. %s suppliedNzOUnable to read the config file at %s. Make sure path is correctcSsg|] }t|tr|jqSr&)rFr5r8)rZactionr&r&r' [s  z.prefixdcs|D]a\}}|r|d|n|}t|tr3|vr3|r&d|qd|dqt|trLd||D] }t|qAqt|trW||qd|t|qdS)Nrr^rD)itemsrFrHrjlistr$dict)rrrprqfull_keyitem process_dictrostore_boolean_argumentsr&r'r_s"    z>FlexibleArgumentParser._load_config_file..process_dict) rgr#openr safe_load Exceptionrerror_actionsr$rr )r@rz extensionconfig config_fileexr&rr'rv2s6  " z(FlexibleArgumentParser._load_config_filerNNN) rNrOrP__doc__r?argparse Namespacerkrr$rfrvrQr&r&rAr'rRs6XrRclscs>|jdtfddtd fdd }t|d||S) a- A replacement for `abc.ABC`. When we use `abc.ABC`, subclasses will fail to instantiate if they do not implement all abstract methods. Here, we only require `raise NotImplementedError` in the base class, and log a warning if the method is not implemented in the subclass. r@c Ssg}t|D].}|drqzt||}t|r|j}Wn ty%Yqwt|}d|vr4||q|rKd |}d|d|}t |dSdS)Nr`NotImplementedError,zMethods z not implemented in ) dirrYgetattrcallable__func__AttributeErrorinspect getsourcerjjoinrwarning)r@unimplemented_methods attr_nameattr attr_funcsrc method_namesmsgr&r&r'find_unimplemented_methodss*       zBwarn_for_unimplemented_methods..find_unimplemented_methodsrNcs"|g|Ri||dSr*r&rTr original_initr&r' wrapped_inits z4warn_for_unimplemented_methods..wrapped_initr?r)r?objectrtype __setattr__)rrr&rr'warn_for_unimplemented_methodsxs  rrq alignmentcCstt|||S)zalign height, width according to alignment Args: value (int): height or width alignment (int): target alignment factor Returns: int: the aligned value )intmathceil)rqrr&r&r'align_tos rqualnamecCs$|dd\}}t|}t||S)z8 Resolve an object by its fully qualified name. rrW)rsplit importlib import_moduler)r module_nameobj_namemoduler&r&r'resolve_obj_by_qualnames  rcCsddlmmm}|S)aV Historical comments: libnvml.so is the library behind nvidia-smi, and pynvml is a Python wrapper around it. We use it to get GPU status without initializing CUDA context in the current process. Historically, there are two packages that provide pynvml: - `nvidia-ml-py` (https://pypi.org/project/nvidia-ml-py/): The official wrapper. It is a dependency of sglang-diffusion, and is installed when users install sglang-diffusion. It provides a Python module named `pynvml`. - `pynvml` (https://pypi.org/project/pynvml/): An unofficial wrapper. Prior to version 12.0, it also provides a Python module `pynvml`, and therefore conflicts with the official one which is a standalone Python file. This causes errors when both of them are installed. Starting from version 12.0, it migrates to a new module named `pynvml_utils` to avoid the conflict. It is so confusing that many packages in the community use the unofficial one by mistake, and we have to handle this case. For example, `nvcr.io/nvidia/pytorch:24.12-py3` uses the unofficial one, and it will cause errors, see the issue https://github.com/vllm-project/vllm/issues/12847 for example. After all the troubles, we decide to copy the official `pynvml` module to our codebase, and use it directly. rN)(sglang.multimodal_gen.third_party.pynvmlmultimodal_gen third_partypynvml)rr&r&r' import_pynvmlsrrcCsN|D] \}}|tjvrtj||krtd|tj|||tj|<qdS)Nz5Overwriting environment variable %s from '%s' to '%s')rosenvironrr)rkvr&r&r'update_environment_variabless robjmethodrUrVcCsnt|trtt||}n"t|tr+zt||}Wnty*td|ddwt||}||i|S)a3 Run a method of an object with the given arguments and keyword arguments. If the method is string, it will be converted to a method using getattr. If the method is serialized bytes and will be deserialized using cloudpickle. If the method is a callable, it will be called directly. zMethod z is not implemented.N) rFbytesr cloudpickleloadsr$rrr)rrrUrVfuncr&r&r' run_methods     rcs&tstdfddtDS)NzExpected dataclass instancecsi|] }|jt|jqSr&)namer)rZfrr&r' sz"shallow_asdict..)r TypeErrorrrr&rr'shallow_asdictsrcCsBd}ddl}|dkrtd}||tjdStddS)NrWrLinuxz libc.so.6z8kill_itself_when_parent_died is only supported in linux.) platformsystemctypesCDLLprctlsignalSIGKILLrr)PR_SET_PDEATHSIGrlibcr&r&r'kill_itself_when_parent_dieds   rcCs&t\}}}dt|||}|S)Nr)rcexc_infor tracebackformat_exception)etyperqtberr_strr&r&r'get_exception_tracebacksrc@s4eZdZdeeeeffddZdefddZ dS)TypeBasedDispatchermappingcCs ||_dSr*)_mapping)r@rr&r&r'r?s zTypeBasedDispatcher.__init__rcCs4|jD]\}}t||r||Sqtd|)NzInvalid object: )rrFr#)r@rtyfnr&r&r'rMs   zTypeBasedDispatcher.__call__N) rNrOrPrtuplerrr?r rMr&r&r&r'rsrcCsnttjtj%}|tjtjd|d|d}td|d WddS1s0wYdS)NrW) localhostrr)hostport) socketAF_INET SOCK_STREAM setsockopt SOL_SOCKET SO_REUSEADDRbind getsocknamer set_trace)srr&r&r'remote_breakpoint&s   "rc@sfeZdZUdZejdBed<dZejdBed<dZejdBed<dZ ejdBed<dZ e dBed<dS)MixedPrecisionStateN param_dtype reduce_dtype output_dtype compute_dtype mp_policy) rNrOrPrrdtype__annotations__rrrr r r&r&r&r'r.s rcCsttds tdtttjS)z&Get the current mixed precision state.statezMixed precision state not set)r"_mixed_precision_stater#r rr r&r&r&r'get_mixed_precision_state;s  rrrrr cCst||||d}|t_dS)zSet mixed precision policy globally. Args: param_dtype: Parameter dtype used for training reduce_dtype: Reduction dtype used for gradients output_dtype: Optional output dtype )rrrr N)rr r )rrrr r r&r&r'set_mixed_precision_policyBs  rcCsttds tSt}|jS)z:Get the current compute dtype from mixed precision policy.r )r"r rget_default_dtyperr)r r&r&r'get_compute_dtypeXs r mask_strategyt_maxl_maxh_maxc sr|dur |durdurdusJdfddt|DSdd|D}|durUdurUdurUtdd|Dd}td d|Ddtd d|Ddn|duraduraduseJd |}fd dt|D}|D];\}}tt|d \} } } d| kr|krnq{d| krkrnq{d| krkrnq{||| | | <q{|S)aN Convert a dictionary of mask indices to a 3D list of tensors. Args: mask_strategy: keys are "t_l_h", values are torch.Tensor masks. t_max, l_max, h_max: if provided (all three), force the output shape to (t_max, l_max, h_max). If all three are None, infer shape from the data. NzBIf mask_strategy is None, you must provide t_max, l_max, and h_maxc"g|] }fddtDqS)cg|] }ddtDqS)cSg|]}dqSr*r&rZr`r&r&r'rt9dict_to_3d_list....ranger)rr&r'rt.dict_to_3d_list...rr)rrr&r'rssz#dict_to_3d_list..cSs g|] }ttt|dqS)r`)rmaprrg)rZrpr&r&r'rxs css|]\}}}|VqdSr*r&)rZtr`r&r&r'r\}r]z"dict_to_3d_list..rWcss|]\}}}|VqdSr*r&)rZr`lr&r&r'r\~r]css|]\}}}|VqdSr*r&)rZr`hr&r&r'r\r]zfEither supply none of (t_max, l_max, h_max) to infer dimensions, or supply all three to fix the shape.cr)cr)cSrr*r&rr&r&r'rrrrr) max_head_idxr&r'rrrrr)r$ max_layer_idxr&r'rsr`r)rmaxrr rrg) rrrrindicesmax_timesteps_idxresultrprqr!r"r#r&)rrr$r%r'dict_to_3d_listas:  r*seedcCsddlm}||dS)Nrr.)r0r/seed_everything)r+r/r&r&r'set_random_seeds r-rW)maxsizecCstjdduS)Nvsa)rutil find_specr&r&r&r'is_vsa_availablesr2cCs>tjddur dSz ddl}|jdkWStyYdSw)Nz!kernel.csrc.attn.vmoba_attn.vmobaFrz2.7.4)rr0r1 flash_attn __version__r)r3r&r&r'is_vmoba_availables  r5F皙?c Cslt|tsJdd|D}dd|D}|r|durt||ddD]]\}}tjd||jd}||krctjd d d |j|d |ddd f |ddd f<t |ddd f|ddd f<q"|ddd f|ddd f<|ddd f|ddd f<q"||fSt||ddD]&\}}t |ddd f|ddd f<t |ddd f|ddd f<q||fS)a Generate binary masks for Text-to-Image-to-Video (TI2V) tasks. Creates masks to control which frames should be preserved vs replaced. Primarily used to fix the first frame to the input image while generating other frames. Args: tensors: List of tensors with shape [C, T, H, W] zero: If True, set first frame (dim 1, index 0) to zero. Default: False generator: Optional random generator for stochastic masking p: Probability of applying special noise when generator is provided. Default: 0.2 Returns: Tuple of two lists of tensors: - When zero=False: Both lists contain all-ones tensors - When zero=True (no generator): First frame set to 0, others to 1 - When zero=True (with generator): First frame set to small random values with probability p Example: >>> latent = torch.randn(48, 69, 96, 160) # [C, T, H, W] >>> _, mask = masks_like([latent], zero=True) >>> # mask[0][:, 0] == 0 (first frame) >>> # mask[0][:, 1:] == 1 (other frames) >>> blended = (1.0 - mask[0]) * image + mask[0] * latent >>> # Result: first frame = image, other frames = latent cS"g|] }tj|j|j|jdqS)r deviceronesshaper r9rZur&r&r'r"zmasks_like..cSr7r8r:r=r&r&r'rr?NF)strictrW) generatorr9g ?)rW)meanstdsizer9rAr) rFrziprrandr9rnormal expand_asexp zeros_like) tensorszerorApout1out2r>r random_numr&r&r' masks_likes<$ "$rRcCs||}||d}||}t|||}t||||} ||dkr2| |dkr2|| |ks4J|| } t|||} t|| ||} | |dkr\| |dkr\| | |ks^J| | } t|| | |t|| | |krx|| fS| | fS)NrBr)rr&)wr#dwdh expected_arearatioowohow1oh1ratio1oh2ow2ratio2r&r&r'best_output_sizes (($r`cCs@t||}||}t|dd}t|dd}||dfS)N )rsqrtround) target_arearWwidthheightr&r&r'calculate_dimensionss  rgrr)NNNN)FNr6)_rrrimportlib.utilrrrrrrc threadingrcollections.abcr dataclassesrrr functoolsrrrtypingr r r rrr remote_pdbr torch.distributed.fsdpr sglang.multimodal_gen.envsrr1sglang.multimodal_gen.runtime.utils.logging_utilsrrrNrrfloat32float16bfloat16PRECISION_TO_TYPErr$r rr(r set_streamr,r+r3r-r4Actionr5ArgumentParserrRrrrrrrrrrrrrrrrrrlocalr rr rrrTensorr*r-rHr2r5rRr`rgr&r&r&r's       ^+             =  B