o ۷ia8 @sddlmZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z dd l mZmZdd lmZeeZGd d d e Z  ddejdejdejde jdBdedef ddZdejddfddZGdddeZdS))chain)AnyN)nn) init_logger) HookRegistry ModelHook)current_omni_platform)OffloadBackend OffloadConfig)ModuleDiscoverycs,eZdZdZdZ  ddejdejde j dBde fd d Z d ejd ejffd d Z e d deeejfdeeejfdejde d eeejejfeejeeeeffff ddZejjd de d dfddZejjd!ddZd ejdeded eeeffddZd ejded efddZZS)"LayerwiseOffloadHookazHook for layerwise (transformer-block-wise) CPU offloading. The hook instance retains parameters for both the current registered block module and those for the next block, as well as flattened CPU tensors which record the parameters of the current block module, so that these parameters could be re-materialized on device in an overlapping way. This hook should be registered to each of the transformer blocks in DiT module(s) of the target pipeline. Based on implementations from: https://github.com/sgl-project/sglang/blob/v0.5.8/python/sglang/multimodal_gen/runtime/utils/layerwise_offload.py layerwise_offloadNT next_blockdevicestream pin_memorycCsVt|tjs Jd||_||_|pt|_||_d|_ i|_ i|_ i|_ i|_ dS)Nz0transformer block must be type `torch.nn.Module`) isinstancerModulerrrcurrent_stream copy_streamr_prefetch_donenext_block_parametersnext_block_buffersdtype_cpu_flattened_weightsdtype_metadata)selfrrrrre/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm_omni/diffusion/offloader/layerwise_backend.py__init__$s zLayerwiseOffloadHook.__init__modulereturncslt|}t||_t||_t|j|_t|j|_ t |j|j |j |j \|_|_|SN)superinitialize_hookdictnamed_parametersblock_parameters named_buffers block_buffersrrrr _to_cpurrrr)rr  __class__rrr$:s  z$LayerwiseOffloadHook.initialize_hookparamsbufsc Csi}i}i}t||D]\}}|j} | |vri|| <||| |<q|D]Z\} } tdd| D} tj| | d|d} d} | D]7\}}|}| | | ||| |vrdg|| <||  || ||j dtjd|| d|_ | |7} qG| || <q)||fS) aHelper method to move block parameters and buffers to CPU, flattening by dtype. Consolidates parameters and buffers into contiguous CPU tensors grouped by dtype for GPU transfers. Replaces original tensors with empty placeholders. Returns: Tuple of flattened CPU tensors by dtype, metadata for reconstruction by dtype css|] \}}|VqdSr")numel).0_trrr jsz/LayerwiseOffloadHook._to_cpu..cpu)dtyperrr)nameoffsetr/shaperrr5) ritemsr5sumtorchemptyr/copy_flattenappendr8data)r-r.rrdtype_grouped_weightsrrr6 param_or_bufr5 name2weights total_numel cpu_tensorcurrent_offsetr/rrrr*Ls8  zLayerwiseOffloadHook._to_cpu non_blockingc Cs|jt|j}|j}t}i}t|j-|j D]\}}t j |j ||j d}|j||d|||<q ||jWdn1sJwY|j D]1\}} ||}| D]&} | d} | |vrl|| n|| } || d| d| d| d| _q^qT||_dS)zCopy layer weights from CPU -> GPU. Pre-fetch target block in an asynchronous way with compute - memory copy overlap, with non_blocking set to True. )r5rrHNr6r7r/r8)r wait_streamrrrrEventrrr:r<r=r8rr>recordrviewrAr) rrH layer_params layer_bufsevt gpu_weightsr5 cpu_weight gpu_weightordered_metadatametadata target_nametarget_param_or_bufrrrprefetch_layers.  z#LayerwiseOffloadHook.prefetch_layercCsz|j}|durt|d|_|jD]\}}tjd|j|j d|_ q|j D]\}}tjd|j|j d|_ q+dS)zFree GPU memory for layer by replacing tensors with empty placeholders. This function does not actually offload weights from GPU back to CPU. Nrr9) rrr wait_eventr'r:r<r=rr5rAr))rrPr1parambufrrr offload_layersz"LayerwiseOffloadHook.offload_layerargskwargscOs|jdd||fS)NTrI)rX)rr r]r^rrr pre_forwards z LayerwiseOffloadHook.pre_forwardoutputcCs ||Sr")r\)rr r`rrr post_forwardsz!LayerwiseOffloadHook.post_forwardNT)Tr!N)__name__ __module__ __qualname____doc__ _HOOK_NAMErrr<rrStreamboolrr$ staticmethodr%str ParameterTensortupler5listrr*compilerdisablerXr\r_ra __classcell__rrr+rr sD    , 6& $ r Tr rrrrr!cCs*t|}t||||}|tj||Sr")r get_or_creater register_hookrh)r rrrrregistryhookrrrapply_block_hooks rxcCs8t|dd}|dur|tjtd|jjdSdS)N_hook_registryzRemoved offload hook from %s)getattr remove_hookr rhloggerdebugr,rd)r rvrrrremove_block_hooks  r~cseZdZdZdedejffdd Zdej ddfd d Z dd d Z e d ej de dBfddZe d ej de ddfddZe d ej deej fddZZS)LayerWiseOffloadBackendaLayer-wise (block-level) offloading backend. Implements sliding window offloading where only a small number of transformer blocks reside on GPU at a time. Blocks are prefetched asynchronously while previous blocks compute, and freed after use. configrcs"t||t|_g|_dSr")r#rrrir_blocks)rrrr+rrrs  z LayerWiseOffloadBackend.__init__pipeliner!Nc Csl|jr tddSt|}|jstddS|jD]}||jq|j durNz |j j|jddWnt yM}z t d|WYd}~nd}~wwt d|j t|jD]\}}|j |}t d|d|jjd t|}t|} |r| std ||jj||jqZt| } | d krtd ||jj||jqZ|D]#\} } | |krt d | q| |jt d| d|jq| d| d} }t| ||j|j|jj}|jddt| ddD]\}}| |d | }t|||j|j|jjqt d| d|j| qZt|jdkr2t|jddkr4d|_dSdSdS)Nz'LayerWiseOffloadBackend already enabledz@No DiT/transformer modules found, skipping layer-wise offloadingTrIzFailed to move VAE to GPU: %sz$Applying layer-wise offloading on %szApplying hooks on z ()z@Target layers (blocks) not found. Skipping offloading on %s (%s)r z<#Target layers (blocks) <= 1. Skipping offloading on %s (%s)zSkipped blocks module zMoved z to device rFz!Layer-wise offloading enabled on z layers (blocks))enabledr|warningr discoverditsencoderstorvae Exceptionr}info dit_names enumerater,rdrget_blocks_attr_nameget_blocks_from_ditlennamed_childrenrxrrpin_cpu_memoryrXrr@)rrmodulesencexci dit_moduledit_nameblocks_attr_nameblocks num_blocksr6m last_block first_blockrwblockrrrrenablesp            $ zLayerWiseOffloadBackend.enablecCsF|jsdS|jD] }|D]}t|q q|jd|_tddS)NFzLayer-wise offloading disabled)rrr~clearr|r)rrrrrrrr8s   zLayerWiseOffloadBackend.disablemodelcCst|jddS)z6Retrieve blocks attribute name from provided DiT model_layerwise_offload_blocks_attrN)rzr,)rrrrrDsz,LayerWiseOffloadBackend.get_blocks_attr_namer6cCs"t|jdst|jd|dSdS)Nr)hasattrr,setattr)rr6rrrset_blocks_attr_nameIs z,LayerWiseOffloadBackend.set_blocks_attr_namecCsht|}|durtd|jjdgSt||d}|dur0td|d|jjdgSt|S)a/ Retrieve a list of blocks from provided DiT model. Blocks attribute name are found by `_layerwise_offload_blocks_attr` set to DiT models. For example, ``` class WanTransformer3DModel(nn.Module): _layerwise_offload_blocks_attr = "blocks" ``` Nz.No _layerwise_offload_blocks_attr defined for z, skipping layerwise offloadingzBlocks (layers) 'z' not found on )rrr|rr,rdrzrp)rrrrrrrNs  z+LayerWiseOffloadBackend.get_blocks_from_ditrc)rdrerfrgr r<rrrrrrrrkrlrrrprrsrrr+rrs O $rrb) itertoolsrtypingrr<r vllm.loggerrvllm_omni.diffusion.hooksrrvllm_omni.platformsrbaser r module_collectorr rdr|r rrrirjrxr~rrrrrs:      7