o پikL@sddlZddlmZddlmZmZmZmZmZddl Z ddl m Z ddl m Z e eZGdddZGdd d Zd e jjfd d ZdS) N)chain)AnyDictListSetTuple) ServerArgs) init_loggerc@sZeZdZdZddddejjdedede d e d ed d fd dZ ded ed BfddZ ej j d/ddZd0ddZded ejfddZej j d0dede d d fddZej j ded d fddZej j d/ddZej j d/d d!Zej j ded d fd"d#Zej j d/d$d%Zej j d&eeejfd eed Bfd'd(Zd)d*Zd/d+d,Zd/d-d.Zd S)1LayerwiseOffloadManagera+A lightweight layerwise CPU offload manager. This utility offloads per-layer parameters/buffers from GPU to CPU, and supports async H2D prefetch using a dedicated CUDA stream. Typical usage: - Construct the manager with the target model and the list-like module attribute that represents transformer blocks (e.g. ``blocks``). - Call :meth:`initialize` once to offload weights and prefetch layer 0. - During forward, call :meth:`prefetch_layer` for the next layer and :meth:`release_layer` for the finished layer. T)pin_cpu_memory prefetch_sizemodellayers_attr_str num_layersenabledr r returnNcCs||_||_||_||_ttd||j|_t|otj |_ |j s%dSt dtj |_ tj |_tdt|d|_i|_i|_t|_i|_i|_i|_g|_|dS)Nr cudaz(^|\.)z \.(\d+)(\.|$))rrrr minmaxr booltorchr is_availablerdevicecurrent_deviceStream copy_streamrecompileescape_layer_name_re_consolidated_cpu_weights_weight_metadataset _gpu_layers_prefetch_events_named_parameters_named_buffers_forward_hooks _initialize)selfrrrrr r r+i/home/ubuntu/.local/lib/python3.10/site-packages/sglang/multimodal_gen/runtime/utils/layerwise_offload.py__init__s*   z LayerwiseOffloadManager.__init__namecCs:|j|}|s dSzt|dWStyYdSw)N)r searchintgroup Exception)r*r.mr+r+r,_match_layer_idxGs  z(LayerwiseOffloadManager._match_layer_idxcCs|jsdSt|j|_t|j|_i}t|j|j}|D]#\}}| |}|dus6||j kr7q$| |i |j g ||fq$|D]e\}}i|j|<i|j|<|D]R\}}tdd|D} tj| ||jd} d} |D]0\}} | } | | | | | || | | jd|j||<tjd|j|d| _| | 7} qx| |j||<q^qL|jdd |td |jd |j dS) Ncss|] \}}|VqdSN)numel).0_tr+r+r, isz6LayerwiseOffloadManager._initialize..)dtype pin_memoryr)r<offsetr7shaper rr<F non_blockingz?LayerwiseOffloadManager initialized with num prefetched layer: z, total num layers: )rdictrnamed_parametersr& named_buffersr'ritemsr5r setdefaultr<appendr!r"sumremptyr r7copy_flattenr?rdataprepare_for_next_reqregister_forward_hooksloggerinfor )r* layer_groups all_tensorsr.tensor layer_idxdtype_to_paramsr<weights total_numel cpu_buffercurrent_offsetweightr7r+r+r,r)PsR       z#LayerwiseOffloadManager._initializecCsJt|jD] }|j||dq|s!|jdur#tj|jdSdSdS)zd Prepare for the next round of denoising loop with prefetching the necessary layers rBN)ranger prefetch_layerrrrcurrent_stream wait_stream)r*rCir+r+r,rOs z,LayerwiseOffloadManager.prepare_for_next_reqcCs&||jvr |j|}|S|j|}|S)z1get the target model weight/buffer to be replaced)r&r')r*r.targetr+r+r,get_target_with_names   z,LayerwiseOffloadManager.get_target_with_namerVrCc Cs`|jr |jdus |jdurdS|dks||jkrdS||jvr!dS||jvr(dS|jtj i}tj |j)|j| D]\}}tj |j ||jd}|j||d|||<qBWdn1sfwYtj}||j||j|<|j| D]%\}} | d}||}||} || d| d| d| d| _q|j|dS) z idempotent Nr)r<rrBr<r>r7r?)rrrrr$r!r`rrr_streamrGrKr?rLEventrecordr%r"rcviewrNadd) r*rVrC gpu_buffersr<rZ gpu_buffereventr.metarbr+r+r,r^s@        z&LayerwiseOffloadManager.prefetch_layercCs|jr|jdur dS|j|d|dkrdS||jvrdS|j|iD]\}}||}t j d|j|dd|_ q'|j |dS)z lightweight release layer weights Basically set the reference count to the gpu weight tensor to zero. The weights on cpu is untouched Nrr@r<rA) rrr%popr$r"getrGrcrrKrNdiscard)r*rVr.rlrbr+r+r, release_layers  z%LayerwiseOffloadManager.release_layercCsN|jr|jdur dS|jdurtj|jt|jD]}| |qdSr6) rrrrrr_r`listr$rpr*rVr+r+r, release_alls  z#LayerwiseOffloadManager.release_allcCs\|jr|jdur dS|jdurtj|jt|jD]}||j vr+|j |ddqdS)z Load all layers from CPU to GPU.NFrB) rrrrrr_r`r]rr$r^rrr+r+r,load_all_layerss  z'LayerwiseOffloadManager.load_all_layersc Cs|jr||jvr dS||jvrdS|jdurtj|j|j |i D].\}}| |}|j }|d}|j||}|d}|d} |||| |q(dS)z,Sync a layer's weights from GPU back to CPU.Nr<r>r7)rr$r!rrrr_r`r"rnrGrcrNrMcpurL) r*rVr.rlrb gpu_weightr<rZr>r7r+r+r,sync_layer_to_cpus   z)LayerwiseOffloadManager.sync_layer_to_cpucCsN|jr|jdur dS|jdurtj|jt|jD]}| |qdS)z5Sync all loaded layers' weights from GPU back to CPU.N) rrrrrr_r`rqr$rwrrr+r+r,sync_all_layers_to_cpu s  z.LayerwiseOffloadManager.sync_all_layers_to_cpu weight_dictc Cs|jsdSt}|D]~\}}||}|durq |j|}|dus(||vr)q ||}t|dt|jkrLtd|dt|ddt|j|d}|d} |d} |j ||} | | | |  |j |d  ||j vr||} | j |j | jd ||q |S) a`Update consolidated CPU buffers with new weights. When layerwise offload (--dit-layerwise-offload) is enabled, the offload manager replaces GPU parameters with small torch.empty((1,)) placeholders while real weights live in consolidated pinned CPU buffers. The refit process writes new weights directly into the CPU buffers, bypassing the placeholders. For any layer that happens to be resident on the GPU at update time, the live GPU tensor is also updated. Args: weight_dict: Mapping of parameter name to new weight tensor. Returns: Set of parameter names that were successfully updated. Raises: ValueError: If a weight's shape does not match the recorded metadata (i.e., the real shape, not the placeholder shape). Nr?zShape mismatch for z : expected=z , loaded=r<r>r7)r<)rr#rGr5r"rntupler? ValueErrorr!rLtorMr$rcrNr<rh) r*ry updated_namesr. loaded_weightrV meta_layerrlr<r>r7rZrbr+r+r,update_cpu_weightss>      z*LayerwiseOffloadManager.update_cpu_weightsc csxt|jD]3}|j|D])\}}|d}|d}|d}|d}|j||}||||||fVqqdS)aYield (name, tensor) pairs from consolidated CPU buffers. This reconstructs the original weight tensors (with correct shapes) from the flat CPU buffers using stored metadata. Unlike model.named_parameters(), which returns (1,) placeholders when offload is enabled, this method returns the real weights and can be used for checksum computation. r<r>r7r?N)sortedr"rGr!reshape) r*rVr.rlr<r>r7r?rZr+r+r,iter_cpu_weightsUs z(LayerwiseOffloadManager.iter_cpu_weightscs|jsdStjj}fdd}fdd}jt|D]\}}|||}|||}j ||gq!dS)Ncfdd}|S)Ncsdkr jddjvrtjjjdkr=tjdjD]}|j}j |ddq.dSdS)NrFrBr/T) rOr%rrr_ wait_eventr r]rr^)moduleinputjlayer_to_prefetchrar*r+r,hookns   zSLayerwiseOffloadManager.register_forward_hooks..make_pre_hook..hookr+rarr*rar, make_pre_hookms zELayerwiseOffloadManager.register_forward_hooks..make_pre_hookcr)NcsdSr6)rp)rroutputrr+r,r~szTLayerwiseOffloadManager.register_forward_hooks..make_post_hook..hookr+rrrr,make_post_hook}szFLayerwiseOffloadManager.register_forward_hooks..make_post_hook) rgetattrrrr(clear enumerateregister_forward_pre_hookregister_forward_hookextend)r*layersrrralayerpre_hook_handlepost_hook_handler+rr,rPgs   z.LayerwiseOffloadManager.register_forward_hookscCs"|jD]}|q|jdS)z$Remove all registered forward hooks.N)r(remover)r* hook_handler+r+r,remove_forward_hookss  z,LayerwiseOffloadManager.remove_forward_hooksrN)T)__name__ __module__ __qualname____doc__rnnModulestrr1rr-r5compilerdisabler)rOTensorrcr^rprsrtrwrxrrrrrPrr+r+r+r,r sZ +  ; *        = %r c@sXeZdZUdZeeed<gZee ed<de fddZ ddZ dd d Z dd dZd S)OffloadableDiTMixinzT A mixin that registers forward hooks for a DiT to enable layerwise offload layer_nameslayerwise_offload_managers server_argsc Csg|_|jD]A}t||d}|dust|tjjsqt|}|jdkr1dt t |j|d}nt |j}t |||d|j |d}|j |qtd|jjd|jdS)Ng?r T)rrrrr r zEnabled layerwise offload for z on modules: )rrr isinstancerr ModuleListlendit_offload_prefetch_sizer1roundr r rIrQrR __class__r)r*r layer_name module_listrr managerr+r+r,configure_layerwise_offloads.    z/OffloadableDiTMixin.configure_layerwise_offloadcCs*|jdurdS|jD]}|jddq dS)NTrB)rrOr*rr+r+r,rOs  z(OffloadableDiTMixin.prepare_for_next_reqrNcCs4|jdurdS|jD] }|jr||q dS)zCDisable layerwise offload: load all layers to GPU and remove hooks.N)rrrrtrr+r+r,disable_offloads  z#OffloadableDiTMixin.disable_offloadcCs<|jdurdS|jD]}|jr|||q dS)zTRe-enable layerwise offload: sync weights to CPU, release layers, and restore hooks.N)rrrxrsrPrr+r+r,enable_offloads  z"OffloadableDiTMixin.enable_offloadr)rrrrrr__annotations__rrqr rrrOrrr+r+r+r,rs    rrccsg}t|tr|jrdd|jD}|s|EdHdSt}|D]}|D]\}}||||fVq)q#|D] \}}||vrJ||fVq=dS)aHYield (name, tensor) pairs with materialized weights, even under offload. When layerwise offload is active, module.named_parameters() returns (1,) placeholders for offloaded layers. This function reads the actual data from the offload manager's CPU buffers and chains it with the non-offloaded parameters. cSsg|]}|jr|qSr+)r)r8r4r+r+r, sz-iter_materialized_weights..N)rrrrEr#rrh)roffload_managersoffloaded_namesrr.rUparamr+r+r,iter_materialized_weightss$   r)r itertoolsrtypingrrrrrr)sglang.multimodal_gen.runtime.server_argsr1sglang.multimodal_gen.runtime.utils.logging_utilsr rrQr rrrrr+r+r+r,s   A