o }oi{i@s4ddlmZddlmZddlmZddlZddlmZddl m Z ddl m Z ddl mZddlmZdd lmZdd lmZmZmZmZdd lmZmZmZdd lmZmZmZdd l m!Z!ddl"m#Z#ddl$m%Z%ddZ&ddZ'eGdddee#j(Z)Gddde Z*Gddde Z+GdddeZ,dS)) nullcontext) dataclass)CallableN)parallel_state) VisionModule)ColumnParallelLinear)TransformerConfig) functional)FluxSingleTransformerBlock MMDiTLayer'get_flux_double_transformer_engine_spec'get_flux_single_transformer_engine_spec)EmbedND MLPEmbedderTimeStepEmbedder) FluxConfigFluxModelParamsMegatronFluxModel)ControlNetConditioningEmbedding)io)loggingcCs|D]}tj|q|S)z Initializes all parameters of the given module to zero. Args: module (nn.Module): The module whose parameters will be initialized to zero. Returns: nn.Module: The same module with zero-initialized parameters. ) parametersnninitzeros_)moduleprk/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/diffusion/models/flux_controlnet/model.py zero_module(s rcCsHt|}t|trt|dkr|d}n|}tdgjdd|d<|S)a# Processes a single step of data from a dataloader iterator for the Flux ControlNet. Args: dataloader_iter (Iterator): An iterator over the dataloader that provides batches of data. Returns: dict: A processed batch dictionary with an added 'loss_mask' key. r?T non_blocking loss_mask)next isinstancetuplelentorchTensorcuda)dataloader_iterbatch_batchrrrflux_controlnet_data_step7s   r/c@seZdZUdZdZeed<dZeed<dZeed<dZ eed<d Z eed <d Z eed <d Z eed<dZ eed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed <dZeed!<dZeed"<d#Zeed$<eZeed%<dS)&FluxControlNetConfigz< Flux config inherits from TransformerConfig class.  num_layers patch_size@ in_channelsnum_joint_layers num_single_layersi hidden_sizenum_attention_headsi vec_in_dimi context_dimTguidance_embedNnum_modemodel_channelsconditioning_embedding_channelsrotary_interleavedgư>layernorm_epsilonrhidden_dropoutattention_dropout add_qkv_biasuse_cpu_initializationload_from_flux_transformerg @guidance_scale data_step_fn) __name__ __module__ __qualname____doc__r2int__annotations__r3r5r7r9r:r<r=r>r?boolr@rBrCrDrEfloatrFrGrHrIrJrKr/rLrrrrrr0Ks0                      r0cseZdZdZdeffdd ZddZddZ dd ej d ej d ej dej dej dej dej dej de fddZ Z S)FluxControlNeta A VisionModule-based neural network designed for Flux ControlNet tasks. Args: config (FluxControlNetConfig): Configuration object containing model parameters such as input channels, hidden size, patch size, and number of transformer layers. configc stj|_j|_j|_t|jdgdd|_t j|j|_ t j |j|_ t j|j|_tj|jd|_jrSjrNtj|jdnt|_tfddtjD|_tfddtjD|_t|_tjD]}|jtt|j|jtj j!dd q}t|_"tjD]}|j"tt|j|jtj j!dd qj#d urt$j#d d |_%t&j j|j|_'d Sd |_%tt&j j|j|_'d S) z Initializes the FluxControlNet model with embeddings, transformer layers, and optional conditioning blocks. Args: config (FluxControlNetConfig): Configuration object with model parameters. i')8rX)dimthetaaxes_dim)in_dim hidden_dimcs g|] }ttj|ddqS)F)rV submodules layer_numbercontext_pre_only)r r r^.0irVrr sz+FluxControlNet.__init__..csg|] }ttj|dqS))rVr^r_)r r r^rardrrresT)rV init_method gather_outputN)rWrWrWrW)rCblock_out_channels)(super__init__r5 out_channelsr:r3r pos_embedrLinear img_embedr> txt_embedrrBtimestep_embeddingrr=vector_embeddingr?Identityguidance_embedding ModuleListranger7 double_blocksr9 single_blockscontrolnet_double_blocksappendrrrnormal_controlnet_single_blocksrCrinput_hint_blockr)controlnet_x_embedder)selfrV_ __class__rdrrjvst      zFluxControlNet.__init__cCstd|j|j|j|j|j|j|j|j|j|j|j j|j dd|j j|j dddS)z Loads pre-trained weights from a Flux Transformer model into the FluxControlNet. Args: flux (FluxTransformer): A pre-trained Flux Transformer model. z-Loading ControlNet layer weights from Flux...F)strictN) rinforlload_state_dict state_dictrnrorprqrvrw)r~fluxrrrrJs z)FluxControlNet.load_from_flux_transformerc Cs|jjs t}|Sddl}|jjdkr|jjjj}n|jjdkr'|jjjj}nt d|jjj |jj |jj ||jj |jjdd|jj fd}d}trRtjdd }|jjd||d }|S) zcontext manager for fp8 reciperNe4m3hybridz3E4M3 and HYBRID are the only supported FP8 formats.F)margininterval fp8_formatamax_compute_algoamax_history_lenoverride_linear_precisionT)with_context_parallel)enabled fp8_recipe fp8_group)rVfp8rtransformer_enginecommonrecipeFormatE4M3HYBRID ValueErrorDelayedScaling fp8_margin fp8_intervalfp8_amax_compute_algofp8_amax_history_len fp8_wgradrmodel_parallel_is_initializedget_amax_reduction_grouppytorch fp8_autocast)r~ fp8_contextrrrrrrrget_fp8_contexts0   zFluxControlNet.get_fp8_contextNr!imgcontrolnet_condtxty timestepsimg_idstxt_idsguidanceconditioning_scalec s||} ||} |jdurF||}|j\} } }}||jj}||jj}|| | ||j||j}|dddddd}|| ||d}| ||} | |j d }| |}|durk|| |j |d }|||}tj||fdd }||}d }t|jD]'\}}||| | ||d \} } || f}Wdn1swYqtj| | gdd } d }t|jD]/\}}||| ||d \} }|| | jdddff}Wdn1swYqd }t||jD]\}}||\}}|r||n|}||f7}qd }t||jD]\}}||\}}|r)||n|}||f7}qfdd|D}fdd|D}t|dkrMdn|}t|dkrXdn|}||fS)a] Forward pass for the FluxControlNet model. Args: img (torch.Tensor): Input image tensor. controlnet_cond (torch.Tensor): Conditioning tensor for ControlNet. txt (torch.Tensor, optional): Text embedding tensor. Default is None. y (torch.Tensor, optional): Vector embedding tensor. Default is None. timesteps (torch.LongTensor, optional): Time step tensor. Default is None. img_ids (torch.Tensor, optional): Image IDs. Default is None. txt_ids (torch.Tensor, optional): Text IDs. Default is None. guidance (torch.Tensor, optional): Guidance tensor. Default is None. conditioning_scale (float, optional): Scaling factor for conditioning. Default is 1.0. Returns: torch.Tensor: The output of the forward pass. Nrr6r1r )rYr) hidden_statesencoder_hidden_statesrotary_pos_embemb)rrr.cg|]}|qSrrrbsamplerrrreRz*FluxControlNet.forward..crrrrrrrreSr)rnror|shaperVr3reshapepermuter}todtyperprs time_projrqr)catrl enumeratervrrwziprxr{r() r~rrrrrrrrrrr batch_sizechannels height_pwwidth_pwheightwidthvec_embidsrdouble_block_samplesid_blockblocksingle_block_samplesrcontrolnet_double_block_samplesdouble_block_sample control_blockbiascontrolnet_single_block_samplessingle_block_samplerrrforwardsz                zFluxControlNet.forward)NNNNNNr!)rMrNrOrPr0rjrJrr)r* LongTensorrTr __classcell__rrrrrUks> U#  rUcs2eZdZdZdedeffdd ZddZZS)FluxControlnetForwardWrapperz] A wrapper combines flux and flux controlnet forward pass for easier initialization. flux_configflux_controlnet_configcsTt||j|_|jD]}d|_qt||_|j r(|j |jdSdS)zL Create flux and flux controlnet instances by their config. FN) rirjrVconfigure_modelrr requires_gradrUflux_controlnetrJ)r~rrparamrrrrjds   z%FluxControlnetForwardWrapper.__init__c CsF|j|||||d|||d\} } |j||||d|||| | d } | S)zJ Forward pass for the FluxControlnetForwardWrapper model. r)rrrrrrrr) rrrrrrrrr)rr) r~packed_noisy_model_input control_image prompt_embedspooled_prompt_embedsrlatent_image_idstext_ids guidance_vecrr noise_predrrrrrs,   z$FluxControlnetForwardWrapper.forward) rMrNrOrPrr0rjrrrrrrr_srcsneZdZdZdedeffdd ZddZdd Zd d Z dd e j fddZ d e j fddZ dddZZS)MegatronFluxControlNetModela Megatron wrapper for flux controlnet model. Args: flux_params (FluxModelParams): Parameters to configure the Flux model. flux_controlnet_config (FluxControlNetConfig): Configuration specific to the FluxControlNet. Methods: configure_model: Configures the model by wrapping the FluxControlNet with the appropriate layers and settings, configuring the VAE, scheduler, and text encoders. data_step: A wrapper around the data-step function specific to FluxControlNet, controlling how data is processed. forward: Executes a forward pass through FluxControlNet. training_step: A wrapper step method that calls forward_step with a data batch from data loader. forward_step: Handles the forward pass specific to training, computing the model's output. validation_step: Calls inference pipeline with current model weights and save inference result together with the control image. flux_paramsrcs"t|||_|j|dSN)rirjroptimconnect)r~rrrrrrjs z$MegatronFluxControlNetModel.__init__cCst|dsEt|j|j|_||j|||j |j |j D]"\}}|jj dkr9d|vs6d|vr9d|_ d|vrDd|vrDd|_ q$dSdS) zo Initialize flux and controlnet modules, vae, scheduler, and text encoders with given configs. rrcontextaddedFrwzself_attention.linear_proj.biasN)hasattrrrVrr configure_vae vae_configconfigure_schedulerconfigure_text_encoders clip_params t5_paramsnamed_parametersr9r)r~namerrrrrs   z+MegatronFluxControlNetModel.configure_modelcCs |j|S)zz Retrive data batch from dataloader iterator and do necessary processing before feeding into train steps. )rrL)r~r,rrr data_steps z%MegatronFluxControlNetModel.data_stepcOs|jjj}||i|S)z6 Calling the controlnet forward pass. )r)r~argskwargsforward_wrapperrrrrs z#MegatronFluxControlNetModel.forwardNreturncCs ||S)z\ A wrapper method takes data batch and returns the results of forward_step. ) forward_step)r~r- batch_idxrrr training_steps z)MegatronFluxControlNetModel.training_stepc s@|jjjr tj|_n|jjjrtj|_ntj|_|j r,|dj dd}|dj dd}n&|dj dd}|j |j |jd}|dj dd}|j |j |jd}|j|jd|jd |jd |j|jd }|j||jd|jd |jd |jd d }|j||jd|jd |jd |jd d dd }|jd}tj||j|jd} |d|} | |jj} |jj| j |jd} |jjj |j|jd} |jjj |jdfdd| D}| j |jd} | |}t|j|jkr|d}t|j|jksd|||| }|dd }|jjr%tj |jd f|jj!|j|jd}nd}|j"rH|dj dddd }|dj dd}|dj dd}n|d}|j#||j|jd\}}}tj j$j%|jtj&tjfv|jd+|j'||||| |||d}| |dd }t(j)||dd}|WdS1swYdS)z1 The main forward step function. latentsTr"control_latentsimages)rhintrrr )rrrdevicerr1)rnum_channels_latentsrr)rr logit_normalrcsg|] }|kqSr)nonzeroitem)rbtschduler_timestepsrrresz.rr!Nrrrr)rrrrrrrrmean) reduction)*rrVbf16r)bfloat16autocast_dtypefp16rTfloat32image_precachedr+vaeencoder_prepare_latent_image_idsrr _pack_latents transpose randn_liker%compute_density_for_timestep_sampling schedulernum_train_timestepslongrsigmasflattenr(ndim unsqueezer?fullrKtext_precached encode_promptampautocasthalfrFmse_loss)r~r-rrrrrrrnoiseuindicesrr! step_indicessigmarrrrrrrtargetlossrr rrs             &z(MegatronFluxControlNetModel.forward_stepc Cstdddlm}||j|j|jjjj|j|j |j |jj |jjjj d}|j r|jr|djdd}|djdd}|d jdddd }|d jdd}||||||jd |j|jd |jdd d|jdd } | d|jjd|jd|jdn]|djdd} |djdd} |d} || | dd | jd | jd d|jdd } | d|jjd|jd|jd| d|| } || } | d|jjd|jd|jdtjdgtjdS)z Initialize flux controlnet pipeline with current model components. Saves the inference results together with the hint image to log folder. zStart validation stepr)FluxControlNetInferencePipeline)paramscontorlnet_configrrt5clipscheduler_stepsrrTr"rrr1rrr g@F) rrrrrrnum_inference_stepsnum_images_per_promptrKr save_to_diskz/step=_rankz.pngrrr)rr;r<rrrKrr=rz _control.pnggr )rr/nemo.collections.diffusion.models.flux.pipeliner4r5rrrrr7r8r9rrr&r+rrvae_scale_factorrsaveloggerlog_dir global_step local_ranktorch_to_numpy numpy_to_pilr)tensorcurrent_device) r~r-rr4piperrrr log_imagesrrtextrrrvalidation_stepJsd     * .  (z+MegatronFluxControlNetModel.validation_stepr)rMrNrOrPrr0rjrrrr)r*rrrMrrrrrrser)- contextlibr dataclassesrtypingrr)torch.nnr megatron.corer7megatron.core.models.common.vision_module.vision_moduler$megatron.core.tensor_parallel.layersr,megatron.core.transformer.transformer_configrr r+4nemo.collections.diffusion.models.dit.dit_layer_specr r r r -nemo.collections.diffusion.models.flux.layersrrr,nemo.collections.diffusion.models.flux.modelrrr8nemo.collections.diffusion.models.flux_controlnet.layersrnemo.lightningr nemo.utilsrrr/IOMixinr0rUrrrrrrs0            u>