o }o™iã @s¬ddlZddlmZddlmmZddlmZddlm Z m Z m Z m Z m Z mZddlmZmZddlmZddlmZddlmZmZmZGdd „d ejƒZGd d „d ejƒZed krÔed ddZedd„e ¡DƒƒZ e!e ƒe "dddd¡Z#e "ddd ¡Z$e %dd¡Z&e %d¡Z'ee#e'e$e&ƒZ(e!e(j)ƒedddZ*edd„e* ¡DƒƒZ e!e ƒe*e +dddd¡e +dddd¡e %d¡e +ddd ¡e %dd¡ƒZ(e!e(j)ƒdSdS)éN)ÚSelfAttentionPooling)ÚConditionalSequentialÚDBlockÚFusedCrossAttentionBlockÚResBlockÚStackedCrossAttentionBlockÚUBlock)ÚLearnedSinusoidalPosEmbÚUnLearnedSinusoidalPosEmb)Ú Downsample)ÚUpsampleLearnable)ÚlinearÚ normalizationÚ zero_modulecsreZdZdZdddgd¢dddddd gd ¢d d dd d d d d d d d f‡fd d„ Z ddd„Zdddœdd„Z‡ZS)Ú UNetModelaŠ The full UNet model with attention and timestep embedding used for Imagen Base and SR model. :param embed_dim: Dimension of embeddings. Also used to calculate the number of channels in ResBlock. :param image_size: Input image size. Used to calculate where to inject attention layers in UNet. :param channels: Input channel number, defaults to 3. :param text_embed_dim: Dimension of conditioned text embedding. Different text encoders and different model versions have different values, defaults to 512 :param num_res_blocks: Number of ResBlock in each level of UNet, defaults to 3. :param channel_mult: Used with embed_dim to calculate the number of channels for each level of UNet, defaults to [1, 2, 3, 4] :param num_attn_heads: The number of heads in the attention layer, defaults to 4. :param per_head_channels: The number of channels per attention head, defaults to 64. :param cond_dim: Dimension of Conditioning projections, defaults to 512. :param attention_type: Type of attention layer, defaults to 'fused'. :param feature_pooling_type: Type of pooling, defaults to 'attention'. :param learned_sinu_pos_emb_dim: Dimension of learned time positional embedding. 0 for unlearned timestep embeddings. Defaults to 16 :param attention_resolutions: List of resolutions to inject attention layers. Defaults to [8, 16, 32] :param dropout: The rate of dropout, defaults to 0. :param use_null_token: Whether to create a learned null token for attention, defaults to False. :param init_conv_kernel_size: Initial Conv kernel size, defaults to 3. :param gradient_checkpointing: Whether to use gradient checkpointing, defaults to False. :param scale_shift_norm: Whether to use scale shift norm, defaults to False. :param stable_attention: Whether to use numerically-stable attention calculation, defaults to True. :param flash_attention: Whether to use flash attention calculation, defaults to False. :param resblock_updown: Whether to use ResBlock or Downsample/Upsample, defaults to False. :param resample_with_conv: When resblock_updown=False, whether to use conv in addition to Pooling&ConvTranspose. Defaults to True. :param low_res_cond: Whether conditioned on low-resolution input, used for SR model. Defaults to False. :param noise_cond_aug: Whether to add noise conditioned augmentation with low-resolution input. Defaults to False. éé)ééréré@ÚfusedÚ attentioné)éré FTc1sàtƒ ¡| dkr t}n| dkrt}ntd | ¡ƒ‚|d}| dks$J‚| dkrDt| ƒ}| d}t |t  ||¡t  ¡t  ||¡¡|_ nt |ƒ}t |t ||ƒt  ¡t ||ƒ¡|_ | dksc| dkscJ‚| |_| dkr|t t|d t |¡t  || ¡¡|_t || ƒ|_t t | ¡t  | |¡t  ¡t  ||¡¡|_|rªt tjdd| |jjjd ¡|_||_g}t| ƒ} ||_| D] }| |t|ƒ¡q¸||_||_ |j rú|jsÔJd ƒ‚| dkrát| ƒ}| d} nt |ƒ}|} t |t  | |¡t  ¡t  ||¡¡|_!|rd |n|}!||d}"t"tj#|!|"||d d ƒ|_$t%|tƒr%|gt&|ƒ}#n|}#d }$|"}%d}&t '|j$g¡|_(|%g}'t)|ƒD]v\}(})|#|(}t*|ƒD]6}*|)|}+t+|%|||+|$||ddg},|+}%|&|vrp|, ||%|||||| d¡|j( t"|,Ž¡|' |%¡qH|(t&|ƒdk}-|-s±|j( t"|ržt+|%|||%|$||ddd nt,|%||$|%dƒ¡|' |%¡|&d 9}&qt+|%|||%|$||ddd nt1|%||$|%d¡|&d }&|j. t"|,Ž¡qôqæt t2|%ƒt  ¡t3tj#|"|||d d ƒ¡|_4dS)NÚstackedrzAttention {} not definedrrrrÚmean©Ú input_dim©ÚdtypezVnoise conditioning augmentation should only be enabled when training with low-res condr©ÚpaddingT)ÚchannelsÚ emb_channelsÚdropoutÚ out_channelsÚdimsÚuse_checkpointÚuse_scale_shift_normÚlearnable_upsampling)r$Ú num_headsÚnum_head_channelsr)Ústable_attentionÚflash_attentionÚ context_dim) r$r%r&r'r(r)r*Údownr+)r$Úuse_convr(r')r$r%r&r(r)r*r+éÿÿÿÿ) r$r%r&r'r(r)r*Úupr+)5ÚsuperÚ__init__rrÚ ValueErrorÚformatr ÚnnÚ SequentialÚLinearÚSiLUÚ time_embedr r Úfeature_pooling_typerÚ LayerNormÚattention_poolingÚ text_to_condÚto_text_non_attn_condÚ ParameterÚtorchÚrandnÚweightr!Únull_text_embeddingÚuse_null_tokenÚsortedÚ image_sizeÚappendÚintÚ low_res_condÚnoise_cond_augÚlowres_time_embedrÚConv2dÚ init_convÚ isinstanceÚlenÚ ModuleListÚ input_blocksÚ enumerateÚrangerr Ú middle_blockÚ output_blocksÚlistÚpopÚUpsamplerrÚout)1ÚselfÚ embed_dimrJr$Útext_embed_dimÚnum_res_blocksÚ channel_multÚnum_attn_headsÚper_head_channelsÚcond_dimÚattention_typer>Úlearned_sinu_pos_emb_dimÚattention_resolutionsr&rHÚinit_conv_kernel_sizeÚgradient_checkpointingÚscale_shift_normr.r/Úresblock_updownÚresample_with_convrMrNÚ attention_fnÚtime_embed_dimÚ sinu_pos_embÚsinu_pos_emb_input_dimÚ attention_dsÚresÚlowres_sinu_pos_embÚlowres_sinu_pos_emb_dimÚ in_channelsÚinit_dimÚres_blocks_listÚCONV_DIMÚchÚdsÚnum_input_block_channelsÚlevelÚmultÚ_r'ÚlayersÚ is_last_levelÚiÚichÚ is_last_block©Ú __class__©út/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/multimodal/modules/imagen/diffusionmodules/nets.pyr6Bs¦    üÿ ý   ü ÿþ   ü ÿ  øÿ ùÿ  õ÷ óÿ €ù ù ùê "øÿ ùÿ  õ÷ óÑ1 ýzUNetModel.__init__NcCs@|jr |dus Jdƒ‚n|dusJdƒ‚|jr |dusJdƒ‚n|dus(Jdƒ‚|durL|j|jkrC|j\}}}} tj||| fdd}tj||gdd}|jd |j} } |j|jksa|j|jkr„|j} |j | d }|j | d }|durz|j | d }|dur„|j | d }|  |¡} |jr•|  |¡}| |7} |  |¡}|j r¿|j | dd¡}tj||gdd}tj|t | d¡  | ¡gdd}n|}|}|jd krÏ|jd d}n |jd krÙ| |¡}| |¡}| |7} |}g}|jD]}||| ||ƒ}| |¡qé| || ||¡}|jD]}| ¡}tj||gdd}||| ||ƒ}q| |¡S)Nzx_low_res cannot be Nonezx_low_res cannot be presentedúEtime_low_res cannot be None when training with noise conditioning augú time_low_res cannot be presentedÚbicubic©Úmoder©Údimrr réþÿÿÿr)rMrNÚshapeÚFÚ interpolaterDÚcatÚdevicer!Útor=rOrArHrGÚrepeatÚonesr>rr@rBrUrKrXrYr[r])r^ÚxÚtimeÚ text_embedÚ text_maskÚ x_low_resÚ time_low_resrÚ new_heightÚ new_widthÚ batch_sizer•r!ÚtÚlowres_tÚ text_condÚ null_contextÚ context_embÚ context_maskÚpooled_text_condÚ text_hiddensÚhÚhsÚmoduleÚh_prevr‡r‡rˆÚforwardSs`        "        zUNetModel.forwardçð?©r›Ú cond_scalecOóJ|j|d|i|¤Ž}|dkr|S|j|dt |¡i|¤Ž}||||S©Nr›r¯©r®rDÚ zeros_like©r^r›r±ÚargsÚkwargsÚlogitsÚ null_logitsr‡r‡rˆÚforward_with_cond_scale™ó z!UNetModel.forward_with_cond_scale)NNNN©Ú__name__Ú __module__Ú __qualname__Ú__doc__r6r®r»Ú __classcell__r‡r‡r…rˆr$s<!ç ÿFrcspeZdZdZddgd¢dddgd¢d d gd ¢d d dd ddd d d f‡fdd„ Z ddd„Zdddœdd„Z‡ZS)ÚEfficientUNetModela¦ The full Efficient UNet model with attention and timestep embedding used for Imagen SR model. :param embed_dim: Dimension of embeddings. Also used to calculate the number of channels in ResBlock. :param image_size: Input image size. Used to calculate where to inject attention layers in UNet. :param channels: Input channel number, defaults to 3. :param text_embed_dim: Dimension of conditioned text embedding. Different text encoders and different model versions have different values, defaults to 512 :param channel_mult: Used with embed_dim to calculate the number of channels for each level of UNet, defaults to [1, 1, 2, 4, 8]. :param num_attn_heads: The number of heads in the attention layer, defaults to 8. :param per_head_channels: The number of channels per attention head, defaults to 64. :param attention_type: Type of attention layer, defaults to 'fused'. :param atnn_enabled_at: Whether to enable attention at each level, defaults to [0, 0, 0, 0, 1]. :param feature_pooling_type: Type of pooling, defaults to 'attention'. :param stride: Stride in ResBlock, defaults to 2. :param num_resblocks: Used with num_res_blocks to calculate the number of residual blocks at each level of Efficient-UNet. Defaults to [1, 2, 4, 8, 8]. :param learned_sinu_pos_emb_dim: Dimension of learned time positional embedding. 0 for unlearned timestep embeddings. Defaults to 16 :param use_null_token: Whether to create a learned null token for attention, defaults to False. :param init_conv_kernel_size: Initial Conv kernel size, defaults to 3. :param gradient_checkpointing: Whether to use gradient checkpointing, defaults to False. :param scale_shift_norm: Whether to use scale shift norm, defaults to False. :param stable_attention: Whether to use numerically-stable attention calculation, defaults to True. :param flash_attention: Whether to use flash attention calculation, defaults to False. :param skip_connection_scaling: Whether to use 1/sqrt(2) scaling for ResBlock skip connection, defaults to False. :param noise_cond_aug: Whether to add noise conditioned augmentation with low-resolution input. Defaults to False. rr)rrrrrrrr)rrrrrrr)rrrrrrFTc%s¸tƒ ¡t|ƒ|_||_|d}| dksJ‚| dkr7t| ƒ}| d}t |t ||¡t  ¡t ||¡¡|_ nt |ƒ}t |t ||ƒt  ¡t ||ƒ¡|_ ||_ |j rx| dkr_t| ƒ}| d}nt |ƒ}|}t |t ||¡t  ¡t ||¡¡|_|}| dks„| dks„J‚| |_| dkrt t|dt |¡t ||¡¡|_t ||ƒ|_t t |¡t ||¡t  ¡t ||¡¡|_|rËt tjdd||jjjd¡|_||_|d}||d}tj||||dd |_t ¡|_t ¡|_ |}t!|ƒD]Y\}} | |}!|t|ƒdk}"| |r |nd}#t"|ƒ}$t#||t$| |ƒ||" | |!|#|||||||d |j|$<t%t$| |ƒ||||" | |!|#|||||||d |j |$<t$| |ƒ}qôt |d||d¡|_&dS) Nrrrrrrr rr")r$r%r'r*Ú conv_downÚstrideÚ num_resblocksrfr`r,r-r)r.r/Úskip_connection_scaling)r$r%r'r*Úconv_uprÅrÆrfr`r,r-r)r.r/rÇ)'r5r6rSÚn_levelsrJr r9r:r;r<r=r r rNrOr>rr?r@rArBrCrDrErFr!rGrHrPrQÚ ModuleDictÚDBlocksÚUBlocksrVÚstrrrLrr])%r^r_rJr$r`rbrcrdrfÚatnn_enabled_atr>rÅrÆrgrHrirjrkr.r/rÇrNrorprqrtrurervrwrzr}r~Ú num_resblockrÚlevel_attention_typeÚ level_keyr…r‡rˆr6¼s %    üÿ   ü ý   ü      ñ  ñzEfficientUNetModel.__init__NcCs@|jr |dus Jdƒ‚n|dusJdƒ‚|j|jks |j|jkrC|j}|j|d}|j|d}|dur9|j|d}|durC|j|d}|jd|j}} | |¡} | |¡} |j|jkrm|j\} } } }tj|| |fdd}t j ||gdd}|jr‚|  |¡}| |7} |j r§|j  |dd¡}t j | |gdd}t j |t  |d¡ | ¡gdd}n| }|}|jd kr·| jd d}n |jd krÁ| |¡}| |¡}| |7} | |¡}tƒ}t|jƒD]}t|ƒ}|j||| ||ƒ}||jdkrò|||<q×t|jdd d ƒD]}t|ƒ}||jdkr|||}|j||| ||ƒ}qü| |¡S) Nr‰rŠr rr‹rŒrrŽrrrr3)rNr!r–r‘r•r=rAr’r“rDr”rOrHrGr—r˜r>rr@rBrQÚdictrWrÉrÍrËrÌr])r^r™ršr›rœrržr!r¡r•r¢r¤rrŸr r£r¥r¦r§r¨r©Úfeatsr}rÑr‡r‡rˆr®Ts^        "     €  zEfficientUNetModel.forwardr¯r°cOr²r³r´r¶r‡r‡rˆr»™r¼z*EfficientUNetModel.forward_with_cond_scale©Nr½r‡r‡r…rˆrás4Þ ÿErÃÚ__main__rr)r_rJccó|]}| ¡VqdSrÔ©Únumel©Ú.0Úpr‡r‡rˆÚ ¤ó€rÜrréXé€éccrÖrÔr×rÙr‡r‡rˆrܱrÝ),rDÚtorch.nnr9Útorch.nn.functionalÚ functionalr’ÚEnemo.collections.multimodal.modules.imagen.diffusionmodules.attentionrÚBnemo.collections.multimodal.modules.imagen.diffusionmodules.blocksrrrrrrÚ@nemo.collections.multimodal.modules.imagen.diffusionmodules.embsr r ÚBnemo.collections.multimodal.modules.imagen.diffusionmodules.layersr r r\r rrÚModulerrÃr¾ÚmodelÚsumÚ parametersÚpytorch_total_paramsÚprintÚrandÚ image_batchr¤r˜rœršÚoutputr‘Úmodel_srrEr‡r‡r‡rˆÚsJ            ûç