o }oiJ@sBddlmZddlmZmZmZddlmZddl Z ddl Z ddl Z ddl mmZddlmZzddlmZWneyOddlmZdZedYnwddlmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z%dd l&m'Z'ddl(m)Z)ddl*m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:m;Z;mfddZ?dee@e j>ffddZAddZBeGdd d e)e=jCZDGd!d"d"eZeGd#d$d$e)e=jCZEGd%d&d&eZFeGd'd(d(e)e=jCZGGd)d*d*e"ZHGd+d,d,ejIe=jCe=jJe.jKZLdS)-) dataclass)CallableDictOptionalN) ModelType)TENorm)loggingzFailed to import Transformer Engine dependencies. `from megatron.core.extensions.transformer_engine import TENorm`If using NeMo Run, this is expected. Otherwise, please verify the Transformer Engine installation.)GPTModel) CLIPViTModel)OptimizerConfig)MegatronModule) AttnMaskType) ModuleSpec)TransformerConfig)tqdm) TokenizerSpec)fn)transformer_engine_layer_spec)default_layer_spec)+build_imagenet_validation_dataloader_params))average_losses_across_data_parallel_group)ClipMegatronLoss)MegatronOptimizerModuleOptimizerModuleget_vocab_sizeioreturncCs |d|dd}|di|S)Nimagescaptions)rrr)modelbatch forward_argsrrX/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/clip/model/base.pyclip_forward_step;sr$cCslt|}t|trt|dkr|d}n|}d|vr+t|djdkr+|d|d<dd|D}|S)NrrcSs*i|]\}}||dur|jddndqS)NT non_blocking)cuda).0keyvalrrr# Ks*z"clip_data_step..)next isinstancetuplelenshapesqueezeitems)dataloader_iterr!_batchrrr#clip_data_step@s r6cCdSNrselftensorrrr#set_input_tensorOsr<c@seZdZUdZdZeed<dZeed<dZ eed<dZ eed <d Z eed <d Z eed <d Z eed<eZeed<dZeed<dZeed<dZeed<dZeed<dddZdS) CLIPViTConfigzClip ViT model config output_dimTadd_class_tokenclass_token_len patch_dimimg_himg_wclipvision_model_typetransformer_layer_specFgated_linear_unitattention_softmax_in_fp32 num_layersnum_attention_headsrr c Csd|j}t|tsddlm}|dd}tj|jjj d<||_t |||j |j |j |j|j|j|jd S)Nr)get_layer_spec_teT)is_vitattn_mask_type)rJr@rBrDrFrG model_subtyper?)rJr.r nemo.collections.vlm.layer_specsrPMCoreAttnMaskTypeno_mask submodulesself_attentionparamsr r@rBrDrFrGrIr?)r:rJrPrrr#configure_modelhs"   zCLIPViTConfig.configure_modelN)rr )__name__ __module__ __qualname____doc__r?int__annotations__r@boolrBrDrFrGrIstrrrJrrKrLrNrOrZrrrr#r=Ts             r=csjeZdZdZ       dded ed ed ed ed edededeffdd Z ddZ fddZ Z S)r zClip ViT modelTrArCrErHtransformer_configrJr@rBrDrFrGrSr?c sZ| |_tj||||||||dt|j|jj|jjd|_tj j |jj|jdd|_ dS)N)rdrJr@rBrDrFrGrSconfig hidden_sizeepsFbias) r?super__init__rrfrglayernorm_epsilonfinal_layernormtorchnnLinearhead) r:rdrJr@rBrDrFrGrSr? __class__rr#rls*  zCLIPViTModel.__init__cCr7r8rr9rrr#r<zCLIPViTModel.set_input_tensorcs4t|}||}|dddf}||}|S)Nr)rkforwardrnrr)r:xrsrr#rvs  zCLIPViTModel.forward)TrArCrErErHrc) r[r\r]r^rrrar_rbrlr<rv __classcell__rrrsr#r s<  'r c@sfeZdZUdZdZeed<dZeed<dZeed<dZ e ed <dZ e ed <e Z eed <ddddZd S)CLIPTextModelConfigzClip text model configr>r?make_vocab_size_divisible_byrcmax_seq_lengthF#share_embeddings_and_output_weights&use_transformer_engine_full_layer_specrJNr CLIPTextModelc Cs|j}t|ts ||}t|dr,|j}|dur+td|d|jd||jdnt||j|j}t ||||j |j |j dS)N vocab_sizezUse preset vocab_size: z, original vocab_size: z, dummy tokens: .)rdrJrmax_sequence_lengthr?r}) rJr.rhasattrrrinforr{rr|r?r})r: tokenizer pre_process post_processrJrrrr#rZs*  z#CLIPTextModelConfig.configure_model)NN)rr)r[r\r]r^r?r_r`r{r|r}rar~rrJrrZrrrr#rys       ryc sTeZdZdZ  ddededededed ef fd d Zfd d Z ddZ Z S)rzClip text modelrcFrdrJrrr?r}c s~||_t||||dd|t|j|jj|jjd|_tj j |jj|jdd|_ d|_ |j r=t|dd|_ dSdS)NTFrerirM)r?rkrlrrfrgrmrnrorprqrr position_idsrarangeexpandr()r:rdrJrrr?r}rsrr#rls0  zCLIPTextModel.__init__csLtj||jdd}||}||jddt|jdf}||}|S)N)rattention_maskrdimrM) rkrvrrnargmaxrorr1rr)r: input_idsrwrsrr#rv s    zCLIPTextModel.forwardcCr7r8rr9rrr#r<ruzCLIPTextModel.set_input_tensor)rcF) r[r\r]r^rrr_rarlrvr<rxrrrsr#rs& &rc@seZdZUdZdZeeed<dZee ed<dZ e ed<e Z eed<eZeed<d Zeed <d Zeed <d Zeed<dZeed<dddZdS) CLIPConfigzClip model configNtext_transformer_configvision_transformer_configTget_attention_mask_from_fusionforward_step_fn data_step_fnrMrNrArOirgP seq_lengthcCst|jt||||dS)Nrrr)print kv_channelsMCoreClipModel)r:rrrrrr#rZ+s zCLIPConfig.configure_modelTT)r[r\r]r^rrryr`rr=rrar$rrr6rrNr_rOrgrrZrrrr#rs        rcsJeZdZdZddeddffdd Zdejd ejfd d Zd d Z Z S)rz Clip modelTrfrNcsvtj|d||_||_|j}|j}|jj|_||_|j|||d|_ t j t gtd|_tj|_dS)Nrfrg$I$I,@)rkrlrrrrr?rZ vision_model text_modelrorp Parameteronesnplog logit_scalerencoder_or_decoder model_type)r:rfrrrrrrsrr#rl9s   zMCoreClipModel.__init__rrcCsF||}||}|jrtj|ddtj|dd|jfS||fS)Nrr)rrrF normalizerexp)r:rrimage_features text_featuresrrr#rvIs  $zMCoreClipModel.forwardcCr7r8rr9rrr#r<RruzMCoreClipModel.set_input_tensorr) r[r\r]r^rrlroTensorrvr<rxrrrsr#r6s  rcseZdZdZ      d*dedeededd eed ed ed effd d Z ddZ d+ddZ de j de j fddZdeee j ffddZde j fddZd,de j fddZd,de j fddZd d!Zd"d#Zd$d%Zedefd&d'Zedefd(d)ZZS)- CLIPModela CLIPModel is the base class for all CLIP models. Args: config: CLIPConfig. The configuration of the CLIP model. Please see the `CLIPConfig` for details. optim: OptimizerModule. This module is just used for init and the actual optimizer is created via trainer API. tokenizer: TokenizerSpec. This module is used for deciding the output length of the language model. # These parameters are just for imagenet validation imagenet_val: Optional[str] = None: Optional path to imagenet validation dataset. mbs: int = 8: Batch size for imagenet validation. gbs: int = 8: Global Batch for imagenet validation. max_workers: int = 4: Maximum number of workers used for imagenet validation. NrArfoptimrr imagenet_valmbsgbs max_workerscsbt||_||_|pttdddd|_|j|d|_d|_ ||_ ||_ ||_ ||_ dS)Ng-C6?T)lruse_distributed_optimizerr)rkrlrfrrr rconnect_training_loss_reduction_validation_loss_reductionrrrr)r:rfrrrrrrrsrr#rlis   zCLIPModel.__init__c CsH|jdur"t|j|jjj|jjj|j|j|j|jj j |j d|_dSdS)z|d ur?q2|j d d tj }|j d d}tj j j d tj d||} tj| dd} d| |} Wd n1suwY|| |dd\} } || 7}|| 7}||d7}q2Wd n1swYtd||}||}||fS)z,Zero shot evaluation for imagenet validationrMcsF|t|dddd}||dd|fdd|DS)NrMTrcs8g|]}td|djdddqS)NrrT)keepdim)floatreshapesumcpunumpy)r)kcorrectrr# s8z>CLIPModel.zero_shot_eval..accuracy..)topkmaxteqview expand_as)outputtargetrpredrrr#accuracysz*CLIPModel.zero_shot_eval..accuracyzStarting zero-shot imagenet.zBuilding zero-shot classifierzUsing classifier)rrrzImagenet Zero-shot EvaluationF)descleaveNTr&rrrgY@)rM)rrzFinished zero-shot imagenet.)r)rrrrrrorrrr(torrrrrsize) r:r classifiervision_encodertop1top5nrrrlogitsacc1acc5rrr#zero_shot_evals>          zCLIPModel.zero_shot_evalcCsl|jdur4td}|\|d<|d<t|}|jd|ddddd|jd|ddddddSdS) z0Run zero shot evaluation for imagenet validationNrrM imagenet_top1T)prog_barrank_zero_only batch_size imagenet_top5)rrozerosr(rrr)r:imagenet_metricrrr#on_validation_epoch_ends z!CLIPModel.on_validation_epoch_endcC|jst|_|jSr8)rrrrrr#training_loss_reductionz!CLIPModel.training_loss_reductioncCrr8)rrrrrr#validation_loss_reductionr z#CLIPModel.validation_loss_reduction)NNNrArAr)rNr8)r[r\r]r^rrrrbr_rlrrZrorrvrrrrrrrrpropertyrr r rxrrrsr#rWsJ / r)M dataclassesrtypingrrrlightning.pytorchpytorchLrrrotorch.distributedtorch.nn.functionalrp functionalrmegatron.core.enumsr+megatron.core.extensions.transformer_enginer ImportError nemo.utilsrwarningmegatron.core.models.gptr MCoreGPTModel*megatron.core.models.vision.clip_vit_modelr MCoreCLIPViTModelmegatron.core.optimizerr megatron.core.transformerr megatron.core.transformer.enumsr rU$megatron.core.transformer.spec_utilsr,megatron.core.transformer.transformer_configrr1nemo.collections.common.tokenizers.tokenizer_specrnemo.collections.llmrnemo.collections.llm.gpt.modelr#nemo.collections.llm.gpt.model.baser2nemo.collections.multimodal.data.clip.clip_datasetr2nemo.collections.nlp.modules.common.megatron.utilsr(nemo.collections.vlm.clip.loss.clip_lossrnemo.lightningrrrrrr$rbr6r<IOMixinr=ryrrrLightningModuleConnectorMixinFNMixinrrrrr#s^                    +9)6"!