o }oi@sdddlmZmZmZddlmZddlZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlmZd d lmZd d lmZd d lmZd d lmZde dej!dej"fddZ#ej$dfdej%de&fddZ'dej$dfde deej!dej%de&fddZ(   d$de dee deeeeefde&dee&d ee d!e)fd"d#Z*dS)%)ListOptionalUnionN)CommonInferenceParams)AbstractModelInferenceWrapper)InferenceWrapperConfig)Image) AutoProcessor)vlm)LlavaInferenceWrapper)MllamaInferenceWrapper) VLMEngine)VLMTextGenerationControllerpathtrainermodelcCs|}|||}|S)z)Setup trainer and restore model from path) to_fabric load_model)rrrfabricrW/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/vlm/inference/base.py _setup_trainer_and_restore_model%s ri params_dtype&inference_batch_times_seqlen_thresholdc Cs|j}|jj}||}|t|tjrt}|j j }nt|tj r,t }|j j }ntd|||t||||jd}|S)z&Set up inference wrapper for the modelzUnknown model config: ) hidden_sizerrpadded_vocab_size)configmodulecudatoeval isinstancer MLlamaModelConfigr language_model_configr LlavaConfigr language_transformer_config ValueErrorr vocab_size) r tokenizerrrr mcore_model wrapper_clsrinference_wrapped_modelrrrsetup_inference_wrapper,s*       r-c CsNd}t|}|j}t}tj||d}t|||dt||||} | |fS)zSet up model and tokenizerz(meta-llama/Llama-3.2-11B-Vision-Instruct)r))rrr)r from_pretrainedr)r MLlamaConfig11BInstruct MLlamaModelrr-) rrrrmodel_id processorr)rrr,rrrsetup_model_and_tokenizerOs r3 wrapped_modelpromptsimagesmax_batch_size random_seedinference_paramsreturnc Cs>t|||d}t|||d} |ptdd} | j||| d} | S)ae Generates text using a NeMo VLM model. Args: wrapped_model (AbstractModelInferenceWrapper): The model inference wrapper. tokenizer: tokenizer for the input text, image_processor: image processor for the input image, prompts (list[str]): The list of prompts to generate text for. images (list): The list of images to generate text for. max_batch_size (int, optional): The maximum batch size. Defaults to 4. random_seed (Optional[int], optional): The random seed. Defaults to None. inference_params (Optional["CommonInferenceParams"], optional): The inference parameters defined in Mcore's CommonInferenceParams. Defaults to None. Returns: list[Union["InferenceRequest", str]]: A list of generated text, either as a string or as an InferenceRequest object. )r,r)image_processor)text_generation_controllerr8r92)num_tokens_to_generate)r6r7common_inference_params)rrrgenerate) r5r)r<r6r7r8r9r:r= mcore_enginer@resultsrrrrAesrA)r4NN)+typingrrrlightning.pytorchpytorchpltorchtorch.distributed/megatron.core.inference.common_inference_paramsrQmegatron.core.inference.model_inference_wrappers.abstract_model_inference_wrapperrImegatron.core.inference.model_inference_wrappers.inference_wrapper_configr PIL.Imager transformersr nemo.lightning lightningnlnemo.collectionsr llava_inference_wrapperr mllama_inference_wrapperr vlm_enginervlm_inference_controllerrstrTrainerLightningModulerbfloat16dtypeintr-r3dictrArrrrsj              %