o eik@sFddlZddlmZmZddlmZmZmZmZm Z m Z m Z m Z ddl mZddlmZddlmZddlmZdd lmZdd lmZmZmZd d lmZmZeeZ Gd ddeZ!GdddeZ"GdddeZ#Gddde Z$Gddde Z%Gddde Z&GdddeZ'GdddeZ(GdddeZ)gdZ*dS) N)InstructBlipQFormerConfigInstructBlipVisionConfig)'BaseModelOutputWithVisionQformerOutputs$InstructBlipForConditionalGeneration/InstructBlipForConditionalGenerationModelOutputInstructBlipModelInstructBlipPreTrainedModelInstructBlipQFormerModelInstructBlipVisionModelTransformersKwargs)PreTrainedConfig)FlashAttentionKwargs)BaseModelOutputWithPooling)!MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)Unpack)auto_docstringcan_return_tuplelogging)CONFIG_MAPPING AutoConfigc@ eZdZdS)InstructBlipVideoVisionConfigN__name__ __module__ __qualname__rr}/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/instructblipvideo/modular_instructblipvideo.pyr-rc@r)InstructBlipVideoQFormerConfigNrrrrrr!1r r!csDeZdZdZdZddiZeeedZ     d fdd Z Z S) InstructBlipVideoConfiga [`InstructBlipVideoConfig`] is the configuration class to store the configuration of a [`InstructBlipVideoForConditionalGeneration`]. It is used to instantiate a Instructblipvideo model according to the specified arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the Instructblipvideo [Salesforce/instruct-blip-flan-t5](https://huggingface.co/Salesforce/instruct-blip-flan-t5) architecture. Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. Args: vision_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`InstructBlipVideoVisionConfig`]. qformer_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`InstructBlipVideoQFormerConfig`]. text_config (`dict`, *optional*): Dictionary of configuration options used to initialize any [`PreTrainedConfig`]. num_query_tokens (`int`, *optional*, defaults to 32): The number of query tokens passed through the Transformer. video_token_index (`int`, *optional*): Token index of special video token. kwargs (*optional*): Dictionary of keyword arguments. Example: ```python >>> from transformers import ( ... InstructBlipVideoVisionConfig, ... InstructBlipVideoQFormerConfig, ... OPTConfig, ... InstructBlipVideoConfig, ... InstructBlipVideoForConditionalGeneration, ... ) >>> # Initializing a InstructBlipVideoConfig with Salesforce/instruct-blip-flan-t5 style configuration >>> configuration = InstructBlipVideoConfig() >>> # Initializing a InstructBlipVideoForConditionalGeneration (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration >>> model = InstructBlipVideoForConditionalGeneration(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a InstructBlipVideoConfig from a InstructBlipVideoVisionConfig, InstructBlipVideoQFormerConfig and any PreTrainedConfig >>> # Initializing Instructblipvideo vision, Instructblipvideo Q-Former and language model configurations >>> vision_config = InstructBlipVideoVisionConfig() >>> qformer_config = InstructBlipVideoQFormerConfig() >>> text_config = OPTConfig() >>> config = InstructBlipVideoConfig(vision_config=vision_config, qformer_config=qformer_config, text_config=text_config) ```instructblipvideovideo_token_idvideo_token_index) text_configqformer_config vision_configN c s|durtd}tdnt|tr#|dd}t|di|}|dur0t}tdn t|trInstructBlipVideoForConditionalGeneration.get_placeholder_maskrQrRrSrUrVlabelsrWrYc KsP| dur| n|jj} |j|f||| dd|}|j}|j}|j}|dur+||}|dur4t|}| |j |j }|j ||d}| ||}|jjr{|jd ||| | | |d|}| ra|jn|d}d}| durz|jd || |jjjd|}n$|jd ||||| | | | |d |}| r|jn|d}| r|jn|d }t|||||d S) a qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. Examples: ```python >>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration >>> import torch >>> from huggingface_hub import hf_hub_download >>> import av >>> import numpy as np >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto") >>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b") >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample uniformly 4 frames from the videWhy is this video funny?o >>> total_frames = container.streams.video[0].frames >>> indices = np.arange(0, total_frames, total_frames / 4).astype(int) >>> clip = read_video_pyav(container, indices) >>> prompt = "What is happening in the video?" >>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device) >>> outputs = model.generate( ... **inputs, ... do_sample=False, ... num_beams=5, ... max_length=256, ... repetition_penalty=1.5, ... length_penalty=1.0, ... ) >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip() >>> print(generated_text) "A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front" ```NTrNrOrXrWrTrer)logitsr vocab_size) rTrQrRrSrUrVrWrrYra)lossrrfrgrhr)rirjrrrgrfrzrnrtrr^r]rrr4ryr loss_functionr&rrrK)r9rMrNrOrPrQrRrSrTrUrVrrWrXrYr:rrrgrfrrrrrrrr{s~Q       z1InstructBlipVideoForConditionalGeneration.forwardcKs t|dr ||jd} |j||||dd} | j} |durM|durG|jjg|jjd} | |jjj g} t j | gt j |j d}|| d}||}|durVt |}| |j |j} |j||d }||| }||d }|jjjsy||d <|jjd i||}|S) a Overrides `generate` function to be able to use the model as a conditional generator. Args: pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width) or (batch_size, num_frames, num_channels, height, width)): Input images or videos to be processed. qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): The sequence used as a prompt for the generation. attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Embedded representation of the inputs. Should be float, not int tokens. interpolate_pos_encoding (`bool`, *optional*, defaults to `False`): Whether to interpolate the positional encoding of the image embeddings. Returns: captions (list): A list of strings of length batch_size * num_captions. hf_device_maprTrNr\rar)rTrQrPr)hasattr_preprocess_acceleraterkrrrir%r1r& bos_token_idrnr{rqr^repeatrzrtrr]rrryis_encoder_decodergenerate)r9rMrNrOrPrQrTrXgenerate_kwargsrrr video_tokens start_tokensrinputsrrrrr s8 "       z2InstructBlipVideoForConditionalGeneration.generate)NF) NNNNNNNNNNFN)NNNNNF)rrrrrrnrrrrr rrrrrrKrno_gradrrrrrrsG       r)r"r!rrIrCrJrLr)+rn;transformers.models.instructblip.configuration_instructbliprr6transformers.models.instructblip.modeling_instructbliprrrrrr r r configuration_utilsr modeling_flash_attention_utilsrmodeling_outputsrmodels.auto.modeling_autorprocessing_utilsrutilsrrrautorr get_loggerrr,rr!r"rCrIrJrKrLr__all__rrrrs.(     mm 8