o ei3e@sdZddlmZddlZddlmZddlmZddlm Z ddl m Z dd l m Z mZmZdd lmZmZdd lmZdd lmZmZmZmZdd lmZmZddlmZddlm Z e!e"Z#Gdddej$Z%Gdddej$Z&  d9dej$dej'dej'dej'dej'dBde(dBde(deefddZ)Gd d!d!ej$Z*Gd"d#d#ej$Z+Gd$d%d%ej$Z,Gd&d'd'ej$Z-Gd(d)d)ej$Z.Gd*d+d+e Z/Gd,d-d-ej$Z0Gd.d/d/ej$Z1eGd0d1d1eZ2eGd2d3d3e2Z3ed4d5Gd6d7d7e2Z4gd8Z5dS):zPyTorch ViViT model.)CallableN)nn)initialization)ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringlogging torch_int)can_return_tuplemerge_with_config_defaults)capture_outputs) VivitConfigcsBeZdZdZdeffdd Zd dejdedejfd d Z Z S) VivitTubeletEmbeddingsa Construct Vivit Tubelet embeddings. This module turns a batch of videos of shape (batch_size, num_frames, num_channels, height, width) into a tensor of shape (batch_size, seq_len, hidden_size) to be consumed by a Transformer encoder. The seq_len (the number of patches) equals (number of frames // tubelet_size[0]) * (height // tubelet_size[1]) * (width // tubelet_size[2]). configcs|t|j|_|j|_|j|_|j|jd|j|jd|j|jd|_|j|_t j |j |j|j|jd|_ dS)Nrr) kernel_sizestride) super__init__ num_frames image_size tubelet_size patch_size num_patches hidden_size embed_dimrConv3d num_channels projectionselfr __class__f/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/vivit/modeling_vivit.pyr/s  zVivitTubeletEmbeddings.__init__F pixel_valuesinterpolate_pos_encodingreturnc Cs|j\}}}}}|s+||jks||jkr+td|d|d|jdd|jdd |ddddd }||}|ddd}|S) NzImage image size (*z) doesn't match model (rrz).rr)shaper ValueErrorpermuter'flatten transpose) r)r.r/ batch_sizerr&heightwidthxr,r,r-forward?s( zVivitTubeletEmbeddings.forwardF) __name__ __module__ __qualname____doc__rrtorchTensorboolr< __classcell__r,r,r*r-r$s $rcs`eZdZdZdeffdd Zdejdededejfd d Z dd ejd e dejfddZ Z S)VivitEmbeddingsz Vivit Embeddings. Creates embeddings from a video using VivitTubeletEmbeddings, adds CLS token and positional embeddings. rcsttttdd|j|_t||_ ttd|j j d|j|_ t |j |_|jdd|_||_dS)Nr)rrr ParameterrBzerosr# cls_tokenrpatch_embeddingsr"position_embeddingsDropouthidden_dropout_probdropoutr r!rr(r*r,r-rWs   zVivitEmbeddings.__init__ embeddingsr9r:r0c Cs|jdd}|jjdd}tjs||kr||kr|jS|jddddf}|jddddf}|jd}||jd} ||jd} t|d} |d| | |}|dddd}t j j || | fdd d }|dddd dd|}tj ||fdd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rNrg?rrbicubicF)sizemode align_cornersdim)r3rKrBjit is_tracingr!rreshaper5r functional interpolateviewcat) r)rOr9r:r" num_positionsclass_pos_embedpatch_pos_embedrV new_height new_widthsqrt_num_positionsr,r,r-r/es(   z(VivitEmbeddings.interpolate_pos_encodingFr.r/c Csr|j\}}}}}|j||d}|j|ddg} tj| |fdd}|r-|||||}n||j}||}|S)Nr/rrU) r3rJrItilerBr]r/rKrN) r)r.r/r8rr&r9r:rO cls_tokensr,r,r-r<s  zVivitEmbeddings.forwardr=) r>r?r@rArrrBrCintr/rDr<rEr,r,r*r-rFPs $(rFmodulequerykeyvalueattention_maskscalingrNkwargsc Ks|dur |dd}t||dd|}|dur||}tjj|dd}tjj|||jd}t||} | dd } | |fS)NrPrrrU)ptrainingr) rRrBmatmulr7rrZsoftmaxrNrr contiguous) rirjrkrlrmrnrNro attn_weights attn_outputr,r,r-eager_attention_forwards  rxcsBeZdZdeffdd ZdejdeejejffddZZ S)VivitSelfAttentionrcst|j|jdkrt|dstd|jd|jd||_|j|_t|j|j|_|j|j|_ |j |_ |jd|_ d|_ tj|j|j |jd|_tj|j|j |jd|_tj|j|j |jd|_dS) Nrembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .rpF)bias)rrr#num_attention_headshasattrr4rrgattention_head_size all_head_sizeattention_probs_dropout_prob dropout_probrn is_causalrLinearqkv_biasrjrkrlr(r*r,r-rs"   zVivitSelfAttention.__init__ hidden_statesr0c Cs|jd}|d|j|jf}||j|dd}||j|dd}||j|dd}t |j j t }|||||d|j |j|jsHdn|jd\}} |dd|jf} || }|| fS)NrrPrrrh)rrnrN)r3r}rrkr\r7rlrjr get_interfacer_attn_implementationrxrrnrrrrRrrY) r)rr8 new_shape key_layer value_layer query_layerattention_interface context_layerattention_probsnew_context_layer_shaper,r,r-r<s*   zVivitSelfAttention.forward) r>r?r@rrrBrCtupler<rEr,r,r*r-rys(rycsBeZdZdZdeffdd Zdejdejdejfdd ZZ S) VivitSelfOutputz The residual connection is defined in VivitLayer instead of here (as is the case with other models), due to the layernorm applied before each block. rcs.tt|j|j|_t|j|_dSN) rrrrr#denserLrMrNr(r*r,r-r zVivitSelfOutput.__init__r input_tensorr0cCs||}||}|SrrrNr)rrr,r,r-r<s  zVivitSelfOutput.forward r>r?r@rArrrBrCr<rEr,r,r*r-rs$rc8eZdZdeffdd ZdejdejfddZZS)VivitAttentionrcs"tt||_t||_dSr)rrry attentionroutputr(r*r,r-rs  zVivitAttention.__init__rr0cCs||\}}|||}|Sr)rr)r)rself_attn_output_rr,r,r-r<s zVivitAttention.forward r>r?r@rrrBrCr<rEr,r,r*r-rrcr)VivitIntermediatercsRtt|j|j|_t|j|_ t |j t r#t |j |_dS|j |_dSr)rrrrr#intermediate_sizerrLrMrN isinstance hidden_actstrrintermediate_act_fnr(r*r,r-rs   zVivitIntermediate.__init__rr0cCs"||}||}||}|Sr)rrrN)r)rr,r,r-r<s   zVivitIntermediate.forwardrr,r,r*r-rs rcs>eZdZdeffdd ZdejdejdejfddZZS) VivitOutputrcs.tt|j|j|_t|j|_ dSr) rrrrrr#rrLrMrNr(r*r,r-r!rzVivitOutput.__init__rrr0cCs ||}||}||}|Srrrr,r,r-r<&s  zVivitOutput.forwardrr,r,r*r-r s$rcs<eZdZdZdeffdd ZdejdejfddZZ S) VivitLayerzNThis corresponds to the EncoderBlock class in the scenic/vivit implementation.rcsbt|j|_d|_t||_t||_t||_ t j |j |j d|_t j |j |j d|_dS)Nreps)rrchunk_size_feed_forward seq_len_dimrrr intermediaterrr LayerNormr#layer_norm_epslayernorm_beforelayernorm_afterr(r*r,r-r0s    zVivitLayer.__init__rr0cCs@||}||}||}||}||}|||}|Sr)rrrrr)r)rhidden_states_normattention_output layer_outputr,r,r-r<:s     zVivitLayer.forwardrr,r,r*r-r-s rcs6eZdZdeffdd ZdejdefddZZ S) VivitEncoderrcs:t|_tfddtjD|_d|_dS)Ncsg|]}tqSr,)r).0rrr,r- Osz)VivitEncoder.__init__..F) rrrr ModuleListrangenum_hidden_layerslayergradient_checkpointingr(r*rr-rLs   zVivitEncoder.__init__rr0cCs&t|jD]\}}||}qt|dS)N)last_hidden_state) enumeraterr)r)ri layer_moduler,r,r-r<Rs  zVivitEncoder.forward) r>r?r@rrrBrCrr<rEr,r,r*r-rKsrcr) VivitPoolerrcs*tt|j|j|_t|_dSr)rrrrr#rTanh activationr(r*r,r-rZs zVivitPooler.__init__rr0cCs(|dddf}||}||}|S)Nr)rr)r)rfirst_token_tensor pooled_outputr,r,r-r<_s  zVivitPooler.forwardrr,r,r*r-rYrrcs^eZdZUeed<dZdZdZdZdgZ dZ dZ dZ dZ eedZefdd ZZS) VivitPreTrainedModelrvivitr.videoTr)r attentionscs6t|t|trt|jt|jdSdS)zInitialize the weightsN)r _init_weightsrrFinitzeros_rIrK)r)rir*r,r-rys   z"VivitPreTrainedModel._init_weights)r>r?r@r__annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendrry_can_record_outputsrBno_gradrrEr,r,r*r-rhs rc sleZdZddedeffdd ZddZeedd e dd e j d Bd ed e e defddZZS) VivitModelTradd_pooling_layercsXt|||_t||_t||_tj|j |j d|_ |r#t |nd|_ |dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer rN)rrrrFrOrencoderrrr#r layernormrpooler post_init)r)rrr*r,r-rs    zVivitModel.__init__cCs|jjSr)rOrJ)r)r,r,r-get_input_embeddingsszVivitModel.get_input_embeddingsF)tie_last_hidden_statesNr.r/ror0cKs\|durtd|j||d}||}|j}||}|jdur&||nd}t||dS)a Examples: ```python >>> import av >>> import numpy as np >>> from transformers import VivitImageProcessor, VivitModel >>> from huggingface_hub import hf_hub_download >>> np.random.seed(0) >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len): ... ''' ... Sample a given number of frame indices from the video. ... Args: ... clip_len (`int`): Total number of frames to sample. ... frame_sample_rate (`int`): Sample every n-th frame. ... seg_len (`int`): Maximum allowed index of sample's last frame. ... Returns: ... indices (`list[int]`): List of sampled frame indices ... ''' ... converted_len = int(clip_len * frame_sample_rate) ... end_idx = np.random.randint(converted_len, seg_len) ... start_idx = end_idx - converted_len ... indices = np.linspace(start_idx, end_idx, num=clip_len) ... indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64) ... return indices >>> # video clip consists of 300 frames (10 seconds at 30 FPS) >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample 32 frames >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=1, seg_len=container.streams.video[0].frames) >>> video = read_video_pyav(container=container, indices=indices) >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400") >>> model = VivitModel.from_pretrained("google/vivit-b-16x2-kinetics400") >>> # prepare video for the model >>> inputs = image_processor(list(video), return_tensors="pt") >>> # forward pass >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 3137, 768] ```Nz You have to specify pixel_valuesrd)r pooler_output)r4rOrrrrr )r)r.r/roembedding_outputencoder_outputssequence_outputrr,r,r-r<sT   zVivitModel.forward)T)NF)r>r?r@rrDrrrrrrB FloatTensorr rr r<rEr,r,r*r-rs"ra ViViT Transformer model with a video classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for Kinetics-400. Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained position embeddings to the higher resolution. ) custom_introc s`eZdZdeffdd Zee   d dejdBdej dBde d e e d e f d d ZZS)VivitForVideoClassificationrcsRt||j|_t|dd|_|jdkrt|j|jnt|_ | dS)NF)rr) rr num_labelsrrrrr#Identity classifierrr(r*r,r-rs $ z$VivitForVideoClassification.__init__NFr.labelsr/ror0c Ksp|j|fd|i|}|j}||dddddf}d}|dur.|j|||jfi|}t|||j|jdS)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> import av >>> import numpy as np >>> import torch >>> from transformers import VivitImageProcessor, VivitForVideoClassification >>> from huggingface_hub import hf_hub_download >>> np.random.seed(0) >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len): ... ''' ... Sample a given number of frame indices from the video. ... Args: ... clip_len (`int`): Total number of frames to sample. ... frame_sample_rate (`int`): Sample every n-th frame. ... seg_len (`int`): Maximum allowed index of sample's last frame. ... Returns: ... indices (`list[int]`): List of sampled frame indices ... ''' ... converted_len = int(clip_len * frame_sample_rate) ... end_idx = np.random.randint(converted_len, seg_len) ... start_idx = end_idx - converted_len ... indices = np.linspace(start_idx, end_idx, num=clip_len) ... indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64) ... return indices >>> # video clip consists of 300 frames (10 seconds at 30 FPS) >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample 32 frames >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=4, seg_len=container.streams.video[0].frames) >>> video = read_video_pyav(container=container, indices=indices) >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400") >>> model = VivitForVideoClassification.from_pretrained("google/vivit-b-16x2-kinetics400") >>> inputs = image_processor(list(video), return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) ... logits = outputs.logits >>> # model predicts one of the 400 Kinetics-400 classes >>> predicted_label = logits.argmax(-1).item() >>> print(model.config.id2label[predicted_label]) LABEL_116 ```r/Nr)losslogitsrr)rrr loss_functionrr rr) r)r.rr/rooutputsrrrr,r,r-r<s$\z#VivitForVideoClassification.forward)NNF)r>r?r@rrrrrBr LongTensorrDr rr r<rEr,r,r*r-rs$ r)rrr)Nrh)6rAcollections.abcrrBrrr activationsrmodeling_layersrmodeling_outputsrr r modeling_utilsr r processing_utilsr utilsrrrr utils.genericrrutils.output_capturingrconfiguration_vivitr get_loggerr>loggerModulerrFrCfloatrxryrrrrrrrrrr__all__r,r,r,r-sj         ,V 3  u {