o
    ei?                    @   s  d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1 e*2e3Z4dZ5ee)ddG dd deZ6ee)ddG dd deZ7G dd dej8Z9G d d! d!ej8Z:G d"d# d#ej8Z;G d$d% d%ej8Z<G d&d' d'ej8Z=G d(d) d)ej8Z>G d*d+ d+ej8Z?G d,d- d-ej8Z@G d.d/ d/ej8ZA		0ded1ej8d2ejBd3ejBd4ejBd5ejBdB d6eCdB d7eCd8e$e( fd9d:ZDG d;d< d<ej8ZEG d=d> d>ej8ZFG d?d@ d@ej8ZGG dAdB dBej8ZHG dCdD dDeZIG dEdF dFej8ZJG dGdH dHej8ZKe)G dIdJ dJe"ZLG dKdL dLeLZMe)dMdG dNdO dOeLZNe)dPdG dQdR dReLZOG dSdT dTej8ZPG dUdV dVej8ZQG dWdX dXej8ZRe)dYdG dZd[ d[eLZSe)d\dG d]d^ d^eLZTG d_d` d`ej8ZUe)dadG dbdc dceLZVg ddZWdS )fzPyTorch BridgeTower Model    )OrderedDict)Callable)	dataclassN)nn)CrossEntropyLoss   )initialization)ACT2FNQuickGELUActivation)CacheDynamicCacheEncoderDecoderCache)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputModelOutputSequenceClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)TransformersKwargsauto_docstringlogging	torch_int)can_return_tuple   )BridgeTowerConfigBridgeTowerTextConfigBridgeTowerVisionConfigRobertaTokenizerz.
    Output type of [`BridgeTowerModel`].
    )custom_introc                   @   st   e Zd ZU dZdZejdB ed< dZejdB ed< dZ	ejdB ed< dZ
eej dB ed< dZeej dB ed< dS )BridgeTowerModelOutputa  
    text_features (`torch.FloatTensor` of shape `(batch_size, text_sequence_length, hidden_size)`):
        Sequence of hidden-states at the text output of the last layer of the model.
    image_features (`torch.FloatTensor` of shape `(batch_size, image_sequence_length, hidden_size)`):
        Sequence of hidden-states at the image output of the last layer of the model.
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size x 2)`):
        Concatenation of last layer hidden-state of the first token of the text and image sequence (classification
        token), respectively, after further processing through layers used for auxiliary pretraining tasks.
    Ntext_featuresimage_featurespooler_outputhidden_states
attentions)__name__
__module____qualname____doc__r&   torchFloatTensor__annotations__r'   r(   r)   tupler*    r3   r3   r/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/transformers/models/bridgetower/modeling_bridgetower.pyr%   1   s   
 
r%   z>
    Output type of ['BridgeTowerForContrastiveLearning']
    c                   @   s   e Zd ZU dZdZejdB ed< dZejdB ed< dZ	e
ej dB ed< dZe
ej dB ed< dZe
ej dB ed< dZe
ej dB ed< dZe
ej dB ed	< dS )
BridgeTowerContrastiveOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Image-text contrastive loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    text_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
        The text embeddings obtained by applying the projection layer to the pooler_output.
    image_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
        The image embeddings obtained by applying the projection layer to the pooler_output.
    cross_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
        The text-image cross-modal embeddings obtained by applying the projection layer to the pooler_output.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
    Nlosslogitstext_embedsimage_embedscross_embedsr)   r*   )r+   r,   r-   r.   r6   r/   r0   r1   r7   r8   r2   r9   r:   r)   r*   r3   r3   r3   r4   r5   I   s   
 r5   c                       sN   e Zd Z fddZdejdejfddZd
dejdejdB fdd	Z  ZS )BridgeTowerResidualAttentionc              
      s   t    t|j|jd | _tj|j|jd| _t	t
dt|j|jd fdt fdt|jd |jfg| _tj|j|jd| _d | _d S )N@   epsc_fc   geluc_proj)super__init__r   MultiheadAttentionhidden_sizeattn	LayerNormlayer_norm_epsln_1
ModuleDictr   Linearr
   mlpln_2	attn_maskselfconfig	__class__r3   r4   rD   j   s   
	
z%BridgeTowerResidualAttention.__init__hidden_stateattention_maskc                 C   sZ   |d ur|j tj|jd}| jd ur| jj |j|jdnd | _| j|||d| j|dd S )NdtypedeviceF)need_weightsrO   key_padding_maskr   )tor/   boolrY   rO   rX   rG   )rQ   rU   rV   r3   r3   r4   	attention{   s    
z&BridgeTowerResidualAttention.attentionNc                 C   sD   ||  | || }| |}| j D ]}||}q|| }|S N)r^   rJ   rN   rM   values)rQ   rU   rV   residual_statelayerr3   r3   r4   forward   s   

z$BridgeTowerResidualAttention.forwardr_   )	r+   r,   r-   rD   r/   Tensorr^   rc   __classcell__r3   r3   rS   r4   r;   i   s    $r;   c                       s8   e Zd Z fddZddejdejdB fddZ  ZS )	BridgeTowerTransformerc                    sr   t     j| _ j| _ jr#t fddt| jd D | _nt fddt| jD | _ j	| _	d S )Nc                       g | ]}t  qS r3   r;   .0_rR   r3   r4   
<listcomp>       z3BridgeTowerTransformer.__init__.<locals>.<listcomp>r   c                    rg   r3   rh   ri   rl   r3   r4   rm      rn   )
rC   rD   rF   num_hidden_layersremove_last_layerr   
ModuleListrange	resblocksstop_gradientrP   rS   rl   r4   rD      s   
zBridgeTowerTransformer.__init__NrU   rV   c                 C   s>   g }| j D ]}|||}| jr||  q|| q|S r_   )rs   rt   appenddetach)rQ   rU   rV   r)   blockr3   r3   r4   rc      s   

zBridgeTowerTransformer.forwardr_   r+   r,   r-   rD   r/   rd   rc   re   r3   r3   rS   r4   rf      s    $rf   c                       sX   e Zd Zdef fddZdejdededejfdd	Zddej	dejfddZ
  ZS )BridgeTowerVisionEmbeddingsrR   c                    s   t    || _|j| _|j| _|j| _tt	
| j| _tj|j| j| j| jdd| _| j| j d | _| jd | _t| j| j| _| jdt	| jddd d S )NF)in_channelsout_channelskernel_sizestridebias   r   position_idsr   
persistent)rC   rD   rR   rF   	embed_dim
image_size
patch_sizer   	Parameterr/   randnclass_embeddingConv2dnum_channelspatch_embeddingnum_patchesnum_positions	Embeddingposition_embeddingregister_bufferarangeexpandrP   rS   r3   r4   rD      s"   
"z$BridgeTowerVisionEmbeddings.__init__
embeddingsheightwidthreturnc                 C   s  |j d d }| jjd}|j d d }tj s(||kr(||kr(| | jS |ddddf }|ddddf }|j d }	|| j }
|| j }t	|d }|
d|||	}|dddd}tjj||
|fdd	d
}|dddddd|	}tj||fddS )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   Nr         ?r   r   bicubicF)sizemodealign_cornersdim)shaper   weight	unsqueezer/   jit
is_tracingr   r   r   reshapepermuter   
functionalinterpolateviewcat)rQ   r   r   r   r   r   r   class_pos_embedpatch_pos_embedr   
new_height	new_widthsqrt_num_positionsr3   r3   r4   interpolate_pos_encoding   s*   



z4BridgeTowerVisionEmbeddings.interpolate_pos_encodingFpixel_valuesc              
   C   s   |j \}}}}|s&|| jks|| jkr&td| d| d| j d| j d	| jjj}| |j|d}|ddd}| j	
|dd}	tj|	|gdd	}
|r[|
| |
|| }
|
S |
| | j }
|
S )
NzInput image size (*z) doesn't match model (z).rX   r   r   r   r   )r   r   
ValueErrorr   r   rX   r\   flatten	transposer   r   r/   r   r   r   r   )rQ   r   r   
batch_sizerk   r   r   target_dtypepatch_embedsclass_embedsr   r3   r3   r4   rc      s    
z#BridgeTowerVisionEmbeddings.forwardF)r+   r,   r-   r"   rD   r/   rd   intr   r0   rc   re   r3   r3   rS   r4   ry      s     )ry   c                       s\   e Zd Z fddZ	ddejdefddZ	ddejdefdd	Zd
ejfddZ	  Z
S )BridgeTowerVisionTransformerc                    s|   t    t | _tj j jd| _t	 | _
tj j jd| _ j| _ js<t fddt jD | _d S d S )Nr=   c                    s   g | ]}t j j jd qS )r=   )r   rH   rF   rI   ri   rl   r3   r4   rm     s    z9BridgeTowerVisionTransformer.__init__.<locals>.<listcomp>)rC   rD   ry   r   r   rH   rF   rI   ln_prerf   transformerln_postshare_layernormrq   rr   ro   ln_separaterP   rS   rl   r4   rD     s   



z%BridgeTowerVisionTransformer.__init__Fr   r   c                 C   s   |  ||}| |}|ddd}| ||}tj|dd}|dddd}| jr1| |}|S g }t|| j	D ]\}}||}|
| q9tj|dd}|S )Nr   r   r   r   r   )r   r   r   r   r/   stackr   r   zipr   ru   )rQ   r   rV   r   r)   hidden_states_stacklnr3   r3   r4   rc     s   

z$BridgeTowerVisionTransformer.forwardc                 C   s*   | j ||d}| |}|ddd}|S )Nr   r   r   r   )r   r   r   )rQ   r   r   r)   r3   r3   r4   forward_pre,  s   
z(BridgeTowerVisionTransformer.forward_prerU   c                 C   s   | ddd}| |}|S )Nr   r   r   )r   r   )rQ   rU   visual_output_postr3   r3   r4   forward_post7  s   
z)BridgeTowerVisionTransformer.forward_postr   )r+   r,   r-   rD   r/   rd   r]   rc   r   r   re   r3   r3   rS   r4   r     s    

r   c                       $   e Zd Z fddZdd Z  ZS )BridgeTowerLinkTowerc                    s   t    |j| _|j| _|jdv r;|jdkr!ttd| _n|jdkr/ttd| _	tj
| j|jd| _
d S td|j d)	N)add
scaled_addr   r   g      ?r   r   r=   link_tower_type  is not implemented)rC   rD   link_tower_typerF   r   r   r/   tensorscaled_factorbetarH   rI   NotImplementedErrorrP   rS   r3   r4   rD   >  s   



zBridgeTowerLinkTower.__init__c                 C   sp   | j dkr| || S | j dkr| || j | S | j dkr/| |d| j  || j  S td| j  d)Nr   r   r   r   r   r   )r   rH   r   r   r   )rQ   r)   cross_modal_hidden_statesrV   r3   r3   r4   rc   K  s   


zBridgeTowerLinkTower.forwardr+   r,   r-   rD   rc   re   r3   r3   rS   r4   r   =  s    r   c                       8   e Zd Z fddZdejdejdejfddZ  ZS )BridgeTowerSelfOutputc                    sB   t    t|j|j| _tj|j|jd| _t|j	| _
d S Nr=   )rC   rD   r   rL   rF   denserH   rI   Dropouthidden_dropout_probdropoutrP   rS   r3   r4   rD   X     
zBridgeTowerSelfOutput.__init__r)   input_tensorr   c                 C   &   |  |}| |}| || }|S r_   r   r   rH   rQ   r)   r   r3   r3   r4   rc   ^     

zBridgeTowerSelfOutput.forwardrx   r3   r3   rS   r4   r   W      $r   c                       2   e Zd Z fddZdejdejfddZ  ZS )BridgeTowerIntermediatec                    sD   t    t|j|j| _t|jt	rt
|j | _d S |j| _d S r_   )rC   rD   r   rL   rF   intermediate_sizer   
isinstance
hidden_actstrr	   intermediate_act_fnrP   rS   r3   r4   rD   g  s
   
z BridgeTowerIntermediate.__init__r)   r   c                 C   s   |  |}| |}|S r_   )r   r   rQ   r)   r3   r3   r4   rc   o  s   

zBridgeTowerIntermediate.forwardrx   r3   r3   rS   r4   r   f  s    r   c                       r   )BridgeTowerOutputc                    sB   t    t|j|j| _tj|j|jd| _t	|j
| _d S r   )rC   rD   r   rL   r   rF   r   rH   rI   r   r   r   rP   rS   r3   r4   rD   w  r   zBridgeTowerOutput.__init__r)   r   r   c                 C   r   r_   r   r   r3   r3   r4   rc   }  r   zBridgeTowerOutput.forwardrx   r3   r3   rS   r4   r   v  r   r   c                       r   )BridgeTowerPoolerc                    s*   t    t|j|j| _t | _d S r_   )rC   rD   r   rL   rF   r   Tanh
activationrP   rS   r3   r4   rD     s   
zBridgeTowerPooler.__init__r)   r   c                 C   s(   |d d df }|  |}| |}|S )Nr   )r   r   )rQ   r)   first_token_tensorpooled_outputr3   r3   r4   rc     s   

zBridgeTowerPooler.forwardrx   r3   r3   rS   r4   r     s    r           modulequerykeyvaluerV   scalingr   kwargsc           
      K   s   |d u r| dd }t||dd| }|d ur|| }tjj|dd}tjj||| jd}t||}	|	dd	 }	|	|fS )Nr         r   r   r   )ptrainingr   )
r   r/   matmulr   r   r   softmaxr   r   
contiguous)
r   r   r   r   rV   r   r   r   attn_weightsattn_outputr3   r3   r4   eager_attention_forward  s   
r  c                       sd   e Zd Zd fdd	Z			ddejdejdB dedB dejdB d	ee	 d
e
ej fddZ  ZS )BridgeTowerSelfAttentionFNc                    s   t    |j|j dkrt|dstd|j d|j d|| _|j| _t|j|j | _| j| j | _	| jd | _
t|j| j	| _t|j| j	| _t|j| j	| _t|j| _|j| _|| _|| _d S Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r   )rC   rD   rF   num_attention_headshasattrr   rR   r   attention_head_sizeall_head_sizer   r   rL   r   r   r   r   attention_probs_dropout_probr   
is_decoder	is_causal	layer_idxrQ   rR   r  r  rS   r3   r4   rD     s&   


z!BridgeTowerSelfAttention.__init__r)   rV   past_key_valuescache_positionr   r   c                 K   s  |j d d }g |d| jR }| |j| dd}| |j| dd}	| |j| dd}
|d urP|}t|trC|j	}|
|	|
| jd|i\}	}
t| jjt}|| ||	|
|f| jsddn| jj| jd|\}}|jg |dR   }||fS )Nr   r   r   r  r   r   r   )r   r
  r   r   r   r   r   r   r   self_attention_cacheupdater  r   get_interfacerR   _attn_implementationr  r   r   r   r   r   r   )rQ   r)   rV   r  r  r   input_shapehidden_shapequery_layer	key_layervalue_layercurrent_past_key_valuesattention_interfacer  r  r3   r3   r4   rc     s@   


z BridgeTowerSelfAttention.forwardFNNNNr+   r,   r-   rD   r/   rd   r0   r   r   r   r2   rc   re   r3   r3   rS   r4   r    s$    r  c                       sd   e Zd Zd fdd	Z			ddejdejdB dejdB dedB d	ee	 d
e
ej fddZ  ZS )BridgeTowerCrossAttentionFNc                    s   t    |j|j dkrt|dstd|j d|j d|| _|j| _t|j|j | _| j| j | _	| jd | _
t|j| j	| _t|j| j	| _t|j| j	| _t|j| _|| _|| _d S r  )rC   rD   rF   r  r	  r   rR   r   r
  r  r   r   rL   r   r   r   r   r  r   r  r  r  rS   r3   r4   rD     s$   


z"BridgeTowerCrossAttention.__init__r)   encoder_hidden_statesrV   r  r   r   c                 K   sN  |j d d \}}|j d }||d| jf}	||d| jf}
| |j|	 dd}|d ur3|j| jnd}|d urL|rL|jj	| j j
}|jj	| j j}n-| |j|
 dd}| |j|
 dd}|d ury|j||| j\}}d|j| j< t| jjt}|| ||||f| jsdn| jj| jd|\}}|||d }||fS )Nr   r   r   FTr   r  )r   r
  r   r   r   
is_updatedgetr  cross_attention_cachelayerskeysr`   r   r   r  r   r  rR   r  r  r   r   r   r   r   r   )rQ   r)   r#  rV   r  r   bsztgt_lensrc_lenq_input_shapekv_input_shaper  r$  r  r  r  r  r  r3   r3   r4   rc     sB   	


z!BridgeTowerCrossAttention.forwardr  r   )r+   r,   r-   rD   r/   rd   r0   r   r   r   r2   rc   re   r3   r3   rS   r4   r"    s$    r"  c                       s|   e Zd Zd fdd	Z					ddejdejdB dejdB dejdB d	edB d
ejdB dee	 de
ej fddZ  ZS )BridgeTowerAttentionFNc                    s:   t    || _|rtnt}||||d| _t|| _d S )Nr  r  )rC   rD   is_cross_attentionr"  r  rQ   r   output)rQ   rR   r  r  r0  attention_classrS   r3   r4   rD   K  s
   
zBridgeTowerAttention.__init__r)   rV   r#  encoder_attention_maskr  r  r   r   c           
      K   sB   | j s|n|}| j|f||||d|\}}	| ||}||	fS )N)r#  rV   r  r  )r0  rQ   r1  )
rQ   r)   rV   r#  r3  r  r  r   attention_outputr  r3   r3   r4   rc   R  s   

zBridgeTowerAttention.forward)FNFNNNNNr!  r3   r3   rS   r4   r.  J  s0    
	r.  c                       s@   e Zd Zd	 fdd	Z			d
dee fddZdd Z  ZS )BridgeTowerBertCrossLayerNc                    sb   t    |j| _d| _t|d|d| _|j| _|j| _t|d|dd| _t	|| _
t|| _d S )Nr   Tr/  Fr  r  r0  )rC   rD   chunk_size_feed_forwardseq_len_dimr.  r^   r  add_cross_attentioncrossattentionr   intermediater   r1  rQ   rR   r  rS   r3   r4   rD   j  s   

z"BridgeTowerBertCrossLayer.__init__r   c                 K   sb   | j |f|d d|\}}|}	| j|	f||||d|\}
}|
}	t| j| j| j|	}|||fS )N)rV   r  )rV   r#  r3  r  )r^   r;  r   feed_forward_chunkr8  r9  )rQ   r)   r#  rV   r3  r  r   self_attention_outputself_attn_weightsr4  cross_attention_outputcross_attn_weightslayer_outputr3   r3   r4   rc   z  s6   	

z!BridgeTowerBertCrossLayer.forwardc                 C      |  |}| ||}|S r_   r<  r1  rQ   r4  intermediate_outputrC  r3   r3   r4   r>       
z,BridgeTowerBertCrossLayer.feed_forward_chunkr_   r   )	r+   r,   r-   rD   r   r   rc   r>  re   r3   r3   rS   r4   r6  i  s    
$r6  c                       s   e Zd Zd fdd	Z					ddejdejdB dejdB dejdB dedB d	ejdB d
ee	 de
ej fddZdd Z  ZS )BridgeTowerTextLayerNc                    s~   t    |j| _d| _t||j|d| _|j| _|j| _| jr3| js*t|  dt|d|dd| _	t
|| _t|| _d S )Nr   r/  z> should be used as a decoder model if cross attention is addedFTr7  )rC   rD   r8  r9  r.  r  r^   r:  r   r;  r   r<  r   r1  r=  rS   r3   r4   rD     s"   

zBridgeTowerTextLayer.__init__r)   rV   r#  r3  r  r  r   r   c                 K   s   d}| j ||f||d|\}	}
|	}| jr<|d ur<t| ds'td|  d| j|	d ||fd|i|\}}|}|f}t| j| j| j|}|||
f S )Nr3   )r  r  r;  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r  )	r^   r  r	  r   r;  r   r>  r8  r9  )rQ   r)   rV   r#  r3  r  r  r   outputsr?  r@  r4  rA  rB  rC  r3   r3   r4   rc     sF   




zBridgeTowerTextLayer.forwardc                 C   rD  r_   rE  rF  r3   r3   r4   r>    rH  z'BridgeTowerTextLayer.feed_forward_chunkr_   r5  )r+   r,   r-   rD   r/   rd   r0   r   r   r   r2   rc   r>  re   r3   r3   rS   r4   rI    s2    	
.rI  c                       s   e Zd Z fddZ								ddejdejdB dejdB dejdB d	edB d
edB dedB dedB dejdB de	e
 deej eB fddZ  ZS )BridgeTowerTextEncoderc                    s4   t     | _t fddt jD | _d S )Nc                    s   g | ]}t  |d qS ))r  )rI  )rj   irl   r3   r4   rm     s    z3BridgeTowerTextEncoder.__init__.<locals>.<listcomp>)rC   rD   rR   r   rq   rr   ro   rb   rP   rS   rl   r4   rD     s
   

zBridgeTowerTextEncoder.__init__NFr)   rV   r#  r3  r  	use_cacheoutput_attentionsoutput_hidden_statesr  r   r   c
                 K   s   |rdnd }|r
dnd }|r| j jrdnd }t| jD ]*\}}||||f|||	d|
}|d }|rE||d f }| j jrE||d f }q|rM||f }t||rS|nd |||dS )Nr3   )r3  r  r  r   r   r   )last_hidden_stater  r)   r*   cross_attentions)rR   r:  	enumeraterb   r   )rQ   r)   rV   r#  r3  r  rM  rN  rO  r  r   all_hidden_statesall_self_attentionsall_cross_attentionsrL  layer_modulelayer_outputsr3   r3   r4   rc     s<   


zBridgeTowerTextEncoder.forward)NNNNNFFN)r+   r,   r-   rD   r/   rd   r0   r   r]   r   r   r2   r   rc   re   r3   r3   rS   r4   rK    sB    
	
rK  c                       s   e Zd ZdZ fddZ					ddejdB dejdB dejdB d	ejdB d
edej	fddZ
edd ZedddZ  ZS )BridgeTowerTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t    tj|j|j|jd| _t|j|j| _	tj
|j|jd| _
t|j| _| jdt|jddd | jdtj| j tjddd |j| _tj|j|j| jd| _d S )	N)padding_idxr=   r   r   Fr   token_type_idsr   )rC   rD   r   r   
vocab_sizerF   pad_token_idword_embeddingstype_vocab_sizetoken_type_embeddingsrH   rI   r   r   r   r   r/   r   max_position_embeddingsr   zerosr   r   longrY  position_embeddingsrP   rS   r3   r4   rD   (  s   
z"BridgeTowerTextEmbeddings.__init__Nr   	input_idsrZ  r   inputs_embedspast_key_values_lengthr   c                 C   s  |d u r|d ur|  || j|}n| || j}|d ur!| }n| d d }|\}}|d u rZt| drO| j|jd d}	tj	|	d|d}	|	||}ntj
|tj| jjd}|d u rc| |}| |}
||
 }| |}|| }| |}| |}|S )Nr   rZ  r   r   )r   indexrW   )"create_position_ids_from_input_idsrY  &create_position_ids_from_inputs_embedsr   r	  rZ  r   r   r/   gatherra  rb  r   rY   r]  r_  rc  rH   r   )rQ   rd  rZ  r   re  rf  r  r   
seq_lengthbuffered_token_type_idsr_  r   rc  r3   r3   r4   rc   <  s2   






z!BridgeTowerTextEmbeddings.forwardc                 C   sJ   |   dd }|d }tj|d || d tj| jd}|d|S )z
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr   r   rW   r   )r   r/   r   rb  rY   r   r   )re  rY  r  sequence_lengthr   r3   r3   r4   ri  l  s   
z@BridgeTowerTextEmbeddings.create_position_ids_from_inputs_embedsc                 C   s6   |  | }tj|dd|| | }| | S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r   r   )ner   r/   cumsumtype_asrb  )rd  rY  rf  maskincremental_indicesr3   r3   r4   rh  ~  s   z<BridgeTowerTextEmbeddings.create_position_ids_from_input_ids)NNNNr   )r   )r+   r,   r-   r.   rD   r/   
LongTensorr0   r   rd   rc   staticmethodri  rh  re   r3   r3   rS   r4   rX  %  s2    
0
rX  c                   @   sF   e Zd ZU eed< dZdZdZddgZdZ	e
 dejfd	d
ZdS )BridgeTowerPreTrainedModelrR   bridgetower)imagetextFr  r;   r  r   c                 C   s  | j j}t|trz| j jd d| j j d  }| j jd }d| j j d }|jjD ]8}tj	|j
j|| d t|j
j tj	|j
jj|| d tj	|jjj|| d tj	|jjj|| d q)tj	|jj|| d tj	|jjj|| d nit|tjtjtjfrtj	|jdd| d nRt|tjrt|j t|j n?t|trt|j| j j  n0t|t!rt"|j#t$%|j&'d nt|t(rt"|j#t$%|j#j)d 'd t|j* t|tjt+fr|jd urt|j d S d S d S )	Nr   r   )stdr   g?)meanry  r   r   ),rR   initializer_factorr   r   rF   ro   r   rs   initnormal_rG   in_proj_weightzeros_in_proj_biasout_projr   rM   r?   rB   r   r   r   r   rL   r   r   rH   r~   ones_!BridgeTowerForContrastiveLearning	constant_logit_scalelogit_scale_init_valuery   copy_r   r/   r   r   r   rX  r   rZ  BridgeTowerMLMHead)rQ   r   ry  proj_stdattn_stdfc_stdrw   r3   r3   r4   _init_weights  s8   



"z(BridgeTowerPreTrainedModel._init_weightsN)r+   r,   r-   r    r1   base_model_prefixinput_modalitiessupports_gradient_checkpointing_no_split_modules_skip_keys_device_placementr/   no_gradr   Moduler  r3   r3   r3   r4   ru    s   
 ru  c                       s@   e Zd ZU eed< dZ fddZedd Zdd	d
Z	  Z
S )BridgeTowerVisionModelrR   )rw  c                    s"   t  | t|| _|   d S r_   )rC   rD   r   visual	post_initrP   rS   r3   r4   rD     s   
zBridgeTowerVisionModel.__init__c                 C   s   | j jjjjS r_   )r  r   r   r   rX   rQ   r3   r3   r4   rX     s   zBridgeTowerVisionModel.dtypeNFc                 K   s   |  || j||S r_   )r  typerX   )rQ   rw  
image_maskr   r   r3   r3   r4   rc     s   zBridgeTowerVisionModel.forward)NF)r+   r,   r-   r"   r1   r  rD   propertyrX   rc   re   r3   r3   rS   r4   r    s   
 
r  a0  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    c                        s   e Zd ZU eed< dZd fdd	Zdd Zdd	 Ze	e
	
	
	
	
	
	
	
	
	
	
	
	
ddejd
B dejd
B dejd
B dejd
B dejd
B dejd
B dejd
B ded
B ded
B ded
B ded
B dejd
B dee deej eB fddZdd Z  ZS )BridgeTowerTextModelrR   )rx  Tc                    sJ   t  | || _d| _t|| _t|| _|rt|nd| _	| 
  dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)rC   rD   rR   gradient_checkpointingrX  r   rK  encoderr   poolerr  )rQ   rR   add_pooling_layerrS   r3   r4   rD     s   

zBridgeTowerTextModel.__init__c                 C      | j jS r_   r   r]  r  r3   r3   r4   get_input_embeddings     z)BridgeTowerTextModel.get_input_embeddingsc                 C      || j _d S r_   r  rQ   r   r3   r3   r4   set_input_embeddings     z)BridgeTowerTextModel.set_input_embeddingsNrd  rV   rZ  r   re  r#  r3  r  rM  rN  rO  r  r   r   c                 K   s  |
d ur|
n| j j}
|d ur|n| j j}| j jr#|	d ur|	n| j j}	nd}	| jr4| jr4|	r4td d}	|	rG|d u rGt	t
| j dt
| j d}|d u |d uA rStd|d ur^|j}|j}n
|j}|jd d }|d }|d urt| nd}|d u rtj||| |d}| j|||||d	}| j||||||d
\}}| j|f|||||	|
|||d	|}|d }| jd ur| |nd }t|||j|j|j|jdS )NFzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...rl   z:You must specify exactly one of input_ids or inputs_embedsr   r   r   rY   )rd  r   rZ  re  rf  )rV   r3  embedding_outputr#  r  r  )	rV   r#  r3  r  rM  rN  rO  r  r   )rP  r(   r  r)   r*   rQ  )rR   rN  rO  r  rM  r  r   loggerwarning_oncer   r   r   rY   r   get_seq_lengthr/   r   r   _create_attention_masksr  r  r   r  r)   r*   rQ  )rQ   rd  rV   rZ  r   re  r#  r3  r  rM  rN  rO  r  r   rY   r  rk  rf  r  encoder_outputssequence_outputr   r3   r3   r4   rc     s   
	zBridgeTowerTextModel.forwardc                 C   sP   | j jrt| j ||||d}nt| j ||d}|d ur$t| j |||d}||fS )N)rR   re  rV   r  r  )rR   re  rV   )rR   re  rV   r#  )rR   r  r   r   )rQ   rV   r3  r  r#  r  r  r3   r3   r4   r  V  s*   	z,BridgeTowerTextModel._create_attention_masks)T)NNNNNNNNNNNN)r+   r,   r-   r!   r1   r  rD   r  r  r   r   r/   rd   r   r]   r   r   r2   r   rc   r  re   r3   r3   rS   r4   r    sd   
 	
^r  zv
    The bare BridgeTower Model transformer outputting BridgeTowerModelOutput object without any specific head on
    c                       s   e Zd Z fddZdd Zdd Ze													dd	ejdB d
ej	dB dejdB dej	dB dejdB dej	dB dej	dB de
dB dedB dedB dedB dejdB dedeej eB fddZdd Z  ZS )BridgeTowerModelc                    s  t     | _ j j jr%tj j| _	tj j| _
n"t fddt jD | _	t fddt jD | _
td j| _t| _t| _jsy jry| jjjD ]}| jjjjj|j_| jjjjj|j_qdtfddt jD | _tfddt jD | _t | _t | _ tj! j j"d| _#tj! j j"d| _$ j%rt& | _'t& | _(n$t fddt jd	 D | _'t fd
dt jd	 D | _(| )  d S )Nc                       g | ]
}t j jqS r3   r   rL   rF   ri   )rR   text_configr3   r4   rm         z-BridgeTowerModel.__init__.<locals>.<listcomp>c                    r  r3   r  ri   )rR   vision_configr3   r4   rm     r  r   c                    rg   r3   r6  ri   r  r3   r4   rm     rn   c                    rg   r3   r  ri   r  r3   r4   rm     rn   r=   c                    rg   r3   r   ri   rl   r3   r4   rm     rn   r   c                    rg   r3   r  ri   rl   r3   r4   rm     rn   )*rC   rD   rR   r  r  $share_cross_modal_transformer_layersr   rL   rF   cross_modal_text_transformcross_modal_image_transformrq   rr   ro   r   r_  r  vision_modelr  
text_modelr   "init_layernorm_from_vision_encoderr  cross_modal_ln_separater   r   datar~   cross_modal_image_layerscross_modal_text_layersr   cross_modal_image_poolercross_modal_text_poolerrH   rI   cross_modal_text_layernormcross_modal_image_layernormshare_link_tower_layersr   cross_modal_text_link_towercross_modal_image_link_towerr  )rQ   rR   r   rS   )rR   r  r  r4   rD     sP   




zBridgeTowerModel.__init__c                 C   s
   | j  S r_   )r  r  r  r3   r3   r4   r    s   
z%BridgeTowerModel.get_input_embeddingsc                 C   s   | j | d S r_   )r  r  r  r3   r3   r4   r    s   z%BridgeTowerModel.set_input_embeddingsNFrd  rV   rZ  r   
pixel_maskre  r9   image_token_type_idxrN  rO  return_dictlabelsr   r   c           -      K   s|  |	dur|	n| j j}	|
dur|
n| j j}
|
rdnd}|
rdnd}|
r$dnd}|
r*dnd}|	r0dnd}|dur>|du r>td|durD|n| j j}|pKd}| }| jj|d}|
r^||f7 }|du rltj	|tj
|jd}| j|||j}t| jjj| j j d }| jjjd| D ]}|||d }|
r||f7 }q|du r| jjj|| jj|d}n|ddd	}|
r||f7 }| jjjjd| D ]}||}|
r||f7 }q| jj|| jj}| |}| tjdtj
|jd|}| || }|  |}| tj!d
|tj
|jd|}|| }| "|}tj	|d|dftj
|jd}| j|| |j}| j#d |||||	d} | d }!| j$d |||||	d}"|"d }#|
rg||!|#ff7 }|	ru|| d |"d ff7 }d}$t%|t| jjjD ]}%| jjj|% ||d }| jjjj|% || jj}|  | jj|| }| j&|$ }&| j'|$ }'|&| || |!|}(|'||#|})| j#|$d  |(|)|||	d} | d }!| j$|$d  |)|(|||	d}"|"d }#|$d7 }$|
r||f7 }||f7 }||!|#ff7 }|	r|| d |"d ff7 }q|!|#}*}+| (|*|+},|
r$|||f}|s5t)dd |*|+|,||fD S t*|*|+|,||dS )a  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        image_token_type_idx (`int`, *optional*):
            - The token type ids for images.
        output_hidden_states (`bool`, *optional*):
            If set to `True`, hidden states are returned as a list containing the hidden states of text, image, and
            cross-modal components respectively. i.e. `(hidden_states_text, hidden_states_image,
            hidden_states_cross_modal)` where each element is a list of the hidden states of the corresponding
            modality. `hidden_states_txt/img` are a list of tensors corresponding to unimodal hidden states and
            `hidden_states_cross_modal` is a list of tuples containing `cross_modal_text_hidden_states` and
            `cross_modal_image_hidden_states` of each brdige layer.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels are currently not supported.

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerModel
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> # prepare image and text
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> text = "hello world"
        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base")
        >>> model = BridgeTowerModel.from_pretrained("BridgeTower/bridgetower-base")

        >>> inputs = processor(image, text, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.keys()
        odict_keys(['text_features', 'image_features', 'pooler_output'])
        ```Nr3   zYBridgeTowerModel does not use `inputs_embeds`.  Make sure to pass in `input_ids` instead.r   )rd  rW   r   r   r   r   )rV   r3  rN  c                 s   s    | ]	}|d ur|V  qd S r_   r3   )rj   vr3   r3   r4   	<genexpr>  s    z+BridgeTowerModel.forward.<locals>.<genexpr>)r&   r'   r(   r)   r*   )+rR   rN  rO  r   use_return_dictr   r  r   r/   onesrb  rY   get_extended_attention_maskr\   lenr  rb   ro   r  r  r   r  rX   r   r   rs   r   r  r_  ra  	expand_asr  r  fullr  r  r  rr   r  r  get_cls_featuresr2   r%   )-rQ   rd  rV   rZ  r   r  re  r9   r  rN  rO  r  r  r   r   all_hidden_states_textall_hidden_states_imageall_hidden_states_crossrS  rT  r  r8   extend_text_maskssplit_indexrb   rw   image_embeds_with_lncross_modal_texttext_token_type_embeddingsimage_token_type_embeddingscross_modal_imageextend_image_maskslayer_outputs_textcross_text_featureslayer_outputs_imagecross_image_featureslink_layer_indexrL  text_link_towerimage_link_towercross_text_features_cross_image_features_r&   r'   cls_featuresr3   r3   r4   rc     s  7












zBridgeTowerModel.forwardc                 C   s&   |  |}| |}tj||gddS )Nr   r   )r  r  r/   r   )rQ   r&   r'   cls_features_textcls_features_imager3   r3   r4   r    s   

z!BridgeTowerModel.get_cls_features)NNNNNNNNNNNNF)r+   r,   r-   rD   r  r  r   r/   rs  r0   r   r]   r2   rd   r%   rc   r  re   r3   r3   rS   r4   r  y  sb    8	
 or  c                       r   )"BridgeTowerPredictionHeadTransformc                    sV   t    t|j|j| _t|jtrt	|j | _
n|j| _
tj|j|jd| _d S r   )rC   rD   r   rL   rF   r   r   r   r   r	   transform_act_fnrH   rI   rP   rS   r3   r4   rD     s   
z+BridgeTowerPredictionHeadTransform.__init__c                 C   s"   |  |}| |}| |}|S r_   )r   r  rH   r   r3   r3   r4   rc     s   


z*BridgeTowerPredictionHeadTransform.forwardr   r3   r3   rS   r4   r    s    	r  c                       s&   e Zd Zd fdd	Zdd Z  ZS )r  Nc                    s`   t    || _t|| _tj|j|jj	dd| _
tt|jj	| _|d ur.|| j
_d S d S )NF)r~   )rC   rD   rR   r  	transformr   rL   rF   r  r[  decoderr   r/   ra  r~   r   )rQ   rR   r   rS   r3   r4   rD     s   

zBridgeTowerMLMHead.__init__c                 C   s   |  |}| || j }|S r_   )r  r  r~   )rQ   x	mlm_scorer3   r3   r4   rc     s   
zBridgeTowerMLMHead.forwardr_   r   r3   r3   rS   r4   r    s    	r  c                       r   )BridgeTowerITMHeadc                    s   t    t|d| _d S Nr   rC   rD   r   rL   fc)rQ   rF   rS   r3   r4   rD        
zBridgeTowerITMHead.__init__c                 C   s   |  |}|S r_   r  )rQ   r  	itm_scorer3   r3   r4   rc        
zBridgeTowerITMHead.forwardr   r3   r3   rS   r4   r        r  z\
    BridgeTower Model with a language modeling head on top as done during pretraining.
    c                       s   e Zd ZddiZ fddZdd Zdd Ze																						dd
ej	d	B dej
d	B dej	d	B dej
d	B dej	d	B dej
d	B dej
d	B ded	B ded	B ded	B dej	d	B deeej
 B fddZ  ZS )BridgeTowerForMaskedLMzmlm_score.decoder.weightz8bridgetower.text_model.embeddings.word_embeddings.weightc                    s,   t  | t|| _t|| _|   d S r_   )rC   rD   r  rv  r  r  r  rP   rS   r3   r4   rD     s   

zBridgeTowerForMaskedLM.__init__c                 C   r  r_   r  r  r  r3   r3   r4   get_output_embeddings  r  z,BridgeTowerForMaskedLM.get_output_embeddingsc                 C   r  r_   r  )rQ   new_embeddingsr3   r3   r4   set_output_embeddings  r  z,BridgeTowerForMaskedLM.set_output_embeddingsNrd  rV   rZ  r   r  re  r9   rN  rO  r  r  r   c                 K   s   |
dur|
n| j j}
| j|||||||||	|
d
}| |
r |jn|d }d}|durCt }||j}||d| j j	j
|d}|
sTt|}|durR|f| S |S t|||j|jdS )a  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForMaskedLM
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000360943.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read())).convert("RGB")
        >>> text = "a <mask> looking out of the window"

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")
        >>> model = BridgeTowerForMaskedLM.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")

        >>> # prepare inputs
        >>> encoding = processor(image, text, return_tensors="pt")

        >>> # forward pass
        >>> outputs = model(**encoding)

        >>> results = processor.decode(outputs.logits.argmax(dim=-1).squeeze(0).tolist())

        >>> print(results)
        .a cat looking out of the window.
        ```N	rV   rZ  r   r  re  r9   rN  rO  r  r   r   r6   r7   r)   r*   )rR   r  rv  r  r&   r   r\   rY   r   r  r[  r2   r   r)   r*   )rQ   rd  rV   rZ  r   r  re  r9   rN  rO  r  r  r   rJ  
mlm_logitsmasked_lm_lossloss_fctr1  r3   r3   r4   rc     s8   4zBridgeTowerForMaskedLM.forwardNNNNNNNNNNN)r+   r,   r-   _tied_weights_keysrD   r   r  r   r/   rs  r0   r]   r   r2   rc   re   r3   r3   rS   r4   r    sT    		
r  z
    BridgeTower Model transformer with a classifier head on top (a linear layer on top of the final hidden state of the
    [CLS] token) for image-to-text matching.
    c                       s   e Zd Z fddZe											ddejdB dejdB dejdB dejdB dejdB d	ejdB d
ejdB dedB dedB dedB dejdB de	e
ej B fddZ  ZS )#BridgeTowerForImageAndTextRetrievalc                    s2   t  | t|| _t|jd | _|   d S r  )rC   rD   r  rv  r  rF   r  r  rP   rS   r3   r4   rD   R  s   
z,BridgeTowerForImageAndTextRetrieval.__init__Nrd  rV   rZ  r   r  re  r9   rN  rO  r  r  r   c                 K   s   |
dur|
n| j j}
| j|||||||||	|
d
}|
r|jn|d }| |}d}|dur;t }||j}|||}|
sLt|}|durJ|f| S |S t	|||j
|jdS )a^  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*):
            Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match.
            The pairs with 0 will be skipped for calculation.

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForImageAndTextRetrieval
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> texts = ["An image of two cats chilling on a couch", "A football player scoring a goal"]

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")
        >>> model = BridgeTowerForImageAndTextRetrieval.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")

        >>> # forward pass
        >>> scores = dict()
        >>> for text in texts:
        ...     # prepare inputs
        ...     encoding = processor(image, text, return_tensors="pt")
        ...     outputs = model(**encoding)
        ...     scores[text] = outputs.logits[0, 1].item()
        ```Nr  r   r  )rR   r  rv  r(   r  r   r\   rY   r2   r   r)   r*   )rQ   rd  rV   rZ  r   r  re  r9   rN  rO  r  r  r   rJ  r(   r7   itm_lossr  r1  r3   r3   r4   rc   \  s:   0

z+BridgeTowerForImageAndTextRetrieval.forwardr  )r+   r,   r-   rD   r   r/   rs  r0   r]   r   r2   rc   re   r3   r3   rS   r4   r
  K  sN    
	
r
  c                       r   )BridgeTowerContrastiveHeadc                    s   t    t||| _d S r_   r  )rQ   rF   
embed_sizerS   r3   r4   rD     r  z#BridgeTowerContrastiveHead.__init__c                 C   s   |  |}|S r_   r  )rQ   r  r3   r3   r4   rc     r  z"BridgeTowerContrastiveHead.forwardr   r3   r3   rS   r4   r    r  r  zl
    BridgeTower Model with a image-text contrastive head on top computing image-text contrastive loss.
    c                       s   e Zd Z fddZe											ddejdB dejdB dejdB dejdB d	ejdB d
ejdB dejdB dedB dedB dedB dedB de	e
ej B fddZ  ZS )r  c                    sl   t  | t|| _t|j|j| _t|j|j| _t|jd |j| _	t
t| jj| _|   d S r  )rC   rD   r  rv  r  rF   contrastive_hidden_sizeitc_text_headitc_image_headitc_cross_modal_headr   r   r/   r   rR   r  r  r  rP   rS   r3   r4   rD     s   
z*BridgeTowerForContrastiveLearning.__init__NTrd  rV   rZ  r   r  re  r9   rN  rO  r  return_lossr   c           !      K   s@  |
dur|
n| j j}
| j||||||||d|
d
}|
r|jn|d }|
r'|jn|d \}}}|d }|d }| jjj|}| jt	j
ddt	j| jjjjd	|}| j|| }tjj| |ddd
ddf ddd}tjj| |ddd
ddf dddj|jd}tjj| |dddj|jd}t	j|||gdd}| j j|jd}t	|| | }t	|| | }t	|| | }d}|rt	jt||jd}tj||}tj||}tj||}|| | d }|
s||||f|dd  } |dur|f|  S | S t||||||j|j dS )a  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image
        >>> import torch

        >>> image_urls = [
        ...     "https://farm4.staticflickr.com/3395/3428278415_81c3e27f15_z.jpg",
        ...     "http://images.cocodataset.org/val2017/000000039769.jpg",
        ... ]
        >>> texts = ["two dogs in a car", "two cats sleeping on a couch"]

        >>> with httpx.stream("GET", urls[0]) as response:
        ...     image1 = Image.open(BytesIO(response.read()))

        >>> with httpx.stream("GET", urls[1]) as response:
        ...     image2 = Image.open(BytesIO(response.read()))

        >>> images = [image1, image2]

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
        >>> model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")

        >>> inputs = processor(images, texts, padding=True, return_tensors="pt")
        >>> loss = model(**inputs, return_loss=True).loss

        >>> inputs = processor(images, texts[::-1], padding=True, return_tensors="pt")
        >>> loss_swapped = model(**inputs, return_loss=True).loss

        >>> print("Loss", round(loss.item(), 4))
        Loss 0.0019

        >>> print("Loss with swapped images", round(loss_swapped.item(), 4))
        Loss with swapped images 2.126
        ```NTr  r   r   r   r  r   rW   r   )r   r   r  r   g      @)r6   r7   r8   r9   r:   r)   r*   )!rR   r  rv  r(   r)   r  r  r   r_  r/   r  rb  r   rY   r  r  r   r   	normalizer  r  r\   r  r   r  expr   tr   r  cross_entropyr5   r*   )!rQ   rd  rV   rZ  r   r  re  r9   rN  rO  r  r  r   rJ  r(   hidden_states_txthidden_states_imghidden_states_cross_modalr8   r  r  r:   r7   r  logits_text_to_imagelogits_text_to_crosslogits_image_to_crossitc_lossr  text_to_image_losstext_to_cross_lossimage_to_cross_lossr1  r3   r3   r4   rc     sp   =**z)BridgeTowerForContrastiveLearning.forward)NNNNNNNNTNN)r+   r,   r-   rD   r   r/   rs  r0   r]   r5   r2   rc   re   r3   r3   rS   r4   r    sN    	
r  )r  r
  r  r  ru  )Nr   )Xr.   collectionsr   collections.abcr   dataclassesr   r/   r   torch.nnr    r   r|  activationsr	   r
   cache_utilsr   r   r   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r   r   utils.genericr   configuration_bridgetowerr    r!   r"   
get_loggerr+   r  _TOKENIZER_FOR_DOCr%   r5   r  r;   rf   ry   r   r   r   r   r   r   rd   floatr  r  r"  r.  r6  rI  rK  rX  ru  r  r  r  r  r  r  r  r
  r  r  __all__r3   r3   r3   r4   <module>   s   
,S:
JN;J7j) "  7
ha
 