o
    }oiq                     @   s   d dl Z d dlm  mZ dddZdddZG dd de jjZ	ddd	Z
G d
d de jjZdddZG dd de jjZdS )    Nc           	      C   s`   t |}t jjt jj|||d}t jjt jj|| |d}t j||g}|D ]}|  q'|S Ngrouptorch
zeros_likedistributedP2POpisendirecvbatch_isend_irecvwait)		from_rankto_ranktensorr   tensor_recvsend_oprecv_opreqsreq r   b/home/ubuntu/.local/lib/python3.10/site-packages/nemo/collections/multimodal/losses/siglip_loss.pyneighbour_exchange   s"   

r   c                 C   s   t |}t |}t jjt jj|| |d}t jjt jj|||d}t jjt jj|| |d}	t jjt jj|||d}
t j|||
|	g}|D ]}|  qF||fS r   r   )	left_rank
right_ranktensor_to_lefttensor_to_rightr   tensor_from_lefttensor_from_rightsend_op_leftsend_op_rightrecv_op_leftrecv_op_rightr   r   r   r   r   neighbour_exchange_bidir*   s<   


r#   c                   @   $   e Zd Zedd Zedd ZdS )NeighbourExchangec                 C   s"   || _ || _|| _t||||dS r   )r   r   r   r   )ctxr   r   r   r   r   r   r   forwardL   s   zNeighbourExchange.forwardc                 C   s   dt | j| j| j|f S N)NNN)r%   applyr   r   r   )r&   grad_outputr   r   r   backwardS   s   zNeighbourExchange.backwardN__name__
__module____qualname__staticmethodr'   r+   r   r   r   r   r%   K   
    
r%   c                 C   s   t | |||S N)r%   r)   )r   r   r   r   r   r   r   neighbour_exchange_with_gradX   s   r3   c                   @   r$   )NeighbourExchangeBidirc                 C   s$   || _ || _|| _t|||||dS r   )r   r   r   r#   )r&   r   r   r   r   r   r   r   r   r'   ]   s   zNeighbourExchangeBidir.forwardc                 G   s    dt j| j| j| jg|R   S r(   )r4   r)   r   r   r   )r&   grad_outputsr   r   r   r+   d   s
   
zNeighbourExchangeBidir.backwardNr,   r   r   r   r   r4   \   r1   r4   c                 C   s   t | ||||S r2   )r4   r)   )r   r   r   r   r   r   r   r   "neighbour_exchange_bidir_with_gradk   s   r6   c                       sZ   e Zd ZdZ					d fdd	Zdd	ejfd
dZdddZdddZ	dd Z
  ZS )
SigLipLossa]  Sigmoid Loss for Language Image Pre-Training (SigLIP) - https://arxiv.org/abs/2303.15343

    @article{zhai2023sigmoid,
      title={Sigmoid loss for language image pre-training},
      author={Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},
      journal={arXiv preprint arXiv:2303.15343},
      year={2023}
    }
    Fr      NTc                    s,   t    || _|| _|| _|| _|| _d S r2   )super__init__cache_labelsrank
world_sizer   bidir)selfr;   r<   r=   r   r>   	__class__r   r   r:   z   s   

zSigLipLoss.__init__returnc                 C   s6   t j||f||d }|sdt j|||d | }|S )N)devicedtype   )r   oneseye)r?   rC   rD   
num_logitsnegative_onlylabelsr   r   r   get_ground_truth   s   zSigLipLoss.get_ground_truthc                 C   s"   || |j  }|d ur||7 }|S r2   )T)r?   image_featurestext_featureslogit_scale
logit_biaslogitsr   r   r   
get_logits   s   zSigLipLoss.get_logitsc           	      C   sN   |  ||||}| j|j|j|jd |d}t||   |jd  }|S )Nr   rI   )rR   rK   rC   rD   shapeF
logsigmoidsum)	r?   rM   rN   rO   rP   rI   rQ   rJ   lossr   r   r   _loss   s   zSigLipLoss._lossc              
   C   s<  |\}}}}|  ||||}| jdkr| jd | j }| jd | j | j }| jru| }	}
t| jd d\}}t|D ]!}t|||
|	| jd}|D ]}|| j ||||dd7 }qI|\}
}	q;|rtt|||	| jd}|| j ||||dd7 }n#|}	t| jd D ]}t|||	| jd}|| j ||||dd7 }|}	q~|d|ifS )Nr8   rE   r   TrS   rX   )	rY   r=   r<   r>   divmodranger6   r   r3   )r?   output_tensorrM   rN   rO   rP   rX   r   r   text_features_to_righttext_features_to_left	num_bidir	remainderitext_features_recvftext_features_from_leftr   r   r   r'      sf   




zSigLipLoss.forward)Fr   r8   NT)Fr2   )NF)r-   r.   r/   __doc__r:   r   TensorrK   rR   rY   r'   __classcell__r   r   r@   r   r7   o   s    

r7   r2   )r   torch.nn.functionalnn
functionalrU   r   r#   autogradFunctionr%   r3   r4   r6   Moduler7   r   r   r   r   <module>   s   

!

