o
    Ni                     @  sn   d Z ddlmZ ddlmZ ddlZddlmZ ddlm	Z	 g dZ
dd ZdddZdddZdddZdS )z=Utilities for Orthogonal Subspace Learning with Adaptive OSF.    )annotations)AnyN)nn)decompose_weight_matrix$project_gradient_to_orthogonal_spacereconstruct_weight_matrixc                 C  s   t | dr	|  S | S )zHWait for AsyncCollectiveTensor if needed, otherwise return tensor as-is.wait)hasattrr   )tensor r   I/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/osf/utils.py_wait_if_async#   s   
r   weighttorch.Tensortop_kintreturndict[str, Any]c           
      C  s$  | j }| j}| tj}tjj|dd\}}}t||jd }|ddd|f 	 
 j||d|d| 	 
 j||d|d|ddf 	 
 j||dt|dd|df 	 
 j||dt||d 	 
 j||dt||dddf 	 
 j||d|d}	|	S )zJPerform an SVD of ``weight`` and split it into frozen and trainable parts.F)full_matricesr   N)devicedtype)U_highS_highV_highU_lowS_lowV_low	rank_high)r   r   totorchfloat32linalgsvdminshape
contiguousdetachr   	Parameter)
r   r   device_local
orig_dtypeWUSVtkr"   r   r   r   r   *   s   $$*"*	r   svd_dictdict[str, torch.Tensor]c           	      C  s   | d }| d }| d }| d }| d }| d }|  dkr/|  dkr/t||d |ntj|d|d|jd	}|  dkrU|  dkrUt||d |ntj|d|d|jd	}|| S )
z4Reconstruct a weight matrix from its SVD components.r   r   r   r   r   r   r      )r   )numelr   mm	unsqueezezerossizer   )	r/   r   r   r   r   r   r   	high_partlow_partr   r   r   r   >   s   r   Nonec                   s  | d j du r| d j du r| d j du rdS | d  | d | d j dur{| d j t d fdd	 }tdfd
d	 }t|dd|}t rbt rbt dkrbtj	|tj
jd |j||dd tdrvj| n| | d j dur| d j tdfdd	 }tdfdd	 }t|dd|}t rt rt dkrtj	|tj
jd t||}|j|dd tdr؈j| dS | dS dS )zUProject gradients of ``U_low`` and ``V_low`` to be orthogonal to the high rank space.r   Nr   r   r   r   to_localc                         S Nr   r   )r   r   r   <lambda>`       z6project_gradient_to_orthogonal_space.<locals>.<lambda>c                     r;   r<   r   r   )dUr   r   r=   a   r>   r   r1   )opg      )alpha_local_tensorc                     r;   r<   r   r   )r   r   r   r=   u   r>   c                     r;   r<   r   r   )dVr   r   r=   v   r>   )gradgetattrr   r3   	transposedistis_availableis_initializedget_world_size
all_reduceReduceOpSUMaddmm_r	   rB   copy_add_)r/   local_U_highlocal_dU
proj_coefflocal_V_highlocal_dVG_localupdater   )r   r   r?   rC   r   r   T   s8   *




r   )r   r   r   r   r   r   )r/   r0   r   r   )r/   r   r   r9   )__doc__
__future__r   typingr   r   torch.distributeddistributedrG   r   __all__r   r   r   r   r   r   r   r   <module>   s   

