o
    Ti                     @   sD   d dl Z d dlmZ d dlZd dlZd dlmZ G dd deZ	dS )    N)log_dist)required_torch_versionc                       s^   e Zd Z							d fdd	Zd	d
 Zdd Zdd Zdd ZdddZdd Z	  Z
S )
EigenvalueFd   {Gz?r       c                    s   t    || _|| _|| _|| _|| _|| _|| _t	| jdkr%|dks'J t
d| d| d| d| d| d| d| dgd	 d S )
Nr   z enabled eigenvalue with verbose=z, max_iter=z, tol=z, stability=z, gas_boundary_resolution=z, layer_name=z, layer_num=ranks)super__init__verbosemax_itertol	stabilitygas_boundary_resolution
layer_name	layer_numlenr   )selfr   r   r   r   r   r   r   	__class__ P/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/runtime/eigenvalue.pyr      s   
,
zEigenvalue.__init__c                 C   sR   t ddrtj|ddddS |j}|  }tj|ddddd}t||S )Ng?)min_version        )nanposinfneginfF)xcopyr   r   r   )	r   torch
nan_to_numdevicecpunumpynp
from_numpyto)r   r   r#   r   r   r   r"   (   s   
zEigenvalue.nan_to_numc                    sB    ||}|d j   fdd|D }fdd|D }|S )Ng      ?c                       g | ]}|  qS r   r   .0vector)normr   r   
<listcomp>5       z(Eigenvalue.normalize.<locals>.<listcomp>c                    s   g | ]}  |qS r   )r"   r*   r   r   r   r.   6   s    )inner_productr   )r   vnorm_squarednormalized_vectorsr   )r-   r   r   	normalize2   s
   zEigenvalue.normalizec                 C   s   t dd t||D S )Nc                 S   s   g | ]\}}t || qS r   )r!   sum)r+   r   yr   r   r   r.   :   s    z,Eigenvalue.inner_product.<locals>.<listcomp>)r6   zip)r   xsysr   r   r   r1   9   s   zEigenvalue.inner_productc                 C   sJ   | j d}t|dksJ |}|D ]}t||sJ dt||}q|S )N.r   z$layer_name configuration is invalid.)r   splitr   hasattrgetattr)r   modulescope_namesmnamer   r   r   
get_layers<   s   zEigenvalue.get_layersN      ?c                    sL  g }g } |}tjD ]}|| }tj }	 d u r'dd | D }
n fdd| D }
tj|	 dd | D }dd | D }dd | D }|| 	|
}
t
|dksit
|dkrvtddgtjd	 g   S d}d
\}}|jk rt|dkrt|| | jkr|}tjj|||
ddd}fdd|D }||
 }	|}
fdd|
D }
|d7 }|jk rt|dkrt|| | jks|9 }|| jrtd| d| d| dgd q|}jrtd| dgd i }tt||D ]\}\}}|t|||f q|S )Nc                 S   s0   g | ]}|j d ur|j jd urt| qS Ngradgrad_fnr!   randnsizer+   pr   r   r   r.   R   s
    z1Eigenvalue.compute_eigenvalue.<locals>.<listcomp>c                    s4   g | ]}|j d ur|j jd urtj|  dqS )Nr#   rF   rK   rM   r   r   r.   W   s
    c                 S   s(   g | ]}|j d ur|j jd ur|j qS rE   rG   rH   r+   paramr   r   r   r.   ]   s
    c                 S   s&   g | ]}|j d ur|j jd ur|qS rE   rN   rO   r   r   r   r.   a   s
    c                 S   s   g | ]}t |qS r   )idrK   r   r   r   r.   f   r/   r   z2The model does NOT support eigenvalue computation.)r
   level)rD   r   T)grad_outputsonly_inputsretain_graphc                    s   g | ]	}  | qS r   )r"   float)r+   hvr0   r   r   r.   {   s    c                    r)   r   r   )r+   r   )scaler   r   r.      r/   r   zblock: z, power iteration: z, eigenvalue: r	   z!post processed block_eigenvalue: )rC   ranger   r!   randomget_rng_state
parametersset_rng_stateappendr5   r   r   loggingWARNINGr   absr   autogradrG   r1   itemr   post_process	enumerater8   updatedictfromkeys)r   r?   r#   rX   block_eigenvalue
param_keyslayersblockmodel_block	rng_stater2   gradsparams
layer_keysieigenvalue_currenteigenvalue_previousHvev_dictvaluer   )r#   rX   r   r   compute_eigenvalueG   sr   








 
zEigenvalue.compute_eigenvaluec                    s"   t t|t d  fdd|D S )N)keyc                    s$   g | ]}|d krt |  ndqS )r   rD   )ra   )r+   r2   	max_valuer   r   r.      s   $ z+Eigenvalue.post_process.<locals>.<listcomp>)ra   max)r   
value_listr   rz   r   rd      s   zEigenvalue.post_process)Fr   r   r   r   r   r   )NrD   )__name__
__module____qualname__r   r"   r5   r1   rC   rx   rd   __classcell__r   r   r   r   r      s    

Pr   )
r!   deepspeed.utilsr   r%   r&   r_   deepspeed.utils.torchr   objectr   r   r   r   r   <module>   s   