o
    Ti                     @   sH   d dl Z d dlmZ dddZdd Zdd	 Zd
d ZG dd dZdS )    N)log_dist   c                 C   s   dd l }|| jjS Nr   )sys	_getframef_codeco_name)framer    r
   Q/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/utils/comms_logging.pyget_caller_func
   s   r   c                 C   s(   dd l m} | dkrt|  d S d S r   )deepspeed.commcommget_rankprint)messagedistr
   r
   r   print_rank_0   s   r   c                 C   sP   | dkrdS d}t tt| d}td|}t| | d}d||| f S )Nr   0B)	BKBMBGBTBPBEBZBYBi      z%s %s)intmathfloorlogpowround)
size_bytes	size_nameipsr
   r
   r   convert_size   s   r*   c                 C   sT  dd l m} | }d}d}| dkr!|| }|| |d |  }nu| dks1| dks1| dks1| dkrD||9 }|| }|| |d |  }nR| dksP| d	ksP| d
krc|d | }|| d|d  |  }n3| dks| dks| dks| dks| dks| dks| dks| dks| dkr|| }|}ntd td |d9 }|d9 }|d }|d }||fS )Nr   all_to_all_single   
all_gatherall_gather_into_tensorreduce_scatterreduce_scatter_tensor
all_reduceall_reduce_coalescedinference_all_reducer   sendrecvisendirecv	broadcastreducegatherscatterbarrierzwrong comm_op specified   g    .A)r   r   get_world_sizer   exit)comm_opsizedurationr   ntputbusbwr
   r
   r   calc_bw_log"   s0    HrF   c                   @   sN   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dddZ
dS )CommsLoggerc                 C   sD   ddl m}m}m}m}m} i | _|| _|| _|| _	|| _
|| _d S )Nr   )COMMS_LOGGER_VERBOSE_DEFAULTCOMMS_LOGGER_DEBUG_DEFAULTCOMMS_LOGGER_PROF_OPS_DEFAULTCOMMS_LOGGER_PROF_ALL_DEFAULTCOMMS_LOGGER_ENABLED_DEFAULT)deepspeed.comm.constantsrH   rI   rJ   rK   rL   
comms_dictverbosedebugprof_opsprof_allenabled)selfrH   rI   rJ   rK   rL   r
   r
   r   __init__E   s   
zCommsLogger.__init__c                 C   s>   |j | _| jr|jj| _|jj| _|jj| _|jj| _d S d S N)comms_logger_enabledrS   comms_loggerrO   rP   rQ   rR   )rT   comms_configr
   r
   r   	configureN   s   


zCommsLogger.configurec                 C   
   d| _ d S NTrR   rT   r
   r
   r   start_profiling_commsZ      
z!CommsLogger.start_profiling_commsc                 C   r[   r\   r]   r^   r
   r
   r   stop_profiling_comms]   r`   z CommsLogger.stop_profiling_commsc                 C   s   t t| jt|B | _d S rV   )listsetrQ   rT   op_name_listr
   r
   r   start_profiling_opa   s   zCommsLogger.start_profiling_opc                    s    fdd| j D | _ d S )Nc                    s   g | ]}| vr|qS r
   r
   ).0opre   r
   r   
<listcomp>e   s    z1CommsLogger.stop_profiling_op.<locals>.<listcomp>)rQ   rd   r
   ri   r   stop_profiling_opd   s   zCommsLogger.stop_profiling_opc                 C   s  t |||\}}|| j v rY|| j|  v rJ| j| | d  d7  < | j| | d | | j| | d | | j| | d | nd|g|g|gg| j| |< n|d|g|g|ggi| j|< | jrd| d|ddt| d	|dd
|d
}t|dg d S d S )Nr   r,   r   r   z	comm op: z | time (ms): z.2fz | msg size: z | algbw (Gbps): z | busbw (Gbps): )rF   rN   keysappendrO   r*   r   )rT   raw_namerecord_namelatencymsg_sizealgbwrE   log_strr
   r
   r   rm   h   s   ,zCommsLogger.appendTFc                 C   sR  dd l }ddlm} dd lm} ddlm} |r1tddddddddd	dd
ddd | j	 D ]R}|r>t| t
| j|  D ]@\}}	|	d }
t|	d }||	d d}||	d d}||	d d}|rtddt|d|
d|d|d|d|d qGq6|r%|rtd td td tdddddddddddddd | j	 D ]n}|rt| t
| j|  D ]Z\}}	|	d }
||	d }||	d }|j||jd |  }||   }|| d}|||  d}|r#tddt|d|
d|d|d|d|d qqd S d S )Nr   )	trim_mean)ReduceOpzComm. Opz <20zMessage SizeCountzTotal Latency(ms)zAvg Latency(ms)ztput_avg (Gbps)zbusbw_avg (Gbps)r,   g?r   r    z <20.2f_______________________________zBreakdown with straggler effectz-------------------------------zTotal comm lat(ms)zTotal straggler(ms)zAvg comm lat(ms)zAvg straggler(ms))rh   )torchdeepspeed.utils.timerrt   r   r   deepspeed.comm.reduce_opru   r   rN   rl   sorteditemssumr*   tensorr1   MINitemtolist)rT   	print_logshow_stragglerry   rt   r   ru   ro   rq   valscount	total_latavg_lat	avg_algbw	avg_busbwlatsmin_latstotal_straggleravg_stragglerr
   r
   r   log_all~   sf   ,0,0zCommsLogger.log_allN)TF)__name__
__module____qualname__rU   rZ   r_   ra   rf   rk   rm   r   r
   r
   r
   r   rG   C   s    	rG   )r   )r    deepspeed.utilsr   r   r   r*   rF   rG   r
   r
   r
   r   <module>   s   
!