o
    Ti                     @   s   d dl mZmZ d dlmZmZmZmZ d dlm	Z	 d dl
Z
d dlmZmZ ddlmZ ddlmZ eG d	d
 d
ZG dd dZdS )    )	dataclassfield)AnyDictListTuple)reduceN)GraphNode   )get_output_node)get_param_nodesc                   @   sp   e Zd ZU eed< ejed< ejed< ejed< e	ed< e	ed< e	ed< ej
ed< ed	d
Zeed< dd ZdS )DSGraphParamnameshapedtypedevicenodeallgather_noderelease_nodeparamF)initnumelc                 C   s   t dd | j| _d S )Nc                 S   s   | | S N )xyr   r   Q/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/compile/graph_param.py<lambda>   s    z,DSGraphParam.__post_init__.<locals>.<lambda>)r   r   r   selfr   r   r   __post_init__   s   zDSGraphParam.__post_init__N)__name__
__module____qualname__str__annotations__torchSizer   r   r
   Tensorr   r   intr!   r   r   r   r   r      s   
 



r   c                
   @   s   e Zd Zdededeeeeef  fddZdefddZ	e
d	ee fd
dZe
d	eeef fddZe
d	eeef fddZd	efddZdS )DSGraphParamManagerfw_graphsample_inputsindex_to_ds_idsc                    s   || _ d | _i | _i | _i | _t||}dd |D | _dd |D | _ fdd|D }dd |D }dd |D }t||||D ]\}}	}
}t	|j
||	j|	j|d d |	d| j|j
< |
| j|j
< qBd S )Nc                 S   s   g | ]}|j qS r   )r   ).0pnr   r   r   
<listcomp>+   s    z0DSGraphParamManager.__init__.<locals>.<listcomp>c                 S   s   g | ]\}}}|qS r   r   r/   i_r   r   r   r1   ,       c                    s   g | ]	\}}} | qS r   r   r2   r-   r   r   r1   .       c                 S   s   g | ]\}}}|qS r   r   )r/   r4   ds_idr   r   r   r1   /   r5   c                 S   s   g | ]\}}}|qS r   r   )r/   r4   ds_shaper   r   r   r1   0   r5   )r   r   r   r   r   r   r   r   )	_fw_graph	_bw_graph_params_param_name_to_grad_ds_idsr   _param_names_param_indiceszipr   r   r   r   )r    r,   r-   r.   param_nodesparam_inputsds_ids	ds_shapesr0   pir8   r9   r   r6   r   __init__#   s.   
zDSGraphParamManager.__init__bw_graphc                    sV   |_ t| fddj jD } fddjD }dd tj|D }||fS )Nc                    s   g | ]
}|j  jv r|qS r   )r   param_names)r/   nr   r   r   r1   A   s    z7DSGraphParamManager.get_bwd_mapping.<locals>.<listcomp>c                    s   g | ]	} j d  | qS )r   )args)r/   r3   )output_noder   r   r1   B   r7   c                 S   s   i | ]\}}||qS r   r   )r/   
param_namegradr   r   r   
<dictcomp>C   r5   z7DSGraphParamManager.get_bwd_mapping.<locals>.<dictcomp>)r;   r   nodesr@   rA   rI   )r    rH   param_nodes_bwgrad_outputsparam_name_to_gradr   )rL   r    r   get_bwd_mapping=   s   z#DSGraphParamManager.get_bwd_mappingreturnc                 C      | j S r   )r?   r   r   r   r   rI   F      zDSGraphParamManager.param_namesc                 C   rV   r   )r<   r   r   r   r   paramsJ   rW   zDSGraphParamManager.paramsc                 C   rV   r   )r>   r   r   r   r   rD   N   rW   zDSGraphParamManager.ds_idsc                 C   s   | j d us	J d| j | S )NzBackward graph is not added yet)r=   )r    rM   r   r   r   get_grad_nameR   s   
z!DSGraphParamManager.get_grad_nameN)r"   r#   r$   r	   r   r   r   r*   rG   rT   propertyr%   rI   r   r   rX   rD   rY   r   r   r   r   r+   !   s    $	r+   )dataclassesr   r   typingr   r   r   r   	functoolsr   r'   torch.fxr	   r
   fxr   utilr   r   r+   r   r   r   r   <module>   s   