o
    Ti/                     @   s   d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	m
Z
 dZded	ed
efddZded	ed
efddZded	edee deded
efddZdS )    )ListN)GraphModule   )get_deepcompile_handle)add_postprocessmove_primals_to_head_make_node_metazero1_compilegmgraph_idreturnc                 C   s*   t  }|| j}||dd |D  | S )Nc                 S   s   g | ]}|d  qS )    ).0vr   r   Z/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/compile/passes/zero1_compile.py
<listcomp>   s    z$add_z1_reduce_fw.<locals>.<listcomp>)r   param_indicesregister_graph_z1)r
   r   profiling_resultsparam_managerdcr   r   r   r   add_z1_reduce_fw   s   
r   c                 C   s   | j }|| }||\}}|jD ]4}|| }||jv s$J d| d|j| }	t||tjjjj	||	gd| t
||dd}
d |
jd< qt|| _ | S )Nzparam_name=z not in ds_idsreduce_param_T)
extra_argsnamemetaval)graphget_bwd_mappingparam_namesds_idsr   torchopsr   reduce_graddefaultr   r   r   )r
   r   r   r   pm_param_name_to_grad
param_name	grad_nodeds_idnew_noder   r   r   add_z1_reduce_bw   s"   




r-   graph_order
mem_budgetbwdc                 C   s   |rt | ||S t| |||S )N)r-   r   )r
   r   r.   r   create_inputs_fnr/   r   r0   r   r   r   add_z1_reduce3   s   r2   )typingr   r"   torch.fxr   utilr   fxr   r   r   NAMEintr   r-   floatboolr2   r   r   r   r   <module>   s   	