o
    Ti                     @   s   d dl mZ d dlZd dlmZmZ d dlmZ ddlm	Z	 de
ded	ed
e
fddZde
ded	ed
e
fddZd	efddZdede
dee
 dede	dedefddZdS )    )ListN)NodeGraphModule)get_last_uses   )DSGraphParamManagergraph_idgmnodeds_idc                 C   r   d }|j |' |f}| |fD ]}||f7 }q|j jdtjjjj|i dd}W d    |S 1 s2w   Y  |S )Ncall_functionoffload_parametername)graphinserting_aftercreate_nodetorchopsdcr   defaultr   r	   r
   r   new_nodeargsa r   _/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/compile/passes/offload_parameters.pyadd_offload_parameter   s   

		r   c                 C   r   )Nr   reload_parameterr   )r   r   r   r   r   r   r   r   r   r   r   r   add_reload_parameter   s   

r    c                 C   s    | j tjjjjksJ | jd S )Nr   )targetr   r   r   allgather_paramr   r   )r
   r   r   r   	get_ds_id)   s   
r#   graph_order
mem_budgetparam_managerbwdreturnc                 C   st   t | j\}}	| jjD ]'}
t|
tr2|
jtjjj	j
kr2t|| |
jd t|
 t|| ||
 t|
 q| j  | S )Nr   )r   r   nodes
isinstancer   r!   r   r   r   r"   r   r    r   r#   r   lint)r	   r   r$   profiling_resultscreate_inputs_fnr%   r&   r'   node_to_last_useuser_to_last_usesr
   r   r   r   offload_parameter_fwd.   s   
r0   )typingr   r   torch.fxr   r   deepspeed.compile.utilr   graph_paramr   intr   r    r#   floatboolr0   r   r   r   r   <module>   s"   