o Ti @sUddlmZmZmZmZddlZddlmZddlZddl m Z m Z ddl m Z mZddlmZeeaeeeeeffed<eaeeed <d efd d Zd e d efddZde dedeeee fdeededed e fddZde dedeededed e f ddZdS))ListDictSetTupleN) defaultdict)GraphNode)get_output_nodemove_primals_to_head)DSGraphParamManager value_to_idused_idsreturncCs2dd}|}|tvr|}|tvs t||S)NcSs tddS)Ni'l)randomrandintrr_/home/ubuntu/.local/lib/python3.10/site-packages/deepspeed/compile/passes/offload_activation.py_gens zget_random_id.._gen)radd)rvrrr get_random_ids rnodecCs t|dsdSd|jvrdSdS)NmetaF tensor_metaT)hasattrr)rrrr_should_offload"s  rgraphgraph_idnodes_to_offload_with_names graph_order mem_budget param_managerc Cs<t|j}ddl}||}|t||D]|\} } | j|vr!qt| s&qt} | |  |j dt j j jj| || fid| jd| d} Wdn1sQwY| |  |j dt j j jj| || fid| jd| d} Wdn1s~wYt|}|| | | t|| <qt|}||S)Nr call_functionoffload__name wait_copy_)set param_namescopydeepcopy erase_noder r'rrinserting_after create_nodetorchopsdcoffload_tensordefault wait_offloadreplace_input_withr r lint)rrrr r!r"r*r+cl_graphr'rval_id offload_node wait_node output_noderrroffload_activation_fwd+s8       r=c st|}dd|jDfdd|D}i}|D]}|jD] } || jvr-| ||<nq q|D]} || j} |||  |jdtjj j j | || fid| jd| d} Wdn1sbwY| |  |jdtjj j j | || fid | jd| d} Wdn1swYi} | jD] }|| kr| | f| |<q| D] \}\}}|||qq1t|}||S) NcSsi|]}|j|qSrr&.0nrrr Tz)reload_activation_bwd..csg|]}|qSrrr> name_to_noderr UrBz)reload_activation_bwd..r#reload_r%r&r()r nodeskeysargsr'inserting_beforer/r0r1r2 reload_tensorr4r. wait_reloadusersitemsr6r r7)rrr r!r"graph_value_to_id act_nodesnode_to_first_useractrr9 reload_noder;rMuold_innew_inrrCrreload_activation_bwdPsH     rW)typingrrrrr collectionsrr0torch.fxrrfxr r graph_paramr dictr intstr__annotations__r)rrboolrfloatr=rWrrrrs2     %