o ߗiB@sddlZddlZddlZddlmZmZmZmZmZm Z m Z ddl Z ddl m Z mZmZddlmZmZddlmZmZmZddlmZmZddlmZddlmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,dd l-m.Z.m/Z/ddl0m1Z1ddl2m3Z3e rddl4m5Z5ee6dZ7ee6dZ8ddZ9ddZ:ddZ;gdZj?j@jAgZBGdddZCdaDdaEdaFejGd$ddZHejGdd ZId%d"d#ZJdS)&N)AnyDictListOptionalTuple TYPE_CHECKINGUnion) call_backward call_hookFakeCompiledAutogradEngine GetItemSource LocalSource)counterslazy_format_graph_codeset_locals_to_steal)getArtifactLoggertrace_structuredclone_preserve_strides)FakeTensorMode) GraphModule) BackwardState) decomposedisable_autocast_cachedisable_proxy_modes_tracingfetch_object_proxyProxyTorchDispatchModePythonKeyTracertrack_tensor_tree) DimDynamicShapeEnv)preserve_node_metaset_stack_trace) OrderedSet)CapturedTraceback)Proxycompiled_autogradcompiled_autograd_verbosecCstjjjdS)Nr()torch_logging _internal log_stateis_artifact_enabledr.r.]/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/torch/_dynamo/compiled_autograd.py snapshot_verbose_logging_enabled+s r0cCs tjjjjSN)r) _inductorconfigtriton cudagraphsr.r.r.r/snapshot_cudagraph_enabled1s r6cCs|durt|S|Sr1r)xr.r.r/ maybe_clone5sr8)inputssizesscalarshooksc @s|eZdZdCddZddZedefddZd ee j d ee d ee e e fd eeee effd dZde fddZddZde fddZddZddZddZdee fddZdd Zd!d"Zd#d$Zd%d&Zed'd(Zed)d*Zd+d,Zd-d.Zd/d0Z d1d2Z!d3d4Z"d5d6Z#d7d8Z$ dDd e%eee effd9d:Z&d;e fded?e d@e%e j(j)fdAdBZ*dS)EAutogradCompilerInstancereturnNcCsT||_t|_|jj|_t|_tdd|jd|_t |_ t |j d|_ d|_ dS)NT)allow_fallback_kernelsallow_non_fake_inputs shape_envsymbolic) compiler_fn contextlib ExitStackstackcloser!rArfake_tensor_moder fx_tracerr proxy_mode hooks_proxy)selfrCr.r.r/__init__Gs   z!AutogradCompilerInstance.__init__cCs t|tjsJ|jj||dS)N)source) isinstancer)TensorrH from_tensor)rLr7rNr.r.r/ wrap_fakeUsz"AutogradCompilerInstance.wrap_fakecCstt||Sr1r )nameidxr.r.r/rNYszAutogradCompilerInstance.sourcer9r:r;originscstddd7<d_i_tjj_tjj t dj_ ij_ fddt D\}}}_jt|\}} } fddt|D}|||fd dt|D}||| t|D]:\} } d | } t| trj| | tj|| <qft| trjjjj| | tjd | | d || <qftd t| ||| jt ijj!jj"jt#j!jdusJj!j}jtjj$j%&||||fS)Nr'captures) tracer_clsc3s"|] }jd|diVqdS) placeholderr.N)rI create_proxy).0rSrLr.r/ js  z9AutogradCompilerInstance.begin_capture..c s$g|]\}}|d|qS)r9)rRrN)r[rTr7r\r.r/ rsz:AutogradCompilerInstance.begin_capture..c s*g|]\}}j|d|tjqS)r:)rA$create_unspecified_symint_and_symbolrNr DYNAMIC)r[rTvalr\r.r/r^ys r;)rN dynamic_dim)hintrNzUnexpected scalar type: )'raot_graph_cls_nameaot_graph_infosr)nnModulerIrootfxGraphrgraph tensor_attrs_graph_placeholdersrKrF enter_contextr" enumeratebind_tensors_to_proxiesrNrOintrAr_r r`floatcreate_symfloatnodecreate_unspecified_symbolAssertionErrortyperrHrJr experimentalsymbolic_shapes_suppress_guards)rLr9r:r;rU args_proxy sizes_proxy scalars_proxyinputs_origins sizes_originsscalars_originsrTrarNenvr.r\r/ begin_capture]sb          z&AutogradCompilerInstance.begin_capture backward_idxc Cs|jdusJ|j|}|jjdt|||g||Rid}t4g}|D]}|dur5|dq)|\} } } } |tj| | | | dq)| ||Wdt |S1s[wYt |S)N call_function)kindtargetargskwargs)sizedtypelayoutdevice) rKrIrZr to_proxyrappendr)emptyrptuple) rLr9output_metadatas saved_tensorsrbackward_c_functionproxiesgrad_insoutput_metadatarrrrr.r.r/proxy_call_backwards6       z,AutogradCompilerInstance.proxy_call_backwardcs(jdt|gfdd|DR|S)Nrcg|]}|qSr.rr[r7r\r.r/r^z.)rIrZr )rLhookrrr.r\r/proxy_call_hooksz(AutogradCompilerInstance.proxy_call_hookicCs||jdusJ|j|}|j|||dd}tt||||<|||g|gWd|S1s7wY|S)Ntensor_pre_hook hook_type)rKrrr8rp)rLr9hook_idrrproxyr.r.r/rs  z(AutogradCompilerInstance.tensor_pre_hookcCsn|jdusJ|j|}|j||dd}tdd|D}|||Wd|S1s0wY|S)Npre_hookrcSg|]}t|qSr.r8rr.r.r/r^z5AutogradCompilerInstance.pre_hook..rKrrrp)rLr9rrrr.r.r/rs  z!AutogradCompilerInstance.pre_hookcCsp|jdusJ|j|}|j|||dd}tdd|D}|||Wd|S1s1wY|S)N post_hookrcSrr.rrr.r.r/r^rz6AutogradCompilerInstance.post_hook..r)rLoutputsr9rrrr.r.r/rs  z"AutogradCompilerInstance.post_hookcCs|t|tjsJ|jdusJ|j|}|j||dd}tt|g}|||gWd|S1s7wY|S)Npost_acc_grad_hookr)rOr)rPrKrrr8rp)rLinputrrrr.r.r/rs   z+AutogradCompilerInstance.post_acc_grad_hookcCs6i}d}t|j}|djdksJ|d}t|j}tt}|||dks+J|t|d}|||dks=Jt|D]>\} } |sS| jdj j dkrSd}qA| jdj j d k} t| jd dk} | r| rt| j} t d d | Dr| || <qA|r| D] } | jd| jd<qt|SgS) NFrr9rWracudaTcpucss,|]}t|jtjjo|jjdvVqdS))primsatenN)rOrr)_ops OpOverload namespacer[userr.r.r/r]$s   zDAutogradCompilerInstance.move_graph_nodes_to_cuda..)listnodesruserskeyslenrmrometarrvrallvaluesr)rLrkto_movehas_cuda_inputsrr9 inputs_usersfirst_getitem_idxlast_getitem_idxrnodeis_cpu is_scalar node_usersr.r.r/move_graph_nodes_to_cudas8   z1AutogradCompilerInstance.move_graph_nodes_to_cudacCs6t|tjjo|jdko|jtjjjj tjjj j fvS)Nr) rOr)riNodeopropsrsym_sizerq sym_numeldefault)rLrr.r.r/ is_sym_node7sz$AutogradCompilerInstance.is_sym_nodecsftt|jjjddD] \}}|jq |tt dks$Jfdd}|jj |dS)NrY)rrWcs0|vp|jdkp|jdkp|jdko|jtvS)NrYoutputr)rr_impure_targetsr unpack_nodesr.r/ is_impureIsz/AutogradCompilerInstance.dce..is_impure) r$rorIrk find_nodesupdaterrrrmeliminate_dead_code)rLrrrr.rr/dce?s  zAutogradCompilerInstance.dcec s|jdtjdi|j|jdd|j||fi| | | | | |||gtrL||jjt|jj|jjdtdgtddddd}td |td |td fd d d fdd}||fS)Nrr.rCompiledAutogradr9zCompiled autograd graphT)include_deviceinclude_stridecoloredz%scompiled_autograd_graphcs jddS)NF) print_output)print_readabler.)rkr.r/s z6AutogradCompilerInstance.end_capture..) payload_fncsrz5daD]}||jdd||<qt|||||WdWdaS1s,wYWdadSdaw)NT) non_blockingF)in_compiled_autograd_region pin_memoryr_disable) compiled_fnr9r:r;r<r)runtime_inputs_to_mover.r/runtime_wrappers z=AutogradCompilerInstance.end_capture..runtime_wrapper)rIrZr _exec_final_callbacks_stubrFrG create_node create_argrrename_aot_dispatcher_nodesreorder_tensor_pre_hook_nodes'reorder_pre_hook_nodes_to_schedule_asapreorder_accumulate_grad_nodes%reorder_pre_hook_nodes_to_mimic_eager reorder_post_acc_grad_hook_nodesreorder_post_hook_nodesrr6rrkrrhrrcompiled_autograd_loginfo verbose_logdebugrrC)rLrlazy_graph_coderr.)rkrr/ end_captureSsV        z$AutogradCompilerInstance.end_capturec Cs|jdurdSdtjjjdtjjjfdd}|jD]\}}|d}|d}|dj}t|j }t |}|dus;Jz|j d krJt |}|j d ksAWn t yTYqwz|t|j jj } t|D]} t | qat | } | j d kr|| |st | } | j d kr|| |rv|j d kr| j d kr| jst | } q|| |st d|d |j| _t|jD]\} } d|d | j| j| _qt |}t | } |j d kr| j d ksWqt ytd |j|||YqwdS) z Renames nodes as they appear in the AOTDispatcher backward graphs, prefixed by AOT id e.g. AOTDispatcher backward graph X's `sin_Y` -> `aotX_sin_Y` NcaaotcSs|j|jk}|st|jdot|jdo|jj|jjk}|sFt|jdrFt|jdrF|jdkrFt|jddrF|j|jdk}|o]|j|jko]|j|jko]t|j t|j kS)N__name__rSz aten::reshape original_aten) rhasattrrrSrgetrrvrall_input_nodes)rr target_matchr.r.r/ is_similars0       zHAutogradCompilerInstance.rename_aot_dispatcher_nodes..is_similarca_node_start_idxaot_idaot_gmrr_zIFailed to match %s%s (NodeCall %s) nodes with AOT backward graph %s nodes)rdr)rirrreitemsrkiterrnextr StopIterationrIrangerrSrorrr)rLrnodecall_indexrrr aot_graphaot_itaot_nodeca_itrca_noderinpr.r.r/rsb           z4AutogradCompilerInstance.rename_aot_dispatcher_nodescCsdd|D}|S)NcSs g|] }t|tjjur|qSr.)rvr)rir)r[nr.r.r/r^s z:AutogradCompilerInstance.get_all_nodes..r.)rrr.r.r/ get_all_nodessz&AutogradCompilerInstance.get_all_nodescCs8|jdks|jdkr|jtjkr|jdjdkrdSdS)NrYrrTF)rroperatorgetitemrrr.r.r/is_placeholders   z'AutogradCompilerInstance.is_placeholdercCs|jjjdtjjjjdD]:}|jd|jd}}d}|j t j kr)|}|jd}t ||g}||j urG||sG|||durG||q dS)a  Usage of AOTAutograd causes all the accumulate_grad_ nodes to get pushed to the end of the graph. This differs from eager mode, which schedules them as soon as possible. This pass attempts to reorder the graph to mimic eager behavior. rrrrrWN)rIrkrr)rinductoraccumulate_grad_rrrrrmaxprevrr)rLr param_node grad_node getitem_nodeargr.r.r/rs       z6AutogradCompilerInstance.reorder_accumulate_grad_nodescCsl|jjjdtdD]*}|jdddkrq |jd}|jd}||jur3||s3| || |q dS)a Usage of AOTAutograd causes all the tensor_pre_hook nodes to get pushed to the end of the graph. This differs from eager mode, which schedules them as soon as possible. This pass attempts to reorder the graph to mimic eager behavior. rrrNrrrW) rIrkrr rrrrrr)rLrr input_noder.r.r/rs     z6AutogradCompilerInstance.reorder_tensor_pre_hook_nodesc Cs|jjjdtdD]s}|jdddkrq |jd}||jd}g}g}|g}|D]}|jdkrJ|j t j krJ| |jd| || |q+t ||D]\}} ||| | qPt|} | |jur||| s|| ||D]}| |qtq dS)a In this function, we schedule the pre hooks as soon as possible. This does not match eager behavior (schedule pre hook right before its registered node), but it can make acc grad be scheduled properly when the pre hooks are registered to them. After reordering acc grad node, we will reorder the pre hooks again to mimic eager behavior. rrrNrrrW)rIrkrr rrrrrrrrrzipremoverrr) rLrr input_nodes to_remove to_append hook_blockrabrr.r.r/r*s4        z@AutogradCompilerInstance.reorder_pre_hook_nodes_to_schedule_asapcCsg}|jjjdtdD]}|jdddkrq ||q t|D]D}|jd}t |j }t |dkr6q!t dd|DsAJtt|dj }||jure|||||D]}||q]q!dS) a% Usage of AOTAutograd causes all the pre_hook nodes to get pushed to the end of the graph. This differs from eager mode, which schedules them right before their registered node execution. This pass attempts to reorder the graph to mimic eager behavior. rrrNrrcss&|]}|jdko|jtjkVqdS)rN)rrrrrr.r.r/r]es  zQAutogradCompilerInstance.reorder_pre_hook_nodes_to_mimic_eager..)rIrkrr rrrreversedrrrrrrrrprepend)rL pre_hooksrhook_getitem_noderregistered_noderr.r.r/rNs.          z>AutogradCompilerInstance.reorder_pre_hook_nodes_to_mimic_eagercCsg}|jjjdtdD]}|jdddkrq ||q t|D]<}|jd}|jd}d}t |j D]}|j dkrJ|j tjjjjkrJ|}nq6|dusSJd||||q!dS) a Usage of AOTAutograd causes all the post_acc_grad_hook nodes to get pushed to the end of the graph. This differs from eager mode, which schedules them as soon as possible. This pass attempts to reorder the graph to mimic eager behavior. rrrNrrrWz8post_acc_grad_hook must have corresponding acc grad node)rIrkrr rrrr)rrrrrrr)rrrr)rLpost_acc_grad_hooksrrr acc_grad_noderr.r.r/rqs.         z9AutogradCompilerInstance.reorder_post_acc_grad_hook_nodesc slg}|jjjdtdD]jdddkrq |q t|D]jd}jd}jd}t |dkr9q!g}| t ||D]}| fd d t |j DqDt|}|jdkr|jtjjjjkr|jd}d} t |j D]} | jdkr| jtkr| jddd kr| } qx| dur| ||q!|jur||s|||q!dS) a  Usage of AOTAutograd causes all the post_hook nodes to get pushed to the end of the graph. This differs from eager mode, which schedules them as soon as possible. This pass attempts to reorder the graph to mimic eager behavior. rrrNrrrWc3s:|]}|jdkr|jtkrjdddks|VqdS)rrNr)rrr rrrrr.r/r]s  zCAutogradCompilerInstance.reorder_post_hook_nodes..r)rIrkrr rrrr)rrextendrrrrrrr)rrrrrr) rL post_hooksr output_nodesr#input_nodes_and_usersr rrpost_acc_grad_hook_noderr.rr/rsN                 z0AutogradCompilerInstance.reorder_post_hook_nodescs|durdSt|trfdd|DSt|tr$tfdd|DSt|tjs,Jtj|}t|tjjj j s=J|j S)Ncrr.rrr\r.r/r^rz5AutogradCompilerInstance.to_proxy..c3s|]}|VqdSr1rrr\r.r/r]sz4AutogradCompilerInstance.to_proxy..) rOrrr)rPrrIrirw proxy_tensor _ProxyTensorr)rLtr6r.r\r/rs   z!AutogradCompilerInstance.to_proxycsttjjrB|r5t|t|ksJg}tt|D]}||\}}|||d||q|n fddtt|Dt|tksLJt|d|j ddS)Ncsg|]}|qSr.r.)r[rrr.r/r^rzDAutogradCompilerInstance.bind_tensors_to_proxies..constanttracer) rOr)rir&rr set_node_originrrrI)rLtensorsrrU bound_proxiesrr  node_namer.r9r/rps z0AutogradCompilerInstance.bind_tensors_to_proxiesindexcCs4|jdusJ|j|}t}t||d|jd|S)Nr:)rKrrrI)rLrArbw_stater.r.r/bind_backward_states  z,AutogradCompilerInstance.bind_backward_stater@r pyobjc Csd}|dur$|j}t|dr$||_|j}t|jjj||jj d|j |<||d|d}t d}|d|}t|dS)N_aot_id)rrrz (NodeCall )rz:raw_stack_trace = CapturedTraceback.extract().format()[-1]) _forward_clsrrdrFrrIrkr_lazy_backward_info bw_modulerer%extractformatreplacer#) rLr@r rD maybe_aot_id forward_clsnew_coderaw_stack_tracenew_stack_tracer.r.r/r=s     z(AutogradCompilerInstance.set_node_originr>Nr1)+r __module__ __qualname__rMrR staticmethodr rNrr)rPrqrrrrstrrrrrrrrrrrrrrrrrrrrrrrrprCautogradFunctionr=r.r.r.r/r=Fsd  I #   (DZ   $#'<  r=Fc cs|r t|tus Jddlm}|jjdkr%dazdVWdadSdawddl}|jj j t t||\}}trC|jj j tdaz)|jd dVWdn1sZwYW|sdda|jj j ||dS|ssda|jj j ||w)Nr) eval_frame force_eagerTF)rvbool torch._dynamorZ_stancestance%compiled_autograd_enabled_force_eagertorch._inductor.cudagraph_trees_C_dynamor'set_autograd_compiler functoolspartialr=r0set_verbose_loggerrcompiled_autograd_enabledrXset_multithreading_enabled)rCdynamicrZr)prior_compiler prior_dynamicr.r.r/_enablesD      rmc cs`tjjjdd\}}dazdVW|rdatjjj||dS|r&datjjj||w)NFT)r)rbrcr'rdrh)rkrlr.r.r/rIs$  rr>cCs2datrJtjjjddtjjjddS)NF)rhrr)rbrcr'rdrgr.r.r.r/reset\srn)FrS)KrDrertypingrrrrrrrr)torch._dynamo.external_utilsr r r torch._dynamo.sourcer rtorch._dynamo.utilsrrrtorch._loggingrrtorch._prims_commonrtorch._subclassesrtorch.fxr%torch.fx.experimental._backward_stater"torch.fx.experimental.proxy_tensorrrrrrrr%torch.fx.experimental.symbolic_shapesr r!torch.fx.tracebackr"r#torch.utils._ordered_setr$torch.utils._tracebackr%torch.fx.proxyr&rrrr0r6r8rmrrrrrrr=rhr`rcontextmanagerrmrrnr.r.r.r/s^$    $        U )