o ߗiD @sddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZddlZddlmZddlmZddlmmZddlmmZddlmZmZddl m!Z!ddl"m#Z#m$Z$gdZ%d e&d ee&e&ffd d Z'd e edej(de e&e ffddZ)dej(de e&e fdejj*fddZ+d7dejj*d ejj*fddZ,dej*d ej*fddZ-dej*de ej(de ej(de ej(fdd Z.ej/ej0ej1ej2ej3ej4ej5ej6ej7ej8ej6ej9ej:g Z;ejej/ej?ej0ej@ej1d!d"iZAde ej(de e&ej*ffd#d$ZBde ej(de e&ej*fd%e ej*ej*ffd&d'ZCGd(d)d)ZDd8d,d-ZEd.eDd eFfd/d0ZGGd1d2d2ZHdejIfdejj*d3ee e&e fd4eejId ejj*fd5d6ZJdS)9N) defaultdict)Enum)AnycastDictIterableListOptionalTupleType)ArgumentTarget) ShapeProp)fuse_conv_bn_evalfuse_linear_bn_eval) matches_module_patternreplace_node_modulefuseremove_dropoutextract_subgraphmodules_to_mkldnn reset_modules MklSubgraphgen_mkl_autotuneruse_mkl_length UnionFindoptimize_for_inferencetargetreturncCs*|dd^}}|r|d|fSd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnamer%`/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/torch/fx/experimental/optimization.py _parent_name$sr'patternnodemodulescCst|jdkr dS|jd|f}t||D]2\}}t|tjs"dS|jdkr*dSt|jts3dS|j|vr;dSt ||j|urGdSqdS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)r(r)r*nodes expected_type current_noder%r%r&r.s     r new_modulecCs<t|jtsJt|j\}}|||j<t||||dSN)r/rr3r'setattr)r)r*r8 parent_namer$r%r%r&rBs rFmodelc Cs4tjtjftjtjftjtjftjtjfg}|st |}|r&t |t j j s,t |}n|}t|}t |j}|D]W}|jD]Q}t|||rt|jdjdkrTqA||jdj} ||j} | jseqA|dtjtjtjfvrwt| | } nt| | } t|jd|| ||jd||qAqeZdZdedeedfdeeefdeffdd Z Z S)z&remove_dropout..DropoutRemoverrr-.kwargsrcs:t|j|tjrt|dksJ|dSt|||S)Nr r)r/ submodulesr=Dropoutr,superr+)selfrr-rY __class__r%r&r+zsz2remove_dropout..DropoutRemover.call_module) __name__ __module__ __qualname__r r r rr3rr+ __classcell__r%r%r^r&DropoutRemoverys  rd)r0rIrG Transformer transform)r<rTrdr%r%r&rss  r orig_moduler5inputsoutputscs|t}i|D] }||j}||<q|D]}||fdd}||<q|fdd|D|t||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. cs|Sr9r%)xenvr%r&z"extract_subgraph..csg|]}|qSr%r%).0outputrkr%r& z$extract_subgraph..)r0Graph placeholderr$ node_copyrplintrH)rgr5rhrirUinputnew_noder)r%rkr&rs     rcCs t|Sr9) th_mkldnnMkldnnBatchNorm)a_r%r%r&rms rmcCsi}|D]9}|jdkr=t|jtsJ||j}t|tvr=tt||tj}t|tj s0Jt |||<t |||q|S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r+) r2r/rr3r4 mkldnn_maprGfloatr=ModulerErFr)r5r* old_modulesr) cur_moduler8r%r%r&rs    rrcCsJ|D] }|jdkr"t|jtsJ||j}||vr"t||||qdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. r+N)r2r/rr3r)r5r*rr)rr%r%r&rs  rc@seZdZdejfddZdS)rfx_graphcCs||_g|_g|_g|_dSr9)rr5 start_nodes end_nodes)r]rr%r%r&__init__s zMklSubgraph.__init__N)r`rarbr0rsrr%r%r%r&rsr r cs*dddtdtffdd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) NrLrcs|j}dur|jj|jjtdd|Dtttj dd|j D}t |j ||fdd}|fdd}t jj t|fdd}||kS) NcSsg|]}t|jqSr%)rGrandnshaperor)r%r%r&rqsz@gen_mkl_autotuner..use_mkl_heuristic..cSsg|]}|jdqS)r)r-rr%r%r&rqscs<tD]}|qt}tD]}|qt|Sr9)rangetime)fr|begin)iterswarmupr%r& benchmarks   z?gen_mkl_autotuner..use_mkl_heuristic..benchmarkcsddddDDS)NcSg|]}|qSr%)to_denseroir%r%r&rqszRgen_mkl_autotuner..use_mkl_heuristic....cSrr%) to_mkldnnrr%r%r&rqrrr%r% sample_inputs submoduler%r&rmsz>gen_mkl_autotuner..use_mkl_heuristic..csSr9r%r%rr%r&rm rn)rr owning_modulerr propagaterrr0r1rrr5rrLrJrK)rL input_nodes output_argsrmkl_time no_mkl_timeexample_inputsrTrrrrr&use_mkl_heuristics" z,gen_mkl_autotuner..use_mkl_heuristic)rbool)rrrrr%rr&rs rrLcCst|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )r,r5)rLr%r%r&rsrc@sFeZdZddZdefddZdedefddZd ed efd d Zd S)rcCsdg||_dg||_dS)Nrr#size)r]nr%r%r&rs zUnionFind.__init__vcCs||j|<d|j|<dS)Nr r)r]rr%r%r&make_sets zUnionFind.make_setrcCsB|j|}||kr |S|dusJ|||j|<tt|j|Sr9)r#findrint)r]rparr%r%r&r#s  zUnionFind.findr{bcCsf||||}}||kr|S|j||j|kr ||}}||j|<|j||j|7<dSr9)rrr#)r]r{rr%r%r&join+s  zUnionFind.joinN)r`rarbrrrrrr%r%r%r&rs r pass_configtracerc s@dddtid}|duri}|||drt|}|dr#t|}|ddur+|St|dts6td d|dvr@td |dd}|}|t |t |j t| }Gd d d t}tjD]}|j} |jd kr||j} t| tvr|j} t| d} | dur| jtjksJd| jtdksJdn|jdkr|jtvr|j} n|jtvr|j} | |jkr"| |jkrtdd|j Dsqk!|t "|j fdd} Wdn1swYt#t$t j%j&| |_ '|(dd|f} |)| |f| _ Wdn 1swYqkt*tj|}|_+jD]B}|jdkrr|jdkrr|j d}t|j,}|D]}|jdkrc|jdkrc|)|-|qKt.|j,dkrr-|q1t.j}t/|fddt0jD]w\}}|jdkr|jdkr||_12|q|jdkr|jdkrĈ|j ddusJ|j d|_3qfdd|j4D}t.|dkrאqtdd|DrJt5|}|d|_6|ddD] }7|d|qqt8fd d}jD]9}t9|d!r|:|j6j;|t9|d"r1|:|j1j<;|t9|d#rC|:|j3j=;|q |>D](}||sp|j<|j=D]}|j d}|)|-|qVt?|j||qId}jD]}|jdks|jdkr|d7}qwt@AtBCd$|Dt |}|S)%a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fusermkldnn_layout_optimizeNrrrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configc@seZdZdZdZdZdS)z*optimize_for_inference..MklSupportr rN)r`rarbNOYESUNKNOWNr%r%r%r& MklSupportasrr+z)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functioncss|]}|jdkVqdS)rN)r)roargr%r%r& sz)optimize_for_inference..csd|fS)Nr) call_methodrrr%r&rmsz(optimize_for_inference..rrrrcs0t|dr |jSt|dr|jSdS)Ncolor start_color)hasattrrrrr)ufr%r& get_colors    z)optimize_for_inference..get_colorcs,g|]}t|tjr|dur|qSr9)r/r0r1r)rr%r&rqs  z*optimize_for_inference..css|]}|duVqdSr9r%rr%r%r&rsr cstSr9)rr%rr%r&rmrnrr end_colorzmkldnn conversions: %s)Erupdaterrr/rJ RuntimeErrortracerErFr0rHrootrKrlistr5rr2rr4mkldnn_supportedrnext parametersdtyperGr~devicemkldnn_supported_unknownranyr-inserting_beforemap_argrr r)r inserting_after create_noderOrrrMrPr,r enumeraterrrall_input_nodessortedrrrrrappendrrvaluesrlogging getLoggerr`inforv)r<rrdefault_pass_configr cur_tracerr*rr)supports_mkldnnrsample_parameter mkldnn_argsdense_xrprv_noderMuser num_nodescur_idx cur_colors other_color mkldnn_graphsrLprvmkldnn_conversionsresultr%)rrrr&r5s                                      r)FF)rr )KrEroperatorr collectionsrenumrtypingrrrrrr r r rGtorch.fxr0torch.nnr=torch.nn.functional functionalFtorch.utils.mkldnnutilsmkldnnry torch.fx.noder r torch.fx.passes.shape_proprtorch.nn.utils.fusionrr__all__r3r'r1rrrrrrr@rDrAReLU MaxPool2d AvgPool2dAdaptiveAvgPool2drelu transposesigmoid avg_pool2dadaptive_avg_pool2draddmulr MkldnnConv2d MkldnnLinearr}rrrrrrrTracerrr%r%r%r&s  (         (   "   .