o Ơi2@sddlZddlZddlmZmZmZmZddlZ ddl Z ddl m Z ddl mZddlm mmmZddlmZddlmZmZddlmZddlmZddlmZmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)e*e+Z,eeZ-dddde)j.dddddddddfdej/de0de0dej/fddZ1ej2GdddZ3de j4de-d edefd!d"Z5d ede3fd#d$Z6d%eege3fdefd&d'Z7ej2dd(Gd)d*d*Z8dS)+N)AnyCallableOptionalSequence) SplitResult)TRTInterpreterTRTInterpreterResult) LowerSetting)LowerPassManagerBuilder)PassFuncvalidate_inference)TimingCacheManager) TRTSplitterTRTSplitterSetting) acc_tracer) TRTModule)LowerPrecision iiFTg?modulemin_acc_module_sizemax_batch_sizereturncCsJ| r|stdt|||||||| | | | | ||d}tj|d}|||S)a Takes in original module, input and lowering setting, run lowering workflow to turn module into lowered module, or so called TRTModule. Args: module: Original module for lowering. input: Input for module. max_batch_size: Maximum batch size (must be >= 1 to be set, 0 means not set) min_acc_module_size: Minimal number of nodes for an accelerated submodule max_workspace_size: Maximum size of workspace given to TensorRT. explicit_batch_dimension: Use explicit batch dimension in TensorRT if set True, otherwise use implicit batch dimension. lower_precision: lower_precision config given to TRTModule. verbose_log: Enable verbose log for TensorRT if set True. timing_cache_prefix: Timing cache file name for timing cache used by fx2trt. save_timing_cache: Update timing cache with current timing cache data if set to True. cuda_graph_batch_size: Cuda graph batch size, default to be -1. dynamic_batch: batch dimension (dim=0) is dynamic. use_experimental_fx_rt: Uses the next generation TRTModule which supports both Python and TorchScript based execution (including in C++). Returns: A torch.nn.Module lowered by TensorRT. zThe experimental unifed runtime only supports explicit batch. Please make sure to set explicit_batch_dimension=True when use_experimental_fx_rt=True)rrmax_workspace_sizeexplicit_batch_dimensionlower_precision verbose_logtiming_cache_prefixsave_timing_cachecuda_graph_batch_size dynamic_batchis_atenuse_experimental_rtcorrectness_atolcorrectness_rtol lower_setting) ValueErrorr Lowerercreate)rinputrrrrrrrr r!r"r#use_experimental_fx_rtr%r&r(lowererr/K/home/ubuntu/.local/lib/python3.10/site-packages/torch_tensorrt/fx/lower.pycompiles,'  r1c@s8eZdZUeed<eed<eddZdefddZ dS) LowerTrtInterpreterr(timing_cache_managercCst|j|j}t||SN)rrr r2)clsr(r3r/r/r0r+as zLowerTrtInterpreter.createrc CsF|jjsJdtd|d|jjd}|jjr$|j|d}d}|jrXz |j|}tdWn!tyW}ztd|dt |d}WYd}~nd}~wwt ||jj|jj |jj |jj rktjjntjjd}|j|jj|jj|jj|jj|||jjrtjjntjj|jjd }|j} | r|jr|j|| |S) Nz$Can't find input specs for lowering!z split_name=z, input_specs=z.jsonzTiming cache is used!zCannot load timing cache for z: ) input_specsrexplicit_precision logger_level)rrrstrict_type_constraintsalgorithm_selector timing_cacheprofiling_verbositytactic_sources)r(r6loggerinfo algo_selectorr3get_timing_cache_trt Exceptionwarningstrrrr7rtrtLoggerVERBOSEWARNINGrunrrrr9verbose_profileProfilingVerbosityDETAILEDLAYER_NAMES_ONLYr=serialized_cacheupdate_timing_cache) selfmodr, split_namer@ cache_datae interpreter interp_resultr;r/r/r0__call__hsT     zLowerTrtInterpreter.__call__N) __name__ __module__ __qualname__r __annotations__r classmethodr+r rWr/r/r/r0r2\s  r2modelinputsr(cCs>t}|j |_|j|_|j|_t|||d}||S)N)settings)rruse_implicit_batch_dimrr$rnode_support_previewgenerate_split_results)r]r^r(splitter_settingsplitterr/r/r0default_split_functions recCs t|Sr4)r2r+r'r/r/r0create_lower_trt_interpreters rfcreate_trt_interpreterc s*dtjdtdtdtdtjf fdd }|S)NrQr,r( module_namerc s|}||||}|jrRddl}ddlm}ddlm}|} | |j | } Wdn1s8wY|| ||j |j |dt jd} | St|j|j |j |jd} | S)zq Create a module transformation pass which lowers an `fx.GraphModule` into a `TRTModule` rN)Device)TorchTensorRTModulezcuda:)nameinput_binding_namesoutput_binding_names target_device)engine input_names output_namesr!)r$iotorch_tensorrt._Deviceri*torch_tensorrt.dynamo._TorchTensorRTModulerjBytesIOwritero serializegetvaluerprqtorchcudacurrent_devicerr!) rQr,r(rhrU interp_resrrrirj engine_bytes engine_str trt_modulergr/r0 lower_passs2     z&default_lower_pass..lower_pass)nnModuleInputr rD)rgrr/rr0default_lower_passs&r)frozenc @sxeZdZUdZeed<eeefde de de ddfddZ dd e j d ed eed ee egefde j f ddZd S)r*a,Lowers a module using fx2trt. This is a composable class to facilitate fx2trt. A normal fx2trt process composes of the following passes to transform an `fx.GraphModule`: 1. trace - use torch.fx to trace the module so we can get the graph representation of the model. 2. split - the graph module is split into several submodules, running either via TensorRT, or via regular CUDA. For each split that need to run via TRT, the following passes are invoked: 3. `TRTInterpreter` - build the TRT engine for the submodule that can be supported through `TRTInterpreter`. 4. Wraps the executable TRT engine into `TRTModule`, which is an `nn.Module`. 5. The converted submodule is then set back onto the top-level module lower_pass_manager_builderr(interpreter_builder split_funcrcsFjs|tfdd|t|ddS|tdd|t|ddS)z!Instantiate a `Lowerer` instance.cstj||jjdS)N)ast_rewriter_allow_listleaf_module_list)rtracerrrr^r'r/r0s z Lowerer.create..)r( trace_funcr lower_func)rcSs t||Sr4) aten_tracer opt_tracerr/r/r0r s)r#r r)r5r(rrr/r'r0r+s" zLowerer.createNrr^additional_inputsfp16_conversion_fncsNjjj}j}t||ddtjdtdtjffdd }|||S)N)atolrtolrr^rcs||jjjtjkr&|durddntfdd|D}jr1j |}nj |}||}|S)NcSs |dur|jtjkr|S|Sr4)dtyperyfloat32half)xr/r/r0r/s z4Lowerer.__call__..do_lower..c3s|]}|VqdSr4r/).0r conversion_fnr/r0 5sz5Lowerer.__call__..do_lower..) evalrr(rrFP16rtupler#build_aten2trt_lower_pipelinebuild_trt_lower_pipeline)rr^pm lower_resultrrr(rPrr0do_lower s$ z"Lowerer.__call__..do_lower)rr(r%r&r rrr)rPrr^rrrrrr/rr0rWs& zLowerer.__call__)NN)rXrYrZ__doc__r r[r\rfrer rr+rrrrrWr/r/r/r0r*s8  &r*)9 dataclassesdcloggingtypingrrrrtensorrtrErytorch.fxfxtorch.nnr4torch_tensorrt.fx.tracer.dispatch_tracer.aten_tracertracerdispatch_tracerrtorch.fx.passes.splitter_baserfx2trtrr r(r !passes.lower_pass_manager_builderr passes.pass_utilsr r tools.timing_cache_utilsrtools.trt_splitterrrtracer.acc_tracerrrrutilsr getLoggerrXr>rrrintr1 dataclassr2 GraphModulererfrr*r/r/r/r0s|           @A    ,