o Ơie@sddlmZddlZddlZddlZddlZddlmZm Z m Z m Z m Z ddl ZddlZddlZddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1ddl2m3Z3ddl4m5Z5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;mZ>ddl?m@Z@eAeBZCee!fdDddZDedEd#d$ZEee!dfdFd,d-ZFe   . / .dGdHdr2)itemsr _fromtotrtDataTypetorchsplitcudarevallowerr!deviceclonereshape contiguous) r5r7r(engine_weight_mapengine_weight_namesd_weight_namenp_weight_type trt_dtype torch_dtypeparamswscaleshiftr2r2r3,construct_refit_mapping_from_weight_name_mapYs2   rVnew_gm old_enginetrt.ICudaEngine input_list Sequence[Any]Optional[dict[str, List[str]]]Nonec Cs*tt}t|}t|t}|}|r|jdkr!tjj ntjj } | di} t || |} i} | D]0\} }|j}t|}t|tj}t|tj}|d||f| | <q9| | |D](}|| vrt|dqq| |\}}t||t|}|||| qqt |!dksJdnMt"|||} tjj } |D]3}|| vrt#|d| |}t|jtj}t||t|}|||| |$|qt |t |krtd|%st&d t#d Wd d S1swYd S) z* Refit a TensorRT Engine in place rEconstant_mappingr9z is not found in weight mapping.rz/Fast refitting failed due to incomplete mappingz is not found in weight mappingz%Not all weights have been refitted!!!z#Error: failed to refit new weights.zRefitting failed.N)'r setrrARefitterr#get_all_weightstypeTensorLocationDEVICEHOSTpoprVr7r>r rC from_numpyrEr?r@rBrIrJrKupdateloggerwarningWeightsdata_ptrnumelset_named_weightslenget_missing_weightsr4AssertionErroraddrefit_cuda_engineerror)rWrXrZr(r5refitted torch_devicerefitter weight_listtrt_wt_locationr^mappingconstant_mapping_with_type constant_namevalrO val_tensorrPrQ layer_nameweight weight_dtype trt_wt_tensorr2r2r3 _refit_single_trt_engine_with_gmsr           $rFTcompiled_module&torch.fx.GraphModule | ExportedProgramnew_weight_moduler arg_inputsOptional[Tuple[Any, ...]] kwarg_inputsOptional[dict[str, Any]] verify_outputbooluse_weight_map_cachein_placec% Cs0d}t|tr |}tt|dkrd}|st|}n|r%tdd}|r`dd|j D} | dd dt } | d ksGJd t | d }t| } |D]\} } | | | <qVni} g}|D]`\} } t| tt tjjjjfr|d | vr}qht| tjjjr|| qhd d| j D} d}z| j}Wn'tydd| Dd dt } | d ksJd t | d }Ynw| | | <qht|jjD]}|jdkr|j|vr|j|q|D]}| |qt!||dusJ|j"rJdt#|j$}|rt|t%j&j's|g}t(||}i}|rt(||}t)*t+}t|ts3tdt,|t-||}|.t/|j0}|}t12dt3|jt4||}t12dt3|jt56|t78||j9\}}|dksz||j:kr|j;st1<|d|d|j:|St12d|d|d|j=rzt1>dt7j?||j:|j9|j@||kd\}}WntjAjBjCjDyt1jEdddd|_=Ynw|j=st1>dt7jF||j:|j9|j@d\}}|rtG||Hdd |DtI| JksJd!nd"d |Dd"d |DksJd!|D]\} }z|red}| | }d#| vr;|K|LWq| d}tM|tN|}|rd| dt } t | d$}|sdd}t1tyt|tjjr|jQrJd&d'd| Dd dt } t | d$}t|tjAjRjSst1|rt|}#tb|"|#tN<tjejfgth|#}$ti|| d-|$nt|tjjrt|}#tb|"|#tN<tjejfgth|#}$|$|_T~tjktjlmq|r|dur|ntX|j$to||||d.rt1>d/|ntp|S|rt1d3|S)4a< Refit a compiled graph module with ExportedProgram. This performs weight updates in compiled_module without recompiling the engine. Args: compiled_module: compiled TensorRT module that needs to be refitted. This compiled_module should be compmiled by torch_tensorrt.dynamo.compile or load it from disk using trt.load. new_weight_module: exported program with the updated weights. This one should have the same model architecture as the compiled module. arg_inputs: sample arg inputs. Optional, needed if output check kwarg_inputs: sample kwarg inputs. Optional, needed if output check verify_output: whether to verify output of refitted module Returns: A new compiled TensorRT module that has the updated weights. FrTzuExported program does not support modifying in place. Please set in_place to false and use the returned graph module.NcS(g|]\}}d|vr|dd|fqSengine_enginereplace.0namerr2r2r3  z(refit_module_weights..rzThe engine provided is either not refittable or was built with a version of Torch-TensorRT that is too old, please recompile using the latest versionr( _run_on_gpucSrrrrr2r2r3r-rcSg|] \}}|dkr|qSrr2rr2r2r3r9s call_modulezSRefitting is not enabled. Please recompile the engine with immutable_weights=False.z6Input graph should be an ExportedProgram but got type z Input graph: zLowered Input graph: z6 supported operations detected in subgraph containing zV computational nodes. Skipping this subgraph, since min_block_size was detected to be zDetected support for z operators out of z in subgraph.z/Partitioning the graph via the fast partitioner)min_block_sizetorch_executed_opsrequire_full_compilation skip_fusionziPartitioning failed on the subgraph with fast partition. See trace above. Retrying with global partition.)exc_infoz1Partitioning the graph via the global partitioner)rrrcSh|]}|dqSrr2rsmr2r2r3 z'refit_module_weights..zSNew weights module is not compatible with previously compiled Torch-TensorRT modulecSrrr2rr2r2r3rr _run_on_accr5zGThis engine does not have a weight map cache. Rebuilding the weight mapzQRefitting a torch retraced module is only supported with use_python_runtime=FalsecSrrr2rr2r2r3rs zYThe module was compiled with an old version of Torch-TensorRT. Rebuilding the weight map.z]The type of graph module is not supported for refitting or two compiled modules do not match.zRefitting Submodule name: %s )rWrXrZr(r5 INCLUDE_REFITr) new_modulerefitted_modulerrzRefitting Succeed!zRRefitting with weight_name_map yielded incorrect result! The outputs do not match.)rrz+Refitting Failed! The outputs do not match.z1Refitting Completed! Output verification skipped.)r isinstancerr$rolistnamed_childrencopydeepcopyrq__dict__r> __getstate__rrdecode_metadatadictrrCnnmodulesModuleexport_unliftGuardsFnappendr(AttributeErrorgraphnodesoptarget erase_nodedelete_submodulerimmutable_weightsr"rH collectionsabcrr rARuntimer#rbrrun_decompositionsr"enable_experimental_decompositionsridebugstrr CONVERTERSset_compilation_settingsr get_graph_converter_supportrrdryrunrjuse_fast_partitionerinfofast_partitionrfxpasses splitter_baseFxNetSplitterInternalErrorrtglobal_partitionrdelete_all_unused_submodulesr_keysload_state_dictr7get_engine_from_encoded_enginergetattrr5use_python_runtime graph_module GraphModulerconstruct_submodule_inputsr+rr!rcreate_serialization_config clear_flagSerializationFlagEXCLUDE_WEIGHTShasattrset_flagrserialize_with_configbytesserialized_engine setup_engineclassestensorrtEnginetuplesetattrgccollectrE empty_cacher@rrrefit_module_weights)%rrrrrrr inline_moduler(compiled_submodulesencoded_metadatacompiled_submodules_mapr submoduleguard_fn_modulesnodeguard_fn_module_namerH torch_inputstorch_kwarg_inputsruntimerWnum_supported_ops total_opsnew_partitioned_module supported_ops new_submoduler5compiled_submodule engine_inforsubmodule_inputseserialization_configrnew_engine_inforefitted_enginer2r2r3rs                                                       rencoded_enginerr trt.RuntimecCst|}||}|S)N)base64 b64decodedeserialize_cuda_engine)r rrrr2r2r3rns  r)r$r%r&r'r(rr)r*)r5r6r7r6r(rr)r6) rWr%rXrYrZr[r(rr5r\r)r])NNFTF)rrrrrrrrrrrrrrr)r%)r rrrr)rY)J __future__rcollections.abcrrrloggingtypingrrrrrnumpynprrArC torch.exportr"torch.fx.experimental.proxy_tensorr torch_tensorrt._enumsr torch_tensorrt._featuresr torch_tensorrt._Inputr torch_tensorrt.dynamor torch_tensorrt.dynamo._exporterrtorch_tensorrt.dynamo._settingsr,torch_tensorrt.dynamo.conversion._conversionr3torch_tensorrt.dynamo.conversion._ConverterRegistryrr0torch_tensorrt.dynamo.conversion._TRTInterpreterr7torch_tensorrt.dynamo.conversion.impl.normalization.opsr0torch_tensorrt.dynamo.conversion.truncate_doublertorch_tensorrt.dynamo.loweringrrrr8torch_tensorrt.dynamo.runtime._PythonTorchTensorRTModuler2torch_tensorrt.dynamo.runtime._TorchTensorRTModulerrrtorch_tensorrt.dynamo.utilsrrrr r!r"torch_tensorrt.loggingr# getLogger__name__rir4rVrrrrr2r2r2r3sb                   ,R