o :ýÆiˆã@spdZddlZddlZddlZddlmZmZmZddlZddl m Z Gdd„dƒZ d de j de d efd d „ZdS) zO This script provides utility functions for working with TensorRT in ZipVoice. éN)ÚAnyÚTupleÚOptionalc @s‚eZdZdZ ddededefdd„Zd ee effd d „Z d ed efdd„Z dde j de j de j dee j d e j f dd„ZdS)ÚTrtContextWrapperz9A wrapper class for managing TensorRT execution contexts.éúcuda:0Ú trt_engineÚtrt_concurrentÚdevicecCstj|d|_||_||_t|ƒD]&}| ¡}tj  tj  t |¡¡¡}|dus/Jd  |¡ƒ‚|j  ||g¡q|j  ¡dusCJdƒ‚d|_dS)a! Initializes the TrtContextWrapper. Args: trt_engine (Any): The TensorRT engine. trt_concurrent (int, optional): The number of concurrent contexts. Defaults to 1. device (str, optional): The device to use. Defaults to 'cuda:0'. )ÚmaxsizeNz`failed to create trt context, maybe not enough CUDA memory, try reduce current trt concurrent {}Fzno avaialbe estimator contextéd)ÚqueueÚQueueÚtrt_context_poolrr ÚrangeÚcreate_execution_contextÚtorchÚcudaÚstreamÚStreamÚformatÚputÚemptyÚfeat_dim)Úselfrr r Ú_Ú trt_contextÚ trt_stream©rú./home/ubuntu/LuxTTS/zipvoice/utils/tensorrt.pyÚ__init__!s   zTrtContextWrapper.__init__ÚreturncCs|j ¡|jfS)z*Acquires a TensorRT context from the pool.)rÚgetr)rrrrÚacquire_estimator7sz#TrtContextWrapper.acquire_estimatorÚcontextrcCs|j ||g¡dS)zª Releases a TensorRT context back to the pool. Args: context (Any): The TensorRT context. stream (Any): The CUDA stream. N)rr)rr$rrrrÚrelease_estimator;sz#TrtContextWrapper.release_estimatorNÚxÚtÚ padding_maskÚguidance_scalec CsÈ| tj¡}| tj¡}| tj¡}|dur| tj¡}| ¡\\}}}tj ¡ ¡| d¡}| d¡} tj|| |j |j |j d} |Œ|  d|| d¡| d¡f¡|  d|f¡|  d|| d¡f¡|durq|  d |f¡|  ¡ ¡|  ¡ ¡|  ¡ ¡g} |dur|  |  ¡ ¡¡t| ƒD] \} } | | | ¡| ¡q“|j}| |d¡}| ||   ¡ ¡¡| tj ¡j¡d usÂJ‚tj ¡ ¡Wdƒn1sÓwY| ||¡|  tj¡S) ad Executes the TensorRT engine. Args: x (torch.Tensor): The input tensor. t (torch.Tensor): The time tensor. padding_mask (torch.Tensor): The padding mask tensor. guidance_scale (torch.Tensor): The guidance scale tensor. Returns: torch.Tensor: The output tensor. Nrr)Údtyper r&ér'r(r)T)ÚtorÚfloat16r#rÚcurrent_streamÚ synchronizeÚsizerrr*r Úset_input_shapeÚ contiguousÚdata_ptrÚappendÚ enumerateÚset_tensor_addressÚget_tensor_nameÚnum_io_tensorsÚexecute_async_v3Ú cuda_streamr%Úfloat32)rr&r'r(r)Ú estimatorrrÚ batch_sizeÚseq_lenÚoutputÚinput_data_ptrsÚiÚjÚ num_tensorsÚoutput_tensor_namerrrÚ__call__Es:      "ê  zTrtContextWrapper.__call__)rr)N)Ú__name__Ú __module__Ú __qualname__Ú__doc__rÚintÚstrr rÚlistr#r%rÚTensorrrErrrrrs0ÿÿÿ ÿûþýüûúrrÚmodelÚ trt_modelr cCs’tj |¡s Jdƒ‚ddl}t|dƒ}| | |jj¡¡ |  ¡¡}Wdƒn1s-wY|dus=Jd  |¡ƒ‚|` t ||dd|_ dS)a3 Loads a TensorRT engine and replaces the model's fm_decoder with a TrtContextWrapper. Args: model (nn.Module): The model to modify. trt_model (str): The path to the TensorRT engine file. trt_concurrent (int, optional): The number of concurrent contexts. Defaults to 1. zPlease export trt model first.rNÚrbzfailed to load trt {}r)r r ) ÚosÚpathÚexistsÚtensorrtÚopenÚRuntimeÚLoggerÚINFOÚdeserialize_cuda_engineÚreadrÚ fm_decoderr)rNrOr ÚtrtÚfÚestimator_enginerrrÚload_trt€s   ÿr_)r)rIÚloggingrQr ÚtypingrrrrÚtorch.nnÚnnrÚModulerKrJr_rrrrÚs b