o
    Ơi                     @   s   d dl Z d dlmZmZmZ d dlZd dlZd dlmZ G dd dZ	ej
jr.ejj ane	jae eZdeddfdd	Zdefd
dZdefddZG dd deZdejjdeejjejjf fddZdeejjejjf defddZdS )    N)AnyOptionalUnion)CudaGraphsTorchTensorRTModulec                   @   s   e Zd ZdZdZdZdS )CudaGraphsModer         N)__name__
__module____qualname__STANDARDSUBGRAPH_CUDAGRAPHSWHOLE_GRAPH_CUDAGRAPHS r   r   V/home/ubuntu/.local/lib/python3.10/site-packages/torch_tensorrt/runtime/_cudagraphs.pyr      s    r   modereturnc                 C   s:   | rt jnt jatjjrtjj	
t td|   d S )NzSet Cudagraphs usage to )r   r   r   _PY_RT_CUDAGRAPHStorch_tensorrtENABLED_FEATUREStorch_tensorrt_runtimetorchopstensorrtset_cudagraphs_modeloggerinfo)r   r   r   r   r      s
   r   c                   C      t tjkrdS dS NTF)r   r   r   r   r   r   r   get_whole_cudagraphs_mode+      
r   c                   C   r   r   )r   r   r   r   r   r   r   get_cudagraphs_mode4   r    r!   c                   @   sV   e Zd ZdZdejjddfddZdeejjej	j
f fddZd	eddfd
dZdS )_CudagraphsContextManagerzrHelper class used in conjunction with `enable_cudagraphs`

    Used to enable cudagraphs as a context manager
    compiled_moduler   Nc                 C   s   t | _|| _d | _d | _d S N)r   old_moder#   cudagraphs_module
old_module)selfr#   r   r   r   __init__C   s   
z"_CudagraphsContextManager.__init__c                 C   s8   t | jtjr| jj| _t| jj| j_| jS t| jS r$   )
isinstancer#   r   MutableTorchTensorRTModulegmr'   get_cuda_graph_module)r(   r   r   r   	__enter__J   s
   

z#_CudagraphsContextManager.__enter__argsc                 G   s2   t | j | jr| j  | jr| j| j_d S d S r$   )r   r%   r&   _reset_captured_graphr'   r#   r,   )r(   r/   r   r   r   __exit__S   s   

z"_CudagraphsContextManager.__exit__)r	   r
   r   __doc__r   nnModuler)   r   fxGraphModuler.   r   r1   r   r   r   r   r"   =   s
    	r"   r#   c                 C   s   d}d}|   D ]!\}}t|dr|jrtdd|v r!|d7 }qd|v r)|d7 }q|dkrEtjatjj	r<t
jjt td t| S |dkrOtd ntd	 td
 | S )Nr   requires_output_allocatorzThe model contains submodules that require a dynamic output allocator at runtime, which is incompatible with CUDA Graphs. Please disable CUDA Graphs._run_on_accr   _run_on_gpuzSFound pytorch subgraphs in module, wrapping module in CudaGraphsTorchTensorRTModulez7No graph breaks detected, using runtime cudagraphs modezNPlease consider dynamo if there is graph breaks. Using runtime cudagraphs modeT)named_childrenhasattrr7   RuntimeErrorr   r   r   r   r   r   r   r   r   r   r   debugr   )r#   num_torch_modulenum_trt_modulenamemoduler   r   r   r-   ]   s<   
r-   c                 C   s   t | S r$   )r"   )r#   r   r   r   enable_cudagraphs   s   rB   )loggingtypingr   r   r   r   r   <torch_tensorrt.dynamo.runtime._CudaGraphsTorchTensorRTModuler   r   r   r   r   r   r!   r   r   	getLoggerr	   r   boolr   r   objectr"   r5   r6   r3   r4   r-   rB   r   r   r   r   <module>   s0    	
		 
,