o
    "iT                  	   @   s^  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ d dlmZ d dlZd dlmZmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d d	l m!Z!m"Z" d d
l#m$Z$ d dl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 e2e3Z4ee Z5e 6dddgZ7g dZ8e9dde:fddZ;d5dee+ fddZ<dee+ dej=fddZ>		 d6de5dee?e?f dee? d e@fd!d"ZAdee?e?f fd#d$ZBd%ejjde5ddfd&d'ZCejDd(d) ZEG d*d+ d+ZFG d,d- d-ZGejHG d.d/ d/ZIeJ ZKd0d1 ZLd2e?fd3d4ZMdS )7    N)AnyDictListOptional)patch)
draw_graphget_aot_graph_nameget_graph_being_compiled)fx)save_graph_reprowrap_compiler_debug)get_debug_dir)GraphModule)_extract_tensor_metadataTensorMetadata)legalize_graph)tree_map   )configir)BaseSchedulerNodeFusedSchedulerNodeNopKernelSchedulerNode
OutputNodeSchedulerNode)VBufMetanamen_origin)dotz-Gnslimit=2z-Gnslimit1=2z-Gmaxiter=5000returnc                   C   s2   zt jddgt jd W dS  t jy   Y dS w )Nwhichr   )stderrTF)
subprocesscheck_outputPIPESubprocessError r'   r'   S/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/_inductor/debug.pyhas_dot.   s   r)   Fnodesc           	   	   C   s   t  s
td dS |du rt }t| }|jD ]?}d|jvr q|jd j}t|t	r<t|d t
r8|d f}n|d }d}t|tjrH|jj}t||ddddd}||jd< q|r^t| ti |}t| |j  t||dtjjd dS )z$
    Draw a graph in fname.svg.
    z*draw_buffers() requires `graphviz` packageNfusion_metar   tensor_metaF)
clear_metadot_graph_shape)r)   logwarningr	   create_fx_from_snodesr*   metagroup
isinstancetupleintr   ComputedBufferdatadtyper   printr   r   graphlintr   r   tracer.   )	r*   print_graphfnamer;   noder3   r9   metadatagmr'   r'   r(   draw_buffers7   s6   






rC   snodesc              
      s8  dd }t dg d}i }tj }d}g }d}| D ]}| r&d}	|	}n-| r/d}	|	}n$t|tr9d}	|	}nt|t	rDd	}	|j
}nt|trOd
}	|j
}ntdtjj| d}
|	 d|
 }||}i }t|drud| i}|j|d|d} fdd  |r|| | }||_||||	|jd< t|tr|jD ]}||| < q|||< |du r|}q| D ]J}| }|jj}|| }g }|D ]2}|j|v r||j }n || ||j}|||j< W d   n1 sw   Y  || qt||_q|t |dkr|d  |S t| |S )B
    Creates a FX Graph from a list of SchedulerNode objects.
    c                 S   s   dd }| |_ |S )Nc                  W   s   dS Nr   r'   )argsr'   r'   r(   func1g      z;create_fx_from_snodes.<locals>.get_fake_func.<locals>.func1)__name__)r   rH   r'   r'   r(   get_fake_funcf   s   z,create_fx_from_snodes.<locals>.get_fake_func
FusionMeta)r3   snodetypeNexterntemplatenopcomputefusedzUnknown node typeoriginal_atenz: 
get_devicedevicer'   rG   kwargsc                    s6   t | trt fdd| jD S tdd | jD S )Nc                 3   s    | ]} |V  qd S Nr'   ).0x	in_outputr'   r(   	<genexpr>   s    z;create_fx_from_snodes.<locals>.in_output.<locals>.<genexpr>c                 s   s    | ]	}t |jtV  qd S rY   )r4   r@   r   )rZ   userr'   r'   r(   r^      s    )r4   r   anyrD   users)rM   r\   r'   r(   r]      s   
z(create_fx_from_snodes.<locals>.in_outputr+   r   r   )!collections
namedtupletorchr
   Graph	is_externis_templater4   r   r   r3   r   RuntimeError	_inductorutilsget_fused_kernel_name	get_nodeshasattrrU   call_functionappendget_namer   r2   rD   read_writesreadsinserting_beforeplaceholderr5   rG   outputlen)rD   rK   rL   buf_to_fx_noder;   
first_nodeoutputsr3   rM   	node_type
fused_name	func_name	node_funcrX   fx_noder   r[   depsnew_argsdepdep_noder'   r\   r(   r1   a   s   









r1   node_name_to_buf_nameparent_buf_name	n_originsc           
      C   s   | d u rd S | D ]S}|  }| }|d ur)t|dkr)t|||d u r%|n| qt|dkr5|d |ks7J |j}|d u sC|jd u rDq|jD ]}|j}	|	|vrZ|d u rV|n|||	< qGqd S )Nr   r   )rp   rl   rv   $update_orig_fx_node_name_to_buf_namer@   originsr   )
r*   r   r   r   r@   buf_namechildren_nodesir_nodeorigin	node_namer'   r'   r(   r      s0   
r   c                 C   sl   i }|   D ]\}}||vr|h||< q|| | qi }|   D ]\}}t|| }t||||< q"|S rY   )itemsaddrv   r   )r   buf_name_to_n_noder   r   node_name_to_buf_metan_noder'   r'   r(   get_node_name_to_buf_meta   s   r   rB   c                 C   sP   i }t || |du rdS t|}| jjD ]}|j|v r%||j|jd< qdS )rE   Nbuf_meta)r   r   r;   r*   r   getr2   )rB   rD   r   r   r@   r'   r'   r(   annotate_orig_fx_with_snodes   s   

r   c               	   c   s    t jdddk} dd l}t|jjj}t	
 }| s,z
d V  W |  d S |  w |tdd t jt d}t j|dd tt j|d	t  d
}|tj |td || zd V  W || |  d S || |  w )NTORCH_COMPILE_DEBUG01r   z*functorch.compile.config.debug_partitionerTtorchinductor)exist_okaot_z
_debug.log3[%(filename)s:%(lineno)d %(levelname)s] %(message)s)osenvironr   torch._functorch.aot_autogradlogging	getLogger
_functorchaot_autogradrJ   
contextlib	ExitStackcloseenter_contextr   pathjoinr   makedirsFileHandlerr   setLevelDEBUGsetFormatter	Formatter
addHandlerremoveHandler)compile_debugrd   r/   stackr   fhr'   r'   r(   enable_aot_logging   s>   




r   c                   @   s   e Zd Ze Zedd Zedede	e fddZ
dd Zd	efd
dZd$dedefddZejd$dedefddZdefddZdd Zdd ZdedefddZdd Zdd  Zd!d" Zd#S )%DebugContextc                    s"   t   fdd}t|ddS )Nc                     s8   t    | i |W  d    S 1 sw   Y  d S rY   )r   rW   fnr'   r(   inner-  s   $z DebugContext.wrap.<locals>.innerinductor)compiler_name)	functoolswrapsr   )r   r   r'   r   r(   wrap+  s   zDebugContext.wrapfolder_namer    c                 C   sV   t jjpt }tjD ]}tj|d|  d| }tj	|s(t
| |  S q
d S )Nr   .)r   r=   	debug_dirr   r   _counterr   r   r   existsr   )r   r   ndirnamer'   r'   r(   create_debug_dir4  s   

zDebugContext.create_debug_dirc                 C   s   d | _ d | _t | _d S rY   )_prof_pathr   r   _stack)selfr'   r'   r(   __init__B  s   zDebugContext.__init__new_pathc                 C   s   | j sd S |dsJ |ddlm} z.|| d tj|r(t| t	| j | W d    W d S 1 s;w   Y  W d S  t
yT   td| j | Y d S w )Nz.debugr   )FileLockz.lockz(Failed to copy debug files from %s to %s)r   endswithfilelockr   r   r   r   shutilrmtreecopytreeOSErrorr/   r0   )r   r   r   r'   r'   r(   copyG  s   
&
zDebugContext.copywfilename
write_modec                 O   s.   | j sJ ttj| j ||g|R i |S rY   r   openr   r   r   )r   r   r   rG   rX   r'   r'   r(   fopenW  s   
$zDebugContext.fopenc                 o   s\    | j sJ ttj| j ||g|R i |}|V  W d    d S 1 s'w   Y  d S rY   r   )r   r   r   rG   rX   fr'   r'   r(   fopen_context[  s
   
&"zDebugContext.fopen_contextsuffixc                 C   s   | j sJ tj| j |S rY   )r   r   r   r   )r   r   r'   r'   r(   r   a  s   
zDebugContext.filenamec                 C   s   t jjd urJdd l}| jsJ tj| jtj| j d}|	|d}|j
| jtj| jd W d    n1 s=w   Y  t j| d S d S )Nr   z.tar.gzzw:gz)arcname)r   r=   
upload_tartarfiler   r   r   r   basenamer   r   )r   r   tar_filetarr'   r'   r(   r   e  s   
zDebugContext.upload_tarc                    s   t jrtd  j} tj  fdd}| j|| | j	t
|  t jjs-d S | t | _t jjr?| dtj t jjrL| dtj d S d S )Nztorch._dynamoc                    s     |  d S rY   )r   )levelr/   r'   r(   reset_log_levelw  s   z/DebugContext.__enter__.<locals>.reset_log_levelz	debug.logzinfo.log)r   debugr   r   r   r   r   r   callbackr   r   set_debug_handlerr=   enabledr   r   r   	debug_log_setup_log_captureinfo_logINFO)r   
prev_levelr   r'   r   r(   	__enter__q  s   
zDebugContext.__enter__r   c                 C   sp   t d}| j| |}t |}|| |t d |	| |t
|j| | j|j| d S )Nztorch._inductorr   )r   r   r   r   r   StreamHandlerr   r   r   r   minr   r   r   )r   r   r   r/   fdchr'   r'   r(   r     s   



zDebugContext._setup_log_capturec                 C   sF   | j r| j   |   | jr|   tdt | j | j	  d S )Nz%s debug trace: %s)
r   disable_save_profile_datar   r   r/   r0   r	   r   r   )r   exc_typeexc_valexc_tbr'   r'   r(   __exit__  s   
zDebugContext.__exit__c                 C   s   | j sJ | j | d | d)}tj| j |d}|  |d |d |d |d W d    d S 1 s?w   Y  d S )Nzcompile.profzcompile.stats)streamcumtimed   tottime)	r   
dump_statsr   r   pstatsStats
strip_dirs
sort_statsprint_stats)r   r   statsr'   r'   r(   r     s   



"zDebugContext._save_profile_datac                 C   sT   t jjr$tt j|r$ztt| |W S  ty#   tjddd Y d S w dd }|S )Nz Ignoring exception in debug codeT)exc_infoc                  _   s   d S rY   r'   rW   r'   r'   r(   ignored  rI   z)DebugContext.__getattr__.<locals>.ignored)r   r=   r   getattrDebugFormatter	Exceptionr/   r0   )r   r   r  r'   r'   r(   __getattr__  s   zDebugContext.__getattr__N)r   )rJ   
__module____qualname__	itertoolscountr   staticmethodr   strr   r   r   r   r   r   contextmanagerr   r   r   r   r6   r   r   r   r  r'   r'   r'   r(   r   (  s$    

r   c                
   @   s   e Zd Zdd Zdejjdeej fddZ	dejjdeej fddZ
d	efd
dZd	efddZded	efddZd	efddZdejjd	efddZdd Zdedeej dedef dedef
ddZdS ) r	  c                 C   s"   |j | _ |j| _|j| _|| _d S rY   )r   r   r   handler)r   r  r'   r'   r(   r     s   
zDebugFormatter.__init__rB   inputsc                 C   sz   |  d}t|||d W d    n1 sw   Y  |  d}||jdd W d    d S 1 s6w   Y  d S )Nzfx_graph_runnable.pyr   zfx_graph_readable.pyFprint_output)r   r   writeprint_readabler   rB   r  r   r'   r'   r(   fx_graph  s   "zDebugFormatter.fx_graphc                 C   sB   |  d}||jdd W d    d S 1 sw   Y  d S )Nzfx_graph_transformed.pyFr  )r   r  r  r  r'   r'   r(   fx_graph_transformed  s   "z#DebugFormatter.fx_graph_transformedr*   c                 C      |  d| d S )Nzir_pre_fusion.txt	_write_irr   r*   r'   r'   r(   ir_pre_fusion     zDebugFormatter.ir_pre_fusionc                 C   r  )Nzir_post_fusion.txtr  r  r'   r'   r(   ir_post_fusion  r!  zDebugFormatter.ir_post_fusionr   c                 C   s`   |  |!}td|j |D ]}||  |d qW d    d S 1 s)w   Y  d S )NzWriting debug ir to  %sz


)r   r/   infor   r  	debug_str)r   r   r*   r   r@   r'   r'   r(   r    s   "zDebugFormatter._write_irc                 C   s   t || dd d S )Nzgraph_diagram.svg)r?   )rC   r   r  r'   r'   r(   graph_diagram     zDebugFormatter.graph_diagramc                 C   s,   t || t|| ddtdtjjd d S )Nzorig_fx_graph_diagram.svgFT)r?   r-   progparse_stack_tracer.   )r   r   r   GRAPHVIZ_COMMAND_SCALABLEr   r=   r.   )r   rB   r*   r'   r'   r(   draw_orig_fx_graph  s   

z!DebugFormatter.draw_orig_fx_graphc                 C   s   t || d d S )Nzoutput_code.py)r   r   r   )r   r   r'   r'   r(   output_code  r&  zDebugFormatter.output_coder   input_nodestimingsChoiceCallerelapseprecompile_elapsec                    s   dd l }ddlm  dtjf fdd|tj tj fdd|D ||d	}| jd
ddd,}|	 D ]\}	}
t
|	 }|| |
|d< ||| |d q8W d    d S 1 sbw   Y  d S )Nr   r   )FixedLayoutr@   c                    sz  t | dr	| j}nd}|t| jd}z^|  }t| rhd}zt|j}W n tyE   zt	j
jj|jdd}W n	 tyB   Y nw Y nw  |j|jtt	j
j|jtt	j
j|j|d}t||d< nt|  |d< W n ty } zW Y d }~nd }~ww z
t|  |d< W n ty } zW Y d }~nd }~ww z
t|  |d	< W n ty } zW Y d }~nd }~ww ztt	j
j|  |d
< W n ty } zW Y d }~nd }~ww ztt	j
j|  |d< W n ty } zW Y d }~nd }~ww ztt	j
j|  |d< W n ty% } zW Y d }~nd }~ww t | dr;t| jtjr;| j|d< |S )Nr    )r   rN   r   )fallback)r9   sizestrideoffsetlayoutr9   rV   r5  r4  numelr8   )rm   r   rN   rJ   
get_layoutr4   r6   r6  r
  r   r;   sizevars	size_hintrV   r9   list
size_hintsr4  r5  r  	get_dtyperU   
get_strideget_size	get_numelr8   r   IRNode)r@   r   	node_infor7  r6  static_layouter1  build_node_infor'   r(   rG    s   


z>DebugFormatter.log_autotuning_results.<locals>.build_node_infoc                    s   g | ]} |qS r'   r'   )rZ   r@   )rG  r'   r(   
<listcomp>5  s    z9DebugFormatter.log_autotuning_results.<locals>.<listcomp>)op_namecuda_device_namecuda_device_countr,  autotuning_timeprecompile_timezautotuning_result_json_list.txtatzutf-8)encodingbenchmark_result
)jsonr   r1  rB  rd   cudaget_device_namedevice_countr   r   dict	info_dictupdatedumpr  )r   r   r,  r-  r/  r0  rR  general_propertiesr   callertimerW  r'   rF  r(   log_autotuning_results  s,   =
"z%DebugFormatter.log_autotuning_resultsN)rJ   r  r  r   rd   r
   r   r   Tensorr  r  SchedulerNodeListr   r"  r  r  r%  r*  r+  r   rB  r   floatr]  r'   r'   r'   r(   r	    s2    

r	  c                   @   s    e Zd ZU eed< ejed< dS )TensorMetadataHoldertensor_metadatarV   N)rJ   r  r  r   __annotations__rd   rV   r'   r'   r'   r(   ra  D  s   
 ra  c            
      O   s   d}t j|st | dd }t|| |f\}}d}| d| dtt d}t|d}t	||f| W d	   n1 sAw   Y  t
tjr[d
| d|d}	t|	 d	S d	S )z
    This function is used to save arguments for a compile_fx_inner function call
    to the file system.  Later on one can replay the compile_fx_inner call
    with the saved arguments using load_args_and_run_compile_fx_inner.
    z/tmp/inductor_saved_argsc                 S   s    t | tjrtt| | jS | S )z
        Pickle FakeTensor will result in error:
        AttributeError: Can't pickle local object 'WeakValueDictionary.__init__.<locals>.remove'

        Convert all Tensor to metadata. This may also makes pickle faster.
        )r4   rd   r^  ra  r   rV   r[   r'   r'   r(   handle_tensorX  s   z5save_args_for_compile_fx_inner.<locals>.handle_tensorcompile_fx_inner/_z.pklwbNz3
Arguments for a compile_fx_inner call is saved to z. To replay the call,
run the following:

from torch._inductor.debug import load_args_and_run_compile_fx_inner
load_args_and_run_compile_fx_inner(z
)
        )r   r   r   mkdirr   nextsave_args_cntr   picklerY  r/   isEnabledForr   r   r:   )
rG   rX   folderre  args_to_savekwargs_to_savefn_namer   r   messager'   r'   r(   save_args_for_compile_fx_innerM  s$   
rt  r   c              	   C   s   ddl m} t| d}t|\}}W d    n1 sw   Y  dd }tjjdd}|6 t	dd	 t
|||f\}}||i |W  d    W  d    S 1 sXw   Y  W d    d S 1 shw   Y  d S )
Nr   )rf  rbc                 S   s0   t | trtjj| jj| jj| jj	| j
S | S rY   )r4   ra  rd   _dynamotestingrand_stridedrb  shaper5  r9   rV   rd  r'   r'   r(   re    s   
z9load_args_and_run_compile_fx_inner.<locals>.handle_tensorT)allow_non_fake_inputs	save_argsF)torch._inductor.compile_fxrf  r   rm  loadrd   _subclassesFakeTensorModer   r   r   )r   rf  r   rG   rX   re  	fake_moder'   r'   r(   "load_args_and_run_compile_fx_inner{  s   Rr  )FNrF   )Nrb   r   dataclassesr   r  r   r   os.pathrm  r   r   r#   typingr   r   r   r   unittest.mockr   rd   functorch.compiler   r   r	   r
   torch._dynamo.repro.after_aotr   r   torch._dynamo.utilsr   torch.fx.graph_moduler   torch.fx.passes.shape_propr   r   torch.fx.passes.tools_commonr   torch.utils._pytreer   r2  r   r   	schedulerr   r   r   r   r   virtualizedr   r   rJ   r/   r_  rc   r   r)  	lru_cacheboolr)   rC   re   r1   r  r6   r   r   r   r  r   r   r	  	dataclassra  r  rl  rt  r  r'   r'   r'   r(   <module>   s   
*_

#

(  .