o
    ߗi:                     @   s  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd dlmZ d dlmZ d dlmZ ddlmZm Z m!Z!m"Z" dd	l#m$Z$m%Z%m&Z&m'Z' dd
l(m)Z)m*Z*m+Z+ ddl	m,Z, ddl-m.Z. dZ/ee	j0 e1d< zd dl2Z/W n e3y   dZ/Y nw e j4Z4dZ5e6e7Z8deej9 deej9 fddZ:de;de;fddZ<dd Z=dej>j?dedededee f
ddZ@dedeAfdd ZBedej9dej9fd!d"ZCedeee eed#f eeef f deDfd$d"ZCdedeej9eDf fd%d"ZCde;fd&d'ZEdde;d)e	jFd*e;ddfd+d,ZGd d-d.e,d/eHd0ed1ed2eHdee+ fd3d4ZIG d5d6 d6ZJG d7d8 d8ZKG d9d: d:ZLG d;d< d<ZMd)e;de;fd=d>ZNd)e;de;fd?d@ZOdAe;de;fdBdCZPd)e;de;fdDdEZQ			ddFedGed#ef dHeHdIeeH dJeeH dKeHddfdLdMZRdNejSdeej9 ded#ef fdOdPZT	Q	RddSeDdTeDdUeAdVeDde;f
dWdXZUejVdYd fdZeeH d[eeH d\ejWd]ee;ejXf d^eHdej9fd_d`ZYedaZZdeAfdbdcZ[dGed#eZf ddeded#eZf fdedfZ\ddgdh didje]dke;dle;ddedmee; dneed#ef ged#ef f de]fdodpZ^dGed#ef ded#ef fdqdrZ_dGed#ef ded#ef fdsdtZ`dGed#ef ded#ef fdudvZadGed#ef ded#ef fdwdxZbdGed#ef ded#ef fdydzZcdGed#ef ded#ef fd{d|ZddGed#ef ded#ef fd}d~ZedGed#ef ded#ef fddZfddeAddfddZgdS )    N)
AnyCallableDictListOptionaloverloadSequenceTupleTypeVarUnion)patch)fx)	aot_eager)OutputGraph   )config
eval_frameoptimize_assertreset)create_instructiondebug_checksis_generatortransform_code_object)CheckFunctionManager	CompileIdGuardedCode)DynamoFrameType)samenp   xreturnc                 C   s    | d u rd S |    | jS N)detachclonerequires_grad_requires_gradr     r(   S/home/ubuntu/transcripts/venv/lib/python3.10/site-packages/torch/_dynamo/testing.pyclone_me7   s   r*   namec                 C      t dd| S )Nz^_orig_mod[.] resub)r+   r(   r(   r)   remove_optimized_module_prefix=      r1   c                    sJ   ddl m  d d  fdd}tj|dd| |i | jfS )Nr   )InstructionTranslatorc                    s   |    jj| S r"   )
current_txoutputregion_tracker)_gmargskwargsr3   gmr6   r(   r)   extract_graph_backendG   s   z8extract_graph_and_tracker.<locals>.extract_graph_backendT)backend	fullgraph)torch._dynamo.symbolic_convertr3   torchcompilegraph)fnr8   r9   r<   r(   r:   r)   extract_graph_and_trackerA   s   
rD   model
predictionlossexample_inputsc                 C   s  g }| | | | i }i }|  D ]'\}}t| tjr"t|}|}	|j}
|jd u r1t|}
|
||d < |	||< q| | | | i }| 	 D ]\}}t| tjrZt|}|||< qL| | |D ] }t|t
tfrz|dd |D  qft|tjr| |j qf|S )Nz.gradc                 s   s"    | ]}t |tjr|jV  qd S r"   )
isinstancer@   Tensorgrad).0inpr(   r(   r)   	<genexpr>s   s     z"collect_results.<locals>.<genexpr>)appendnamed_parametersrI   r   OptimizedModuler1   rK   r@   
zeros_likenamed_bufferstuplelistextendrJ   )rE   rF   rG   rH   resultsgradsparamsr+   param
param_copyrK   buffersbufferexampler(   r(   r)   collect_resultsR   s:   








r_   outc                 C   sZ   t | tjr	| jS t | ttfrtdd | D S | d u rdS t | tr&dS tdt	| )Nc                 s       | ]}t |V  qd S r"   )requires_bwd_passrL   r    r(   r(   r)   rN   ~       z$requires_bwd_pass.<locals>.<genexpr>FDon't know how to reduce)
rI   r@   rJ   r&   rU   rT   anyintNotImplementedErrortyper`   r(   r(   r)   rb   z   s   
rb   c                 C      d S r"   r(   rj   r(   r(   r)   reduce_to_scalar_loss   s   rl   .c                 C   rk   r"   r(   rj   r(   r(   r)   rl      s   c                 C   s   t | tjr|  |   S t | ttfr"tdd | D t|  S t| j	dv r.t
| jS t| j	dkr:| j S t | trPtdd |  D t|   S tdt| )z/Reduce the output of a model to get scalar lossc                 s   ra   r"   rl   rc   r(   r(   r)   rN      rd   z(reduce_to_scalar_loss.<locals>.<genexpr>)MaskedLMOutputSeq2SeqLMOutput!CausalLMOutputWithCrossAttentionsSquashedNormalc                 s   ra   r"   rm   )rL   valuer(   r(   r)   rN      rd   re   )rI   r@   rJ   sumnumelrU   rT   lenri   __name__rl   logitsmeandictvalueskeysrh   rj   r(   r(   r)   rl      s   


c                  C   s0   t jt jtd} t j| st |  | S )Nz../debug)ospathjoindirname__file__existsmkdir)r}   r(   r(   r)   	debug_dir   s   
r   r-   codeextrac              	   C   sl   t tjt | d!}|t|  dt|  d| d W d    d S 1 s/w   Y  d S )Nwz


)	openr|   r}   r~   r   writedisBytecodeinfo)r+   r   r   fdr(   r(   r)   
debug_dump   s
   ("r   )skipframe
cache_sizehooks_r   c                C   s|   dt t dtddfdd}t| jrdS t| j t| j|}ti dddddd	it t | jg d

}t	|t
|jtd	d	S )zused to debug jump updatesinstructionscode_optionsr!   Nc                 S   s$   |  dtd |  dtd d S )Nr   NOP)insertr   )r   r   r(   r(   r)   insert_nops   s   z&debug_insert_nops.<locals>.insert_nopsF_idr   )
r   compiler_fnroot_txexportexport_constraintsframe_statelocal_scopeglobal_scopef_codetorch_function_mode_stack)r   r   r   r   r   r   r   localsglobalsr   r   guard_managerr   )r   r   r   r   r   r   r   rB   r(   r(   r)   debug_insert_nops   s$   

r   c                   @   sH   e Zd ZdddZdejjdeej de	de
f fdd	Zdd
dZdS )CompileCounterr!   Nc                 C      d| _ d| _d S Nr   frame_countop_countselfr(   r(   r)   __init__      
zCompileCounter.__init__r;   rH   .c                 C   s:   |  j d7  _ |jjD ]}d|jv r|  jd7  _q|jS )Nr   call)r   rB   nodesopr   forward)r   r;   rH   noder(   r(   r)   __call__   s   
zCompileCounter.__call__c                 C   r   r   r   r   r(   r(   r)   clear   r   zCompileCounter.clearr!   N)rv   
__module____qualname__r   r@   r   GraphModuler   rJ   r   r   r   r   r(   r(   r(   r)   r      s    


	r   c                   @   sF   e Zd ZdeddfddZdejjdeej	 de
def fd	d
ZdS )CompileCounterWithBackendr=   r!   Nc                 C   s   d| _ d| _|| _g | _d S r   )r   r   r=   graphs)r   r=   r(   r(   r)   r      s   
z"CompileCounterWithBackend.__init__r;   rH   .c                 C   s\   ddl m} |  jd7  _|jjD ]}d|jv r|  jd7  _q| j| || j	||S )Nr   )lookup_backendr   )
backends.registryr   r   rB   r   r   r   r   rO   r=   )r   r;   rH   r   r   r(   r(   r)   r      s   
z"CompileCounterWithBackend.__call__)rv   r   r   strr   r@   r   r   r   rJ   r   r   r   r(   r(   r(   r)   r      s    
r   c                   @   >   e Zd Zd
ddZdejjdeej de	de
f fdd	ZdS )EagerAndRecordGraphsr!   Nc                 C   s
   g | _ d S r"   )r   r   r(   r(   r)   r      s   
zEagerAndRecordGraphs.__init__r;   rH   .c                 C   s   | j | |jS r"   )r   rO   r   )r   r;   rH   r(   r(   r)   r        zEagerAndRecordGraphs.__call__r   rv   r   r   r   r@   r   r   r   rJ   r   r   r   r(   r(   r(   r)   r      s    

r   c                   @   r   )AotEagerAndRecordGraphsr!   Nc                 C   s   g | _ g | _g | _d S r"   )r   	fw_graphs	bw_graphsr   r(   r(   r)   r   
  s   
z AotEagerAndRecordGraphs.__init__r;   rH   .c                    st    j | dtjjdttj dtdtf f fdd}dtjjdttj dtdtf f fdd}t	||||d	S )
Nr;   rH   r!   .c                        j |  | jS r"   )r   rO   r   r;   rH   r   r(   r)   fw_compiler  r   z5AotEagerAndRecordGraphs.__call__.<locals>.fw_compilerc                    r   r"   )r   rO   r   r   r   r(   r)   bw_compiler  r   z5AotEagerAndRecordGraphs.__call__.<locals>.bw_compiler)r   r   )
r   rO   r@   r   r   r   rJ   r   r   r   )r   r;   rH   r   r   r(   r   r)   r     s*   

z AotEagerAndRecordGraphs.__call__r   r   r(   r(   r(   r)   r   	  s    

r   c                 C   r,   )Nz(?m)^ *#.*\n?r-   r.   r   r(   r(   r)   strip_comment(  r2   r   c                 C   s   d dd | dD S )Nr   c                 S   s   g | ]}|  qS r(   )rstrip)rL   liner(   r(   r)   
<listcomp>-  s    z)remove_trailing_space.<locals>.<listcomp>)r~   splitr   r(   r(   r)   remove_trailing_space,  s   r   gm_strc                 C   s   t t| S r"   )r   r   )r   r(   r(   r)   normalize_gm0  s   r   c                 C   s   t dd| }|S )z-
    Normalize code: remove empty lines.
    z[\r\n]+r   r.   )r   normal_coder(   r(   r)   empty_line_normalizer6  s   r   r   rC   nargsexpected_opsexpected_ops_dynamicexpected_frame_countc                 C   s   t js	|d ur	|}t }dd t|D }dd t|D }|| }	|| }
t  t||}|| }|| }|| }|| }t  | t||	 | t||	 | t||
 | t||
 | |j	| |d urv| |j
| d S d S )Nc                 S      g | ]}t d d qS 
   r@   randnrL   r   r(   r(   r)   r   K      z!standard_test.<locals>.<listcomp>c                 S   r   r   r   r   r(   r(   r)   r   L  r   )r   assume_static_by_defaultr   ranger   r   
assertTruer   assertEqualr   r   )r   rC   r   r   r   r   actualargs1args2correct1correct2opt_fnval1aval2aval1bval2br(   r(   r)   standard_test>  s,   r   r;   c                 C   s   | j S r"   )r   r   r(   r(   r)   dummy_fx_compile_  s   r   T皙?speeduppvalue
is_correctpvalue_thresholdc                 C   s.   |sdS ||kr| ddS | dd|dS )NERRORz.3fzx SAMEzx p=z.2fr(   )r   r   r   r   r(   r(   r)   format_speedupe  s
   r   cpusizestridedtypedevice
extra_sizec                 C   s   t dd t| |D d | }|jr0|jdkr'	 tj|tj|dj|d}ntj|||d}n	tj|g||d}t	|| |S )Nc                 s   s     | ]\}}|d  | V  qdS )r   Nr(   )rL   shaper   r(   r(   r)   rN   z  s    zrand_strided.<locals>.<genexpr>r   )r   r   )r   )r   r   r   )
rs   zipis_floating_pointitemsizer@   r   float16tozeros
as_strided)r   r   r   r   r   needed_sizer]   r(   r(   r)   rand_stridedr  s   
r  _Tc                   C   s   t j S r"   )r   r   r(   r(   r(   r)   check_dynamic_shape_capture  s   r
  patchesc                    s*   t  dtdtdtf fdd}|S )Nr8   r9   r!   c               	      s^   t  !}D ]\}}}|t||| q | i |W  d    S 1 s(w   Y  d S r"   )
contextlib	ExitStackenter_contextr   object)r8   r9   stackmoduleattrvalrC   r  r(   r)   _fn  s
   
$z"_make_fn_with_patches.<locals>._fn)	functoolswrapsr   r	  )rC   r  r  r(   r  r)   _make_fn_with_patches  s   r  c                 C   s   | S r"   r(   r'   r(   r(   r)   <lambda>  s    r  )
xfail_prop	decoratorcls
cls_prefix	fn_suffixr  r  c                G   s   t | | j | ji }|j|_t| D ]P}|drVt| |}t|s.t||t| | q| | }	t	|g|R  }
|	|
_|d urMt
||rMt|
}
t||	||
 qt
||sdt||t| | q|S )Ntest_)ri   rv   	__bases__r   dir
startswithgetattrcallablesetattrr  hasattrunittestexpectedFailure)r  r  r  r  r  r  DummyTestClassr+   rC   new_namenew_fnr(   r(   r)   make_test_cls_with_patches  s$   



r,  c                 C   s   t jdkr| S t| S )N)r      sysversion_infor'  r   rC   r(   r(   r)   skipIfNotPy311  s   

r2  c                 C      t jdkr| S td| S )Nr      zRequires Python 3.12+r.  r1  r(   r(   r)   skipIfNotPy312  s   
r6  c                 C   s   t jdkr
t| S | S )Nr4  )r/  r0  r'  r(  r1  r(   r(   r)   xfailIfPy312  s   

r7  c                 C   s   t jdkrtd| S | S )Nr4  zNot supported in Python 3.12+r.  r1  r(   r(   r)   skipIfPy312  s   
r8  c                 C   r3  )N)r   r   zRequires Python 3.10+r.  r1  r(   r(   r)   requiresPy310  s   
r9  c                 C   
   d| _ | S NT)_expected_failure_dynamicr1  r(   r(   r)   expectedFailureDynamic     r=  c                 C   r:  r;  )!_expected_failure_codegen_dynamicr1  r(   r(   r)   expectedFailureCodegenDynamic  r>  r@  c                 C   r:  r;  )!_expected_failure_dynamic_wrapperr1  r(   r(   r)   expectedFailureDynamicWrapper  r>  rB  Fuse_xlac                 C   sV   t d td trtjd | r)dd lm  m} |dt	|
  d S d S )Ni9  r   )r@   manual_seedrandomseedr   torch_xla.core.xla_modelcore	xla_modelset_rng_stater   
xla_device)rC  xmr(   r(   r)   reset_rng_state  s   

rM  )r-   )NNr   )Tr   )F)hr  r   r  loggingos.pathr|   rE  r/   r/  typesr'  typingr   r   r   r   r   r   r   r	   r
   r   unittest.mockr   r@   r    torch._dynamo.backends.debuggingr   torch._dynamo.output_graphr   r-   r   r   r   r   bytecode_transformationr   r   r   r   guardsr   r   r   r   utilsr   r   
ModuleType__annotations__numpyModuleNotFoundErrorunsupportedthree	getLoggerrv   logrJ   r*   r   r1   rD   nnModuler_   boolrb   rl   floatr   CodeTyper   rg   r   r   r   r   r   r   r   r   r   r   r   r   r   float32r   r   r  r	  r
  r  ri   r,  r2  r6  r7  r8  r9  r=  r@  rB  rM  r(   r(   r(   r)   <module>   sB  
 0

( 


!

	

&
""""""	""