o "ig @sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZddlmZmZmZmZmZmZddlZddlmZddlZddlmZddlmZddlmZddlm Z dd l!m"Z"m#Z#d d l$m%Z%d d lm&Z&m'Z'e(e)Z*ed Z+edZ,e,-Z.e.rddl/Z0gZ1dZ2e.rgdZ1e0j3j4j567ddZ8d9dde1DZ2gdZ:GdddZ;ddZe?dddZ@d d!d"d#ZAd$d%ZBd&d'ZCGd(d)d)eDZEd*d+ZFdQd,d-ZG dRd d d.d/d0ZHd1d2ZId3d4ZJd5d6ZK dRd d d.d7d8ZLd9ed:d;dd?ZMd@e+d=eee+ge+ffdAdBZNeNejOZPeNeQdCZReNdZSeNd ZTeNd ZUGdDdEdEZVGdFdGdGZWGdHdIdIZX J  dSdKeeegeefdLeYdMeeeYeZfdNeeZd=eeYeff dOdPZ[dS)TN)Counter) import_module)AnyCallableDictListOptionalTypeVar)Tensor) rand_strided)is_float_dtype)StorageWeakRef)ContentStoreReaderContentStoreWriter)config) clone_inputs get_debug_dirTztorch._inductor.config)z1//caffe2/torch/fb/sparsenn:sparsenn_operators_gpuz-//caffe2/torch/fb/sparsenn:sparsenn_operatorsz///deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpuz+//deeplearning/fbgemm/fbgemm_gpu:sparse_opszfbcode:// cCg|]}d|dqS)ztorch.ops.load_library("z").0xrrW/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/torch/_dynamo/debug_utils.py 6r)buck2runz@mode/dev-nosanc@s&eZdZddZddZd ddZdS) BuckTargetWritercCstjtj|\|_|_|jdd|_|jddd|j|_|j|jdd|_|jdd|_|j}||dddd}d|d |j|_ dS) Nz.pyr/.zfbcode.zfbcode/r:) ospathsplitabspathsubdirpy_filereplacetargetfind cmd_line_path)selffilenametmprrr__init__=szBuckTargetWriter.__init__c CsDdddtD}td|jd|jdtd|d|jd S) NrcSr)z "z",rrrrrrLrz*BuckTargetWriter.build..za load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") python_binary( name="z", srcs = ["z"], compile = False, deps = [ "//caffe2:torch", "//caffe2/functorch:functorch", "//triton:triton", "z", ], cpp_deps = [ z ], main_module = "z", par_style = "xar", ) )join extra_depstextwrapdedentr.r, cur_targetr()r1extra_cpp_depsrrrbuildKs zBuckTargetWriter.buildTcCsntj|jd}t|d}||Wdn1swYt|jg}|r5t dd||S)NTARGETSwzFFound an example that reproduces the error. Run this cmd to repro - %s ) r'r(r5r+openwriter;BUCK_CMD_PREFIXr0logwarning)r1 print_msg target_filefd cmd_splitrrrr@ds  zBuckTargetWriter.writeN)T)__name__ __module__ __qualname__r4r;r@rrrrr"<sr"cCsLtjtd}|durtdt}tj|s$tj |dd|S)Nminifierz /minifier_T)exist_ok) r'r(r5rtempfile gettempdirgetpassgetuserexistsmakedirs)r(rrr minifier_dirrs  rSc@seZdZejjejjejjejjejj ejj ejj ejj ejj ejjejjejjejjejjejjejjejjejjejjejjejjgZeddZeddZdS)NNModuleToStringcCsLt}|D]\}}t|tjvr||qt|dkr$td|dS)Nrz-We have not tested reprs of some modules - %sT) setnamed_childrentyperU safe_reprsaddlenrBrC)gm cant_convert_modulerrrcan_convert_to_strings   z&NNModuleToString.can_convert_to_stringcCsddlm}d}td}|D]+\}}|}t|d}|dur-|jr-|d}||dd|d|d 7}q|j D]X\}} | durKqB| t krcdd l m} | jt ks^Jt| } n t| rvd t| jd | jd } n dt| jd | jd } | jr| d} ||dd|d| d7}qB|j D].\} } | durqd}| jrd}dt| jd | j|d} ||dd| d| d 7}q|||jdd 7}|S)Nr) _addindent z from torch.nn import * class Repro(torch.nn.Module): def __init__(self): super().__init__() z.cuda()zself.z = r) PRINT_OPTSz torch.randn(z, dtype=)ztorch.randint(1, size=zself.register_buffer('z', z) rz, device="cuda"ztorch.nn.Parameter(torch.randn(z))rT)torch.nn.modules.modulerar7r8rW__repr__next parametersis_cuda_buffersitemsnumelMAX_CONSTANT_NUMEL_INLINEtorch._tensor_strrd thresholdreprtorchis_floating_pointlistshapedtype _parameterscode)r\ratab model_str module_namer_ module_str example_param buffer_namebufferrd tensor_str param_nameparam maybe_devicerrrconvertsF            zNNModuleToString.convertN)rHrIrJrrnnLinearConv1dConv2dConv3d BatchNorm1d BatchNorm2d BatchNorm3d LayerNormDropoutSoftmaxReLUGELUIdentity MaxPool2d EmbeddingTanhConvTranspose1dGLULSTMFlattenAdaptiveAvgPool2drY staticmethodr`rrrrrrU~s6  rUc CstjsdSd}z!tddg}|d}ddd|D}||d7}Wnttj fy:|d 7}Ynwt d d t tj D}|d 7}| D]\}}|d |d|d7}qQ|d7}|S)Nz:# torch.cuda.is_available()==False, no GPU info collected z# CUDA Info: nvccz --versionrrcSs g|] }|dvrd|dqS))r#  r)rsrrrrs z-_cuda_system_info_comment..z# nvcc not found css|] }tj|VqdSN)rrcudaget_device_name)rirrr s  z,_cuda_system_info_comment..z# GPU Hardware Info: rz : r)rrr is_available subprocess check_outputdecoder)r5FileNotFoundErrorCalledProcessErrorrrange device_countrl)rzcuda_version_outcuda_version_linescomment gpu_namesnamecountrrr_cuda_system_info_comments&   rF) stable_outputc Cs\ddl}ddl}|r dS|jjj}d|jjd|jjd|j jd|d S)Nrz*# config omitted due to stable_output=Truez~import torch._dynamo.config import torch._inductor.config import torch._functorch.config import torch.fx.experimental._config r) torch._functorch.configtorch._inductor.configfx experimental_configcodegen_config_dynamor _inductor _functorch)rrrexperimental_configrrrgenerate_config_strings   rcCstjtdS)Nzminifier_launcher.py)r'r(r5rSrrrrget_minifier_repro_pathsrc Cst}td|trt|zt|d}||WdWdS1s)wYWdStyF}z tdt d|d}~ww)NzWriting minified repro to: %sr=rz(Could not write to {minified_repro_path}) rrBrCuse_buckr"r@r?OSError exceptionNotImplementedError)contentsminified_repro_pathrFerrrhelper_for_dump_minifys    &  rc@s eZdZdS) AccuracyErrorN)rHrIrJrrrrr"srcCsBt|}tt|D]}t||tjr||||jq |S)z This clone inputs is different from utils clone_input. In case of minifier, all the tensors are leaf tensors while creating a new graph. So, we set the requires_grad field w/o checking the leafness of the tensor. )rrr[ isinstancerrr requires_grad_ requires_grad)example_inputs cloned_inputsidxrrrclone_inputs_retaining_gradness&s rc Csddlm}m}m}t|}|st|}t|dr|dt|dr(||n||}|r0|S||r<||}| |||d|S)z Runs a forward and possibly backward iteration for a given mod and args. When disable_clone is True, we will use args as-is without cloning. This is higher fidelity but we may destroy the args in the process. r)collect_resultsreduce_to_scalar_lossrequires_bwd_pass zero_gradT _boxed_callN) testingrrrcopydeepcopyrhasattrrbackward) r\argsonly_fwd disable_clonerrroutlossrrrrun_fwd_maybe_bwd3s   r require_fp64 ignore_non_fpc Csddlm}t|||}d}tjr:ztt|t|\} } t| | |}Wnt y9|r2t dt dYnwzt|||} Wnt yZ} z t dWYd} ~ dSd} ~ ww||| |tjd|d} | S)aa Check two models have same accuracy. require_fp64: if True, raise an error if we unable to calculate the fp64 reference ignore_non_fp: if True, do not compare outputs which are not floating point. This is mostly useful for the minifier (which wants to avoid quantizing floating point error into integer/boolean error) r)sameNzCould not generate fp64 outputszWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graph.T)tol equal_nanr)utilsrrrsame_two_models_use_fp64 cast_to_fp64rrr Exception RuntimeErrorrBrCrrepro_tolerance)r\opt_gmrrrrrreffp64_ref fp64_model fp64_examplesresrpassingrrrsame_two_modelsNs@    rcCs|jjD]Q}|jdkr5|jtjjjjkr5t |j dksJt |j dr5|j dtj kr5|j dtj f|_ |jdkrU|j d}|durUt |rUt|j }tj |d<||_ q|j||S)N call_functionrcrrrv)graphnodesopr.rropsprimsconvert_element_typedefaultr[rr float64kwargsgetdictlint recompile)modelnoderv new_kwargsrrrcast_dtype_args_to_fp64s       rcsBddlm}|}tjkrt|}|fdd|}||fS)Nr)tree_mapcs"t|tjr|r|S|Sr)rrrr rstorrvrrs  zcast_to..)torch.utils._pytreerrrrrr)rvrinputsrrr rcast_tos    rcCsttj||Sr)rrrr)rr rrrrsrc Cs^z|t|t|}t||||||d WSty.}z tdWYd}~dSd}~ww)NrzWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graphF)rrrrrrBr)r\r compiler_fnrrr compiled_gmrrrrbackend_accuracy_failss&  rstrideztorch._prims_common.StrideTyperuztorch._prims_common.ShapeTypereturncCs|dur|St|Sr)rmake_contiguous_strides_for)rrurrr_stride_or_defaultsrdcs fddS)Ncs|dur|SSrrr rrrr sz_mk_defaulter..rrrrr _mk_defaulters rcpuc@s4eZdZddZdddddZddZd d ZdS) NopInputReadercCs d|_dS)Nrtotal)r1rrrr4s zNopInputReader.__init__Ndevice dtype_hintcCs|jd7_dS)Nrr)r1 storage_hashnbytesrrrrrstorageszNopInputReader.storagecOdSrrr1rrrrrtensorzNopInputReader.tensorcOr#rrr$rrrsymintr&zNopInputReader.symintrHrIrJr4r"r%r'rrrrrs  rc@sLeZdZd ddddZdddddZ d dddddd d Zd d ZdS) InputReaderN)pbarcCs8|dur td|durt|nd|_g|_||_dS)Nz0no save_dir specified, will generate random data)rBrCrstorerr*)r1save_dirr*rrrr4s   zInputReader.__init__rcCs|jdur |jdt|}t|}|jdur=|dur=z|j|}Wn ty-Ynw||jkr;t d||j|St d|||j f}t d|d}t |||| S)Nrzdevice mismatch: %s != %sz1could not load %s, generating random data insteadru)r*update_device_or_default_dtype_or_defaultr+ read_storagerrrBrCitemsizerr untyped_storage)r1r r!rrr"rurrrrr" s"       zInputReader.storage)storage_offsetrvris_leafc Kst||d}t|}t|}t|}t|}tjg||j|d} t| ||||Wdn1s7wY|sut | j tj d} Wdn1sTwYt| ||||Wdn1spwYtj j| |ksJtj| ||j| | S)Nr-)rvrr) memory_format)r_storage_offset_or_defaultr0_is_leaf_or_default_requires_grad_or_defaultrrr%rno_gradset_ enable_gradclonepreserve_format _subclasses meta_utils safe_is_leaf_utilsset_tensor_metadatarappend) r1r"rurr4rvrr5metadatatrrrr%s,      zInputReader.tensorcCs|j||Sr)rrD)r1valrrrr'@s zInputReader.symintrr(rrrrr)s  !r)c@sLeZdZddddZddZdddd efd d Zdd d ZdddZdS) InputWriterF stable_hashcCs:g|_t|_||_|durt||dnd|_i|_dS)NrI)_lines itertoolsrstorage_counterr,rr+ seen_storages)r1r,rJrrrr4Rs   zInputWriter.__init__cCs*dg}|dd|jD|d|S)Nzdef load_args(reader):css|]}d|VqdS)rbNr)rlrrrrbsz$InputWriter.lines..zload_args._version = 0)extendrKrD)r1rrrrlines^s  zInputWriter.linesNr device_hintrc Cst|}|j|}|dur|Sdt|j}d}tdt|kr'd|}d}|j}|jdkr9|dus7J|}td|krDd|}| } d} |j dur[|jjdkr[|j |} |j |d| d| ||d||j|<|S) Nbufrz , dtype_hint=metaz , device=z = reader.storage(, re)r rNrrhrMr0rrXr/r!r+ write_storagerKrD) r1r3rrTwsvmaybe_dtype_hintrrr!r rrrr"ms0        zInputWriter.storagec Cs4|j||j|jd}g}td|jd|kr$|tt |t d|jkr4|d|jt d| krF|d| t j|}|rZ|dd|Dtd|jkrj|d|jt jj|}td|kr|d||jd d |tt |jg|d |dS) NrSr-zdtype=zstorage_offset=css"|] \}}|d|VqdS)=Nr)rkrZrrrrs z%InputWriter.tensor..zrequires_grad=zis_leaf=zreader.tensor(rW) # )r"r3rvrrrurrDstrtupler0r7r4rrrBget_tensor_metadatarPrlr9rr?r@rAr8rKr5)r1rrFr"rtensor_metadatar5rrrr%s2  zInputWriter.tensorcCs0t|tjr |jj}|jd|d|dS)Nzreader.symint(r^)rrrSymIntrhintrKrD)r1rrGrrrr's zInputWriter.symint)rN) rHrIrJr4rRr_r"r%r'rrrrrHQs   rHrfuncr sym_shapesdefault_sym_shapecs~ddlm}dd|D}d|}t|}d|d}d|d } d } Gd d d } i} p5ifd ddtffdd } |j}|D]:\}}|dkrWqNt | |}|rw| \}}t | d}||}| ||| |<t | |}|r|d| |<qNdt|jvr| }|| d<t ||D]}| \}}}}t | d}||}t||| ||q| S)a Takes in a function which has been printed with print_readable() and constructs kwargs to run it. Handles Tensor inputs, Symints, and a graph module which might have tensor constants. Consider a function `forward` defined as follows: def forward(self, primals_1: "f32[1001, 6]", primals_2: "f32[s0]", primals_3: "Sym(s0)",): _tensor_constant0: "i64[4190]" = self._tensor_constant0 # Further implementation kwargs = aot_graph_input_parser(forward) forward(**kwargs) r) dtype_abbrscSsi|]\}}||qSrr)rkeyvaluerrr sz*aot_graph_input_parser..|z(_tensor_constant\d+): \"(z0)\[\s*(.*?)\s*\]\" = self\.(_tensor_constant\d+)(z)\[\s*(.*?)\s*\]z Sym\((s\d+)\)c@seZdZdZdS)z/aot_graph_input_parser..TensorContainerz#Container for tensors as attributesN)rHrIrJ__doc__rrrrTensorContainersrocs,tvp dufddS)Ncs dS)Nz; not in symbolic_shapes and default sym shape not passed inrrr'rrr s z=aot_graph_input_parser..get_sym_int..)rr_checkrrp)rgrfrpr get_sym_ints   z+aot_graph_input_parser..get_sym_intrc sg}g}t|D]"\}}|}d|vr#|}||||q|t|q|jr1tjntj}|||d}|D] } tj || q=|S)Nr)rvr) enumeratestriprDintrsrrrandnzerosr mark_dynamic) rurvresolved_shape dynamic_dimsrdimr constructorrr)rrrrr gen_tensors  z*aot_graph_input_parser..gen_tensor,rr1)torch.fx.graphrhrlr5valuesinspect getsourcer __annotations__researchgroupsr`r)group signaturerifinditersetattr)rerrfrgrh dtype_map dtype_patternsourcetensor_assignment_regex tensor_regexsym_shape_regexrorr} annotationsr annotationmatch data_type shape_strrurv container attr_namer^r)rgrrrrfraot_graph_input_parsersF       r)FF)F)rNN)\r functoolsrOrrLloggingr'rrrMr7 collectionsr importlibrtypingrrrrrr rrtorch._prims_common _prims_commonrtorch._subclasses.meta_utilsr torch._dynamo.testingr r torch.multiprocessing.reductionsr torch.utils._content_storerrrrrr getLoggerrHrBrinductor_config is_fbcoderlibfb.py.build_infolibfbr6 extra_importspy build_info BuildInfoget_build_ruler-r9r5rAr"rSrnrU lru_cacherrrrrrrrrrrrrrrfloat32r0rr/r7r9r8rr)rHr_rurrrrrs          6 d   8 )  T[