o i;%@sUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z mZmZmZddlmZmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,ddl-Z-dd l-m.Z.m/Z/m0Z0ddl1m2m3m4Z5ddl6m7m8Z9ddl:Z;ddlZ?dd l@mAZAdd lBmCZCdd lDmEZEddlFmGZGddlHmIZIddlJmKZKmLZLmMZMmNZNmOZOddlPmQZQddlRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[ddl\m]Z]ddl^m_Z_ddl`maZambZbmcZcddldmeZeddlfmgZgmhZhddlimjZjmkZkmlZlmmZmmnZnddlhmoZompZpmqZqmrZrmsZsddltmuZuddlvmwZwmxZxmyZymzZzddl{m|Z|ddl}m~Z~mZddl8mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd lmZmZmZe rdd!lmZdd"lRmZdd#l\mZdd$lmZdd%lmZdd&lmZdd'l8mZneZd(ed)<z ddlZejZd*ZWneydZd+ZYnwe'd,Ze!d-Ze!d.Ze!d/Ze"ee.fZd(ed0<e"eee.fZd(ed1<e"e;jje;jjfZd(ed2<eeZeje jd3d4Ze;jjZeed5d6Zeed7d8Z e"ed9eed9fd:d;eee"eeed9fd9d:d;ffZd(ed<<ddAdBZejd*dCGdDdEdEZŐddIdJZƐddNdOZǐddSdTZȐddUdVZɐdd\d]Zgd^Zgd_Z dddddeZ͐ddgdhZ dddidjZedddmdnZedddpdnZ *dddsdnZАddwdxZѐdd{d|ZҐdd~dZӐdddZԐdddZՐdddZ֐dddZאdddZؐdddZGdd;d;Zed+dCGdddZeGdddeڃZܐdddZeGddde܃ZeGdddeރZededededededdZded< *ddddZeGddde܃Zde/dfddd„Zeee.ee.gefZd(ed<GddńdeZGddDŽdeZGddɄdeZeGdd˄de܃ZeGdd̈́deZeGddτde܃ZdddфZdddӄZ * +  + dddd܄ZdddބZdddZeGdddeڃZeGdddeZeGdddeZeGdddeZeGdddeZeGdddeZeGdddeZeGdddeZGdddeZeGdddeڃZeGdddeZeGdddeZdddZdddZGdddZeGdddeZGdddeZGdddeZGdddeZGdd d eZGd d d eZeGd d d eZGdddeZed+dCGdddeekZed+dCGdddeeۃZGdddeZ Gddde Z Gddde Z eGdddeڃZ eGdddeڃZ ed+dCGdddeZGd d!d!eZGd"d#d#eZe"eeeeee"eeeeffZGd$d%d%ZGd&d'd'eZGd(d)d)eZGd*d+d+eZGd,d-d-eZGd.d/d/eZdd3d4Zed+dCGd5d6d6eZGd7d8d8eZGd9d:d:eZed+dCGd;d<d<eZed+dCGd=d>d>eZGd?d@d@eZ GdAdBdBeZ!GdCdDdDeZ"GdEdFdFeZ#GdGdHdHe#Z$GdIdJdJe#Z%GdKdLdLeZ&GdMdNdNeZ'GdOdPdPeZ(GdQdRdReZ)GdSdTdTeZ*GdUdVdVe*Z+GdWdXdXe!Z,GdYdZdZeZ-Gd[d\d\eZ.Gd]d^d^eZ/Gd_d`d`eZ0GdadbdbeZ1GdcddddeZ2ed+dCGdedfdfZ3Gdgdhdhe!Z4ed+dCGdidjdje4Z5Gdkdldle4Z6eGdmdndneZ7GdodpdpeZ8ejGdqdrdreڃZ9Gdsd9d9e9Z:Gdtdudue9Z;ed+dCGdvdwdweڃZ<ddydzZ=ed+dCGd{d|d|eZ>ed+dCGd}d~d~eZ?dddZ@ed+dCGdddeZAGddde4ZBGdddeڃZCeGdddeCZDeGdddeCZEGddde4ZFGdddeFZGGdddeFZHGdddeFZIdddZJdddZKdS() annotationsN) Container GeneratorIterableIteratorSequence)AbstractContextManager nullcontext)Enum)partial) AnyCallablecastClassVarLiteralOptionaloverload SupportsFloat SupportsInt TYPE_CHECKINGTypeVarUnion) assert_neverNeveroverride ParamSpecSelf TypeAliasTypeIs)patch)ExprIntegerSymbol)identity)GraphModuleSerializer)can_auto_functionalize)metricsget_free_symbols)compute_required_storage_lengthis_boolean_dtypeis_float_dtypemake_channels_last_strides_for StrideType)get_schema_info) &_remove_effect_token_unbacked_bindingscompute_unbacked_bindings free_symbolsfree_unbacked_symbols IterateExprsrebind_unbackedresolve_unbacked_bindingsShapeEnvSymTypes)Node OrderedSet)CleanDivFloorDivModularIndexing)SymT)config dependencies)BackendFeature CodegenSymbolget_scheduling_for_deviceindex_prevent_reorderingKernel)Depextract_free_symbols#extract_input_node_reduction_rangesextract_read_writes var_builder)LoopBody) OpCounterCSE OpCountResult ReductionType StoreMode) benchmarker)DeviceProperties ReductionHint)argsort argsort_sym cache_on_selfcache_on_self_and_argsceildivconvert_shape_to_inductorconvert_shape_to_symintdeveloper_warningdo_bench_using_profilingdtype_from_sizeget_dtype_sizeget_kernel_metadataGPU_ALIGN_BYTES ir_dataclass is_dynamicis_gpu sympy_dotsympy_index_symbolsympy_index_symbol_with_prefix sympy_product sympy_substensor_is_aligned)opsOpsValueV)FakeScriptObject) SympyBoolean)Argument) CUDATemplate)PythonWrapperCodegen) GraphLowering)IndentedBufferrrpTF_P_T_U_V_IntLike_NumLike _OpOverloadsz prefixTORCH_AUTOTUNE_WARMUPTORCH_AUTOTUNE_REPd TensorBoxr"IRNode _NodeOrNodesxobjectreturnboolcCst|ttfSN) isinstanceintr!rrH/home/ubuntu/vllm_env/lib/python3.10/site-packages/torch/_inductor/ir.py _is_staticr)frozenc@s>eZdZUded<ded<ded<ded<d ed <d ed <d S)GraphPartitionSignatureOrderedSet[sympy.Symbol] symbol_inputsz5dict[str, Union[IRNode, sympy.Expr, TorchBindObject]] input_nodes list[IRNode] output_nodeszdict[str, bool]input_deallocationrskip_cudagraph list[str]constant_namesN__name__ __module__ __qualname____annotations__rrrrrs  r node_or_nodesOptional[_NodeOrNodes]Nonecsdfdd |dS)Nnodesrrrc s|durdSt|ttfr|D]}|qdSt|tr*|D]}|q!dSt|ttttt j j j t tttf sEJdt|ddS)NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])rlisttupledictvalues ExpandView DynamicScalar AssertScalarrsympylogicboolalgBooleanr rEffectfulKernelShapeAsConstantBuffertype)rnode_check_tensorboxrrrs2    z%validate_ir.._check_tensorbox)rrrrr)rrrr validate_irs rnamestrCallable[..., OpsValue]cs(tts Jtdfdd }|S) Nargsrkwargsrrkcstt|i|Sr)getattrrjrrrrrfnzops_wrapper..fn)rrrrrrk)rrr)rrrrr ops_wrappersrorder Sequence[int]&Callable[[Sequence[_T]], Sequence[_T]]cs(tt|tt|dfdd }|S)Nindex Sequence[_T]rc0ttks JfddttDS)Ncg|]}|qSrr.0i)r inv_orderrr "z4inverse_reorder..reindex..lenrangerrrrreindex z inverse_reorder..reindexrrrr)rziprrrrrrrinverse_reordersrcsdfdd }|S)Nrrrcr)Ncrrrr)rrrrr*rz1same_reorder..reindex..rrrrrr(rzsame_reorder..reindexrrrrrr same_reorder'srreindex1&Callable[[Sequence[_U]], Sequence[_V]]reindex2&Callable[[Sequence[_T]], Sequence[_U]]&Callable[[Sequence[_T]], Sequence[_V]]csdfdd }|S)Nrrr Sequence[_V]c |Srrrrrrrr3 z fuse_reindexing..reindex)rrrrr)rrrrrrfuse_reindexing/sr)rr?)rrrr?seq(Sequence[Union[int, torch.SymInt, Expr]] shape_envOptional[ShapeEnv]cCs4|dus tdd|Drt|}|St||}|S)z1 Convert strides to fill order (argsort) Ncss |] }t|ttjfVqdSr)rrrr!rsrrr Csz!get_fill_order..)allrTrU)rr sorted_idxrrrget_fill_order=s  rSequence[Union[int, Integer]]cs0ddt|Dfddtt|D}|S)z Convert stride order to fill order For channel last format, stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0] cSi|]\}}||qSrrridxposrrr Rz+stride_order2fill_order..cg|]}|qSrrrlookuprrrSz+stride_order2fill_order..) enumeraterr)r fill_orderrrrstride_order2fill_orderKsrcCs>t||}ddtt|D}t|D]\}}|||<q|S)z) Convert strides to stride order cSg|]}dqSrrr_rrrr^z$get_stride_order..)rrrr)rrroutrelemrrrget_stride_orderWs  r Literal[None] guard_shapecCdSrrrr rrrir_node_to_tensordr  torch.TensorcCr rrr rrrr hr Optional[IRNode]Optional[torch.Tensor]cs|durdS|stjjjntfdd|D}t|r,fdd|jD}nt |}| }| }t |}t |}tjjjtj||||d}Wd|S1s_wY|S)Ncg|]}|qSrrrshape_fnrrrwrz%ir_node_to_tensor..crrrrrrrrzr)sizestridedtypedevice)rlgraphsizevars size_hintr#get_sizeis_storage_and_layout get_layoutrFlexibleLayoutcontiguous_strides get_dtype get_devicerZrsuppress_guardstorch empty_stridedzero_)rr rrrrtrrrr ls.   valueOptional[Sequence[_T]] Optional[Sequence[Optional[_T]]]cCst|tr |s dgS|Sr)rrr'rrrmay_convert_to_optionalsr+2Union[IRNode, OutputSpec, torch.device, None, str] Optional[str]cCsbt|ts |dur |St|tjr|jSt|ttfr!t|St d|dt|j ddS)Nzget_device_type(: )) rrr#rrr OutputSpecget_device_typer!rrrrrrr1s   r1&Union[IRNode, torch.device, None, str]cCstt|}|dvrtt|ddkrdSdS|dus"t|}dur$dSddlm}t|ts5Jt|t||S)N)cpucuda_backendtritonTFr?)TritonScheduling) r1rr@rDcodegen.tritonr7rr issubclass)rrdevice_schedulingr7rrr is_tritons  r;cCs t|dkS)Nr3)r1rrrris_cpurr<Union[Buffer, TensorBox] alignmentrcsttrduststrdStfddttdD}t j j ddkpFt j j ddk}|oJ|S)NFc3s.|]}tjj|dkVqdS)rN)rlrrsize_hint_or_throw get_striderr>rrrrs  z2is_aligned_realized_tensor_hint..r?) rrmaybe_get_strider2r@rrrrrlrrr?)rr>aligned_stridesaligned_last_dimrrAris_aligned_realized_tensor_hints   rFstrides1Sequence[_IntLike]strides2shapecCst|t|krt|t|ksJt|||D]'\}}}tjj|dr&qtjj||s?tjj|tjj|ks?dSqdS)zP Returns true if the strides are equal, ignoring dimensions of size 1 . r?FT)rrrlrrstatically_known_leqstatically_known_equals symbolic_hint)rGrIrJdims1s2rrrsignificant_strides_equals$rQtensorstrides"Sequence[Union[int, torch.SymInt]]cCst|s|Stddt||Dr|St|||s"|St|\}}g|j}t|D]\}}t j j |drE||||<q3t |j|j|j||j|j}tt||dS)a Tries to match the strides of the tensor to those in the meta_strides. Strides of insignificant dimensions - size 0 or 1 - will be updated. If there are real stride differences (NHWC vs NCHW), or the tensor is not realized, then the input will be returned cs$|] \}}tjj||VqdSrrlrrrLrrOrPrrrr  z2try_match_insignificant_strides..r?datalayout)rrrr@rQras_storage_and_layoutrrrlrrrK FixedLayoutrrroffset is_pinnedrReinterpretView)rRrSstorage old_layout new_striderr new_layoutrrrtry_match_insignificant_stridess.     regmtorch.fx.GraphModulecCsD|jjddd}ddt|jD|jd<ddlm}||dS)Noutput)oprcSsg|]\}}|qSrr)rrrrrrrsz.gm_original_output_strides..user_visible_output_idxs)record_original_output_strides)r find_nodesrrmetatorch._inductor.compile_fxrk)rf output_noderkrrrgm_original_output_strides s   rpinputsSequence[IRNode] list[Expr]cCs@t}|D]}|t|ddO}|t|ddO}qt|S)NF unbacked_only)r:r(rr@r)rqsym_varsinprrrget_symbolic_inputss rxc@seZdZUdZeZded<ejddZ ded<ejddZ ded <ejddZ d ed <e e jdddZe dddZdddZdddZddd Zdd!d"Zdd#d$Zdd&d'Zdd(d)Zddd-d.Z *ddd2d3Zdd5d6Zdd8d9Zdd;d<Zdd>d?ZddAdBZddDdEZddFdGZ ddIdJZ!ddLdMZ"e#ddOdPZ$ddRdSZ%ddTdUZ&ddWdXZ'ddd\d]Z(dd_d`Z)ddbdcZ*ddddeZ+ddgdhZ,ddjdkZ-ddmdnZ.ddodpZ/ddqdrZ0ddsdtZ1ddudvZ2dddydzZ3dd}d~Z4dddZ5dddZ6dddZ7 ddddZ8dddZ9dddZ: ddddZ;dddZdddZ? ddddZ@dddZAdddZBdddZCdddZDdddZEdddZFdddZGdddZHeIrie#dddZJdYSdYS)rzBase class for all intermediate representation (IR) nodes in TorchInductor. Note: This is an abstract base class. Most methods raise NotImplementedError and must be overridden by concrete subclasses. zClassVar[OrderedSet[Any]]_current_originsF)initOrderedSet[Any]originsOptional[list[str]] tracebackOptional[torch.fx.Node] origin_nodeOrderedSet[Node]rGenerator[None, None, None]ccs.tj}||Bt_z dVW|t_dS|t_wr)rry)r|oldrrrcurrent_origins/s  zIRNode.current_originsrrcCst|tttttfSr)rComputedBuffer InputsKernel InputBufferr`TemplateBuffer)rrrris_realized_node9szIRNode.is_realized_nodeattrrr'r rcCst|||dSr)r __setattr__)selfrr'rrr_post_init_setattrFszIRNode._post_init_setattrcCs@t|j}|d||dtjrtnd|dddS)Nr|r~r)r:ryrr@debug_ir_tracebackr~ format_stack)rr|rrr __post_init__Ls  zIRNode.__post_init__OrderedSet[str]cCtdd|DS)Ncs|]}|jVqdSrrrdeprrrrUz(IRNode.get_read_names..r: get_readsrrrrget_read_namesTzIRNode.get_read_namescC|jSr)r~rrrr get_tracebackWzIRNode.get_tracebackcCrrrrrrrget_origin_nodeZrzIRNode.get_origin_nodeOptional[Operation]cCr rrrrrrget_defining_op]r zIRNode.get_defining_opcCst}|j}t|tr|}|jrt|g}|D]:}t|dr*|jr*||jqt j j j di |jg}t|ts>q|D]}t j j j |d}|rR||q@q|S)N stack_trace postToPre)r:r|r ExternKernelrrhasattrraddr# _inductordebug _inductor_post_to_pre_grad_nodesgetrr#_inductor_pre_grad_node_stack_trace)r stack_tracesr|rrpre_grad_nodes node_namerrrrget_stack_traces`s4       zIRNode.get_stack_tracesTshorten Sequence[str]cCsdt|dd}|rt|dkr|ddd}|s!|gSg}|D]}|d||d7}|d q'|g|S) Nzorigins=r|@=z...zstack_traces = { })rrrappendsplit)rrr|stack_trace_strrrrr common_reprs    zIRNode.common_reprlinesSequence[object] multilinecCsbt|t||}ttt|}|r&td|}t|jd|dSt|jd|dS)Nz, z( z )(r/)rrmaprindentjoinrr)rrrr new_linesrrr str_helpers zIRNode.str_helper torch.dtypecCrrrrrrrr rzIRNode.get_dtypeOptional[torch.dtype]cC z|WStyYdSwr)r NotImplementedErrorrrrrmaybe_get_dtype   zIRNode.maybe_get_dtypeLayoutcCtdt|d)Nz#get_layout() is not implemented by !rrrrrrrrzIRNode.get_layoutOptional[Layout]cCrr)rrrrrrmaybe_get_layoutrzIRNode.maybe_get_layoutr0cC|Sr)rrrrrget_output_speczIRNode.get_output_specOptional[OutputSpec]cCrr)rrrrrrmaybe_get_output_specrzIRNode.maybe_get_output_speccCst|tS)z4True for single tensor output (excludes MultiOutput))rrrrrrrhas_tensor_outputszIRNode.has_tensor_outputSequence[Expr]cCr)Nz!get_size() is not implemented by rrrrrrrrzIRNode.get_sizeOptional[Sequence[_IntLike]]cCrr)rrrrrrmaybe_get_sizerzIRNode.maybe_get_size.Union[_IntLike, sympy.Rel, Sequence[_IntLike]]cCrrrrrrrrJz IRNode.shaper cC t|Sr)rgrrrrr get_numelrzIRNode.get_numelcCtjjt|dSNrrlrrstatically_known_truerEqrrrrris_zero_elementszIRNode.is_zero_elementsr-cCtdt|)a) If the IRNode refers to data which has not been materialized (e.g., it is a Pointwise/Reduction that could potentially have more compute fused into it), realize the IRNode into physical memory, ending the possibility of fusing into it, but allowing, e.g., multiple users to access the data without having to recompute. Check StorageBox.realize for a particularly notable implementation. TODO(ezyang): I think, in principle, every IRNode should have an implementation of this, and most of the time no-op is OK, but you really do have to audit each IRNode for this, so for now, raise an error if it's not implemented. Note that some code in graph.py will catch this thrown error and suppress it with a warning. zrealize NYI on rrrrrrealizeszIRNode.realizeNwriterOptional[IndentedBuffer]cCr)Nzcodegen_reference NYI on rrrrrrcodegen_referencezIRNode.codegen_referenceOptional[torch.device]cCr rrrrrrr!r zIRNode.get_device torch.devicecCs|}|dus J|Srr!rrrrrget_device_or_errors zIRNode.get_device_or_errorcCdSNFrrrrrhas_exceeded_max_readsr zIRNode.has_exceeded_max_reads$Callable[[Sequence[Expr]], OpsValue]cCtt|jrrrrrrrr make_loaderrzIRNode.make_loader Callable[[Sequence[Expr]], Expr]cCrrrrrrr make_indexerrzIRNode.make_indexerrHcCrrrrrrrr@rzIRNode.get_stridecCrr)r@rrrrrrCrzIRNode.maybe_get_stridecCrrrrrrrget_namerzIRNode.get_namecCrr)rrrrrrmaybe_get_namerzIRNode.maybe_get_namecCs(z |tjjvWStyYdSwr)rrlr graph_inputsrrrrris_input_buffers  zIRNode.is_input_buffer threshold Optional[int]cCrrrrrrrrhas_large_inner_fn r zIRNode.has_large_inner_fnusersrcCr rrrrrrr mark_reuse r zIRNode.mark_reusecCr rrrrrr realize_hintr zIRNode.realize_hintcCrrrrrrr unwrap_viewrzIRNode.unwrap_viewcCrrrrrrr freeze_layoutrzIRNode.freeze_layoutrr allow_paddingcCrrrrrr rrrfreeze_layout_with_stride_orderz&IRNode.freeze_layout_with_stride_ordercCrrrrrrrrfreeze_layout_with_fill_orderrz$IRNode.freeze_layout_with_fill_orderrcCrrrrrrrrfreeze_layout_with_same_order rz$IRNode.freeze_layout_with_same_order exact_stridescCrrrrrr rrr freeze_layout_with_exact_strides#r z'IRNode.freeze_layout_with_exact_stridesdependencies.ReadWritescCrrrrrrrget_read_writes(rzIRNode.get_read_writesOrderedSet[Dep]cC |jSrrreadsrrrrr+ zIRNode.get_readscCrr)rrrrrr num_reads.rzIRNode.num_readsrxcCrrrrrrrget_storage_numel1rzIRNode.get_storage_numelrurcCrrrrrurrrget_free_symbol_uses4r zIRNode.get_free_symbol_usescCrrrrrrrget_reduction_type9rzIRNode.get_reduction_typecCrrrrrrrget_reduction_size<rzIRNode.get_reduction_sizecCrrrrrrr is_extern?r zIRNode.is_externcCrrrrrrris_no_opBr zIRNode.is_no_oprcCrrrrrrrconstant_to_deviceErzIRNode.constant_to_devicecCrrrrrrrget_mutation_namesHrzIRNode.get_mutation_namescCrrrrrrrget_operation_nameKrzIRNode.get_operation_namecCrrrrrrrget_inputs_that_alias_outputNrz#IRNode.get_inputs_that_alias_outputcCr rrrrrrrSr z IRNode.dtype)r|rrrrrrr)rrr'r rrrrrr)rr}rrrrT)rrrr)TT)rrrrrrrrrr)rrrr)rrrr0)rrrrrr)rr)rrrr rr-rrrrrrrrrrrrrrrHrrrrrrrrrrrrFrrr rrrrrrrrrHrrrrHr rrrrrrrrrrrxrurrrrrrrrr)Krrr__doc__r:ryr dataclassesfieldr|r~r staticmethod contextlibcontextmanagerrrrrrrrrrrrr rrrrrrrrpropertyrJrrrrr!rrrrr@rCrrrrrrrr r rrrrrrrrr r!r"r#r$r%r&r'rrrrrrrs                                                   c@seZdZd0ddZd1ddZd2d d Zd3d d Zd4ddZd5ddZd5ddZ d6ddZ d7ddZ d8ddZ d9d d!Z d:d#d$Zd;d&d'Z (dd-d.Zd/S)? OperationrrcCs d|_dSroperation_namerrrrrYrzOperation.__post_init__rcCtrrrrrrr!\r zOperation.get_devicercCt|dsJ|jSNr)rrrrrrr_zOperation.get_origin_noder{cCrW)Nr|)rr|rrrr get_originscrYzOperation.get_originsrcCs|jdusJ|jSrrSrrrrr&grYzOperation.get_operation_namercCrrrrrrrr"kr zOperation.is_externcCrrrrrrrr#nr zOperation.is_no_oprcCrUrrVrrrrrqr zOperation.get_read_writesrcCs ||vSr)r)rrrrr is_user_oftrzOperation.is_user_ofrcCr)Ncsrrrrrrrrxrz+Operation.get_read_names..rrrrrrwrzOperation.get_read_namesrcCrrrrrrrrzrzOperation.get_reads list[Buffer]cCrUrrVrrrr get_outputs}r zOperation.get_outputsrcCtSrr9rrrrget_unbacked_symbol_defsrz"Operation.get_unbacked_symbol_defsFrucCr^)a When unbacked_only=True: Returns the unbacked symbols which are required to be in scope in order to successfully perform codegen for this buffer. For example, a buffer that corresponds to an extern kernel call that takes i0 as an argument would return {i0} here. This is used to generate necessary dependencies that ensure we actually bind i0 in codegen before you try to use it. Note that this is NOT transitive; in particular, if this buffer takes in as input another buffer with dynamic shape (e.g., (i0,)), we will not report it here, because you will already have a dependency on that buffer, which will eventually have a dependency on i0 if necessary. When unbacked_only=False: Similar to `unbacked_only=True` but including all free symbols instead of only free unbacked symbols. r9rrrrrszOperation.get_free_symbol_usesrcCr)z Gets extra global memory size needed by this buffer. Some algorithms (e.g. group gemm) may require extra global memory in the generated code. rrrrrrget_workspace_sizeszOperation.get_workspace_sizeNr)r6r+)rr{r;r1rD)rrrrr*rErr\rrr?rHrF)rrrrr!rrZr&r"r#rr[rrr]r_rr`rrrrrRWs"              rRcs2eZdZUded<ded<ded<ded<ed dPdQddZdRddZdSfdd ZdTddZeZ dUddZ dVddZ dWd!d"Z dWd#d$Z edXd)d*ZeejfdYd-d.ZedZd0d1Zd[d3d4ZedTd5d6Zd\d]d:d;ZdPd^d=d>Zd_d@dAZd`dCdDZdadFdGZdWdHdIZdbdKdLZdcdNdOZZ S)dLoopsrrrrCallable[..., Any]inner_fnrHrangesFrurrrcs,tjgfdd|jD|RS)Nc3|]}t|VqdSrr'rertrrrz-Loops.get_free_symbol_uses..)r:unionrfinner_fn_free_symbolsrrrtrrs zLoops.get_free_symbol_usesnamesrrcsFdjjdtjgfdd|DdjgS)N'cs g|] }|dt|qS=)rrrrrrrs z!Loops._to_str.. origin_node=)rrrrr inner_fn_strr)rrmrrr_to_strs z Loops._to_strrcstdSr)superrr __class__rrrrzLoops.__post_init__cC |dS)Nrfrtrrrr__str__rz Loops.__str__rcCrrrrrrrr!rzLoops.get_devicercCrrrrrrrrrzLoops.get_origin_nodercCrrryrrrrrrzLoops.get_sizecCrrryrrrrget_pointwise_sizerzLoops.get_pointwise_sizerr r'Union[TensorBox, ShapeAsConstantBuffer]cOsN|dd}|dd}||i|}|d||d|p |jt|S)Nrr~)poprr~rcreate)clsrrrtbrrrrrs    z Loops.creater|r>csfddt|DS)Ncs*g|]\}}|dkrtjjnt|qSr?)rSZerorf)rnrr{rrrz Loops._index..)r)rfr|rr{r_indexs z Loops._indexrNc Cstt}t|2ttdd|j|| WdWdS1s0wYWddS1s@wYdSNallow_indexingT) rMrl MockHandlerset_ops_handlerrrrre inner_fn_argsgetvalue)r opcounterrrrinner_fn_opcounts  RzLoops.inner_fn_opcountSequence[Sequence[_IntLike]]cCs||jfSr)rrfrrrrrrzLoops.inner_fn_argscCstjj|jg|RSr)rlKernelFormatterHandler ir_to_stringrerrrrrrss zLoops.inner_fn_strNrrcCs&|durd}t|tj}|j|kSr)maxr@realize_opcount_thresholdrnum_opsrrrrrs zLoops.has_large_inner_fnOrderedSet[Symbol]cCs||j}t|j||dSNrt)rrfrHre)rrurrrrrls zLoops.inner_fn_free_symbolsrcCsvttdd*|r t|||jWdSt||jWdS1s4wYdSr) rrrr rJrrr!rrrrrrs$zLoops.get_readsrcCt|jSr)r:r read_buffersrrrrrrzLoops.get_read_namesrcCrr)rrrrrrrrrzLoops.num_readscCr)Nz+get_reduction_size() is not implemented by rrrrrrr!zLoops.get_reduction_sizer-cCr)Nz+get_reduction_type() is not implemented by rrrrrrr rzLoops.get_reduction_typercCr)Nz+constant_to_device() is not implemented by rrrrrrr$"rzLoops.constant_to_devicer?rH)rmrrrr)r;r6r+r2)rr rr rr~)rfrHr|r>rr)rrNrrrr<rurrrrEr*rFr4rI)!rrrrrWrrtrr{__repr__r!rrr} classmethodrrNr>INDEXrrVrrrsrrlrrrr!r r$ __classcell__rrrvrrcsB                   rcrUnion[Expr, Sequence[Expr]]rrrkcCs"|jr ttd|Std|S)Nnanr)is_floating_pointrjconstantfloat)rrrrr nop_loader_fn(s rc@s>eZdZdddZdddZdd d ZdddZdddZdS) PointwiserrcCs|r tt|jdS|jSNr)rr rrrerrrrr1szPointwise.make_loaderSequence[sympy.Expr]cCsgSrrrrrrr!8r zPointwise.get_reduction_sizer-cCr rrrrrrr ;r zPointwise.get_reduction_type output_nameindexer!Callable[[Sequence[Expr]], Never]varsrrcCs"|}t|p d||||SNunnamed)rrjstorerrrrloaderrrr store_output>szPointwise.store_outputrrrcCs.|}ttd||}t||j||jdSFMove this to a given device. Requires that all reads are to constants.override_devicerrrerf)rrrConstantBufferrrrfrrrrrrr$GszPointwise.constant_to_deviceNr8)rrr4)rr-rrrrrrrI)rrrrr!r rr$rrrrr/s      rc@s6eZdZUded<dZded<dd d ZdddZdS)Scatterroutput_indexerNrP scatter_moderrrrcCs6|}ttd||}t||j||j|j|jdS)rr)rrrerfrr) rrrrrrrfrrrrrrr$XszScatter.constant_to_devicerr-rrrrr cCs6|}|dur d}tj|||||||jdS)Nr)mode)rrjrrrrrrrres zScatter.store_outputrI)rr-rrrrrr )rrrrrr$rrrrrrSs    r logical_ormaximumminimummulr bitwise_xor)anyrminprodsumxor_sumz"dict[str, Callable[..., OpsValue]]REDUCTION_COMBINE_FNreduction_typearg_break_ties_leftCallable[..., object]csRtvrtSdvrdfdd }|Sd kr"dd d }|Std )Nargmaxargminatuple[object, object]brtuple[OpsValue, OpsValue]c s|\}}|\}}dkrt||}nt||}t||}trCt||}t||} t|t|| }t|t|| }rKt||nt||} t|t|| }t|||t|||fS)Nr) rjltgteqr+ner logical_andwhere) rra_valuea_indexb_valueb_indexmaskequala_isnanb_isnantierrrrrargmax_combine_fns&       z3get_reduction_combine_fn..argmax_combine_fnwelford_combine#tuple[OpsValue, OpsValue, OpsValue]c SsR|\}}}|\}}}||}||} || } ||| |||||| | fSrr) rra_meana_m2a_weightb_meanb_m2b_weightdelta new_weight w2_over_wrrrwelford_combine_fns   z4get_reduction_combine_fn..welford_combine_fnzunknown reduction_type=)rrrrrr)rrrrrr)rr)rrrrrrrrget_reduction_combine_fns rcs\eZdZUded<ded<ded<ded<dkd d ZeZeddldmfdd ZdnddZdoddZ dpd d!Z dqd#d$Z drd&d'Z dldmd(d)Z dsd-d.Ze /dtdud:d;Zedvd>d?Zeejd/fdwdBdCZedxdFdGZedxdHdIZedydMdNZedzdPdQZe /dtd{dWdXZed|d`daZed}dedfZe /dtd~dgdhZeddidjZZS) ReductionrHreduction_rangesrOrr src_dtyperSreduction_hintrrcCrx)N)rfrrrzrrrrr{rzReduction.__str__Frurrcs(ttjfdd|jDBS)Nc3rgrr'rhrtrrrrjz1Reduction.get_free_symbol_uses..)rurr:rkrrrvrtrrszReduction.get_free_symbol_usesrcCrr)rrrrrr!rzReduction.get_reduction_sizer-cCrr)rrrrrr rzReduction.get_reduction_typerrrrreduction_varsSequence[Symbol]rc Cs8t|j|j|j|||}t|pd|||dSr)rj reductionrrrrestore_reduction)rrrrrr'rrrrs zReduction.store_reductionrcCt|jt|jSr)rrfrrrrr index_lengthrzReduction.index_lengthSequence[Sequence[Expr]]cCs$||j}||jtj}||fSr)rrfrr>R0_INDEX)rrrindexrrrrs zReduction.inner_fn_argscCs.||j}||jtj}t|j|||dSr)rrfrr>rrHre)rrurrrrrrls  zReduction.inner_fn_free_symbolsrrrc Cs>|}ttd||}t||j||j|j|j|j t j dS)rrrrrerfrrrr) rrrrrrrfrrrrSDEFAULTrrrrr$szReduction.constant_to_deviceN dst_dtypereCallable[_P, OpsValue]rf%Union[ReductionType, Literal['scan']]reduction_numelr input_nodertuple[ReductionHint, _IntLike]c " Cstjj|} tjjt|} |dkp#tj|tj o#|dvo#tj } t | r,t | s1t j dfSt |} | j} d}| rRtjtjj|dd}tjtjj|dd}ndd d }|}| dkr|| | }|dkrkt j|fS|durt|trttddt|\}}Wdn1swY|dur|durtjjt||}| |krtd|||||t jdfSt j|fS| |ks| | ddkrt j dfSt||||||dkr|nd|t j d}ddd}||\}}|r||\}}t|dkrt j dfSt |!|"\\}}}d}d}|D].}tjj#||}tjj$||t%|&} t'dd| D}!|!r;|d7}q|d7}q||krNt j|| | fSt j(|| | fS)Nscanrr? T)inner_reductionFreduction_numel_hintr numel_hintrcSrNr?r)r r rrrinner_reduction_splits-z4Reduction.num_splits..inner_reduction_splitsrzUse previous IRNode's range and reduction_ranges instead of split. current ranges: %s, current reduction ranges: %s, current split: %d, new ranges: %s, new reduction ranges: %srBrrrrrtuple[Sequence[Expr], bool]c s|}|dus Jtdt|||d|d}|}|jdus%Jdd|jD}g}d}t|jdddD]7t fd d |Drq| j j t jjvrqt jjj }t|jd d}|t|jd d|krqd }q:||fS) Nrrrrr[rZcSs&g|]}t|trt|tjs|qSr)rr rNumberrrrrrrvs zBReduction.num_splits..get_read_indices..FcSrrrrrrr}z@Reduction.num_splits..get_read_indices..keyc3s|] }|jjvVqdSr)rr1rmdrrr~zAReduction.num_splits..get_read_indices..rT)r!rrr rr range_varssortedrrrrrrlrname_to_bufferrr[ decide_layout) rrcb read_writesrindiceschangedbuforiginal_striderrrget_read_indiceses:   z.Reduction.num_splits..get_read_indicesrcss|]}|dkVqdSr?Nrrrrrrz'Reduction.num_splits..)r rr rrr)rrrr))rlrrrMrg has_featurerBREDUCE_TO_SINGLE_ELEMENTr@split_reductionsrrSrrRrmulti_processor_count functoolsr choicesreduction_split_factorINNERrrrrrrIlogrrrrAindex_vars_squeezerr!simplify_with_ranges stride_hintsrkeysrOUTER)"rrrrerfrrrrr r  should_splitpropsnum_smmin_elements_per_threadrouter_reduction_splitsr new_rangesnew_reduction_rangesextracted_numel_hintrr&r"r#rrranges1 num_outer num_innerrjrSouterrrr num_splitss            #        zReduction.num_splits.fn..cSsg|]}t|qSr)rrrrrrrrz>Reduction._unroll_reduction_fn..fn..)r-reduce itertoolsproductr) combine_fnrrGrrrs  z*Reduction._unroll_reduction_fn..fnrrrrcs*dd|D}||t|tjfS)NcSg|]}t|qSrrexpandrrrrrrzDReduction._unroll_reduction_fn..value_fn..)rj index_exprr#int64rr) flatten_indexrerrrGsz0Reduction._unroll_reduction_fn..value_fncs |dSr rr)rrrrs z0Reduction._unroll_reduction_fn..N)rrHrr )rrHrrHrr)rlrr guard_int_seqr_fixed_indexerrr)rerrrr)rLrTrrerrGr_unroll_reduction_fns   zReduction._unroll_reduction_fnrdr~c  s$tjjtdkrDdfdd } | d| d| d| dd vs0Jd dfdd } tj||| t|dSdkredvrTdfdd } ndfdd } tj|| |dSt t rtjj t j krt|dkst|jrtj||||dS||||| \} }dfdd }||}|tjkr| }|dkr| dusJttddt| \}}Wdn1swY|dusJ|dusJ||||||| S|dkr||||||| Stt||||dS)NrvalrrUnion[bool, float, int]csTtjkr t|Sjrt|tsJt|t|St|ts&Jt|t|Sr) r#rrrrrrrrrXrrrpy_cnsts z!Reduction.create..py_cnstr?)rrrrz* not supported for zero-dimension tensors!rrrkcstSrrjrr)rrrtypes_to_initsrrconst_fnz"Reduction.create..const_fnrrMcs tdSrr]rr[rrr rzReduction.create..fncddD}||S)NcSg|]}tjjqSrrrrrrrrrrz0Reduction.create..fn..rrreduction_index)rerrrr rcs$tr|S|dkrt|tjS|Sr )rrr@ min_num_split)r)rrr_maybe_increase_split6s  z/Reduction.create.._maybe_increase_splitrBrTr)rXrrrY)rrrrk)rrrr)rlrrsimplifyrgr5rrrrr!r?r@unroll_reductions_thresholdrcrrWrDrSrrrrrI!create_multilayer_existing_rangescreate_multilayerrr)rrrrrerfrrrrr\r_rhintrrhr<r=r)rrerrrr^rrs            zReduction.creater#Union[_NumLike, Sequence[_NumLike]]cCs|dvrt|r tdSt|rdSt|jS|dvr0t|r$tdSt|r*dSt|jSt|r6dnd}t|r>dnd}|||||||f|||ftd|fd |S) N)rrz-infF)rrinfTrr?)rrrrwelford_reduceronline_softmax_reduce)r+rr*r#iinforr)rrzeroonerrrdefault_accumulatorvs0   zReduction.default_accumulatorcCs|dkrdSt||S)Nrpr)rrurrrrr default_values zReduction.default_valuerrxr cCsP|dkr|S|dkr|dkr|tjkrtjS|dkr&|dkr&|tjkr&tjS|S)NrBi)rSr6 OUTER_TINY)rr rrrr_multilayer_second_step_hints z&Reduction._multilayer_second_step_hintrcCs|durdStjj||sdS|zt|Wn ty&YdSw|}t |ddD]\}}tjj|drC|Sq3dS)z If we are reducing over the full tensor, and it is non-dense in the last dimension, reindex so we reduce over the dense dimension. initially just handle complete reduction case NrBr?) rlrrrLrrr\rr@r)rrrrSrrrrr$check_for_split_dense_dim_reindexings$   z.Reduction.check_for_split_dense_dim_reindexingrr block_sizedefaultrc sT||}t|g|tjjt|d d fdd } | S) Nrrrrerrkcsl|\}|^}||dfdd }r3t}tt|t|}t||S|S)NrrkcsgSrrr)r"r new_indexrrrbodyr`zCReduction._multilayer_wrap_loader..wrapper_fn..body)rrk)r]rjrrQmasked)rrereduction_blockr index_dtyperr}r~r need_maskrr)r"rr wrapper_fns    z5Reduction._multilayer_wrap_loader..wrapper_fn)rrrerrrk) r|Viewdynamic_reshape_indexerrlrrrrr) rrrrrr}r~r dense_indexrrrr_multilayer_wrap_loaders z!Reduction._multilayer_wrap_loader4Callable[[Sequence[Expr], Sequence[Expr]], OpsValue]original_rangesoriginal_reduction_rangesr<Sequence[Integer]r=@Callable[[Sequence[sympy.Expr], Sequence[sympy.Expr]], OpsValue]csNtddDsJdt|t|t|d fd d }|S) Ncss|]}|dkVqdSr'rrrrrrr(zDReduction._multilayer_wrap_loader_existing_ranges..z8Only enabled for numel_hint == 1, found original_ranges= merged_indexrnew_reduction_indexrrkcs:|dt}|td}|t|t|Sr)rr)rr original_idxrrrrrrr s zEReduction._multilayer_wrap_loader_existing_ranges..wrapper_fn)rrrrrrk)rrrr)rrrrr<r=rrrr'_multilayer_wrap_loader_existing_rangess  z1Reduction._multilayer_wrap_loader_existing_rangesrrs list[Integer]c  s|tjtjfvr |ntj} t|| ||||| | } | | d fdd }tj j t |}| | || } ||dt|ksDJtt|||||t|d| || d S) a Break a large reduction up into multiple smaller reductions recursively rrHrerrkcsg||Srrrdintermediate_loaderrrintermediate_fn>sz;Reduction.create_multilayer_helper..intermediate_fnNr)rrHrerHrrk)r#float16bfloat16rrrrrrlrrrrgr{rr)rrrrrrrr<r=rrrintermediate_dtype intermediaterr rrrcreate_multilayer_helpersD z"Reduction.create_multilayer_helperc  Csdt|} t| |d|} |||} |||| || | | }|||||||g||| g||| S)rr?)rgr<rwrr)rrrrrerfrrrrrrr}r~rrrrrlVs2   zReduction.create_multilayerc Cs8||||||} ||||| ||g|||| d| S)rrB)rr) rrrrrerrr<r=rrrrrrrks( z+Reduction.create_multilayer_existing_rangesr;r?rr2r4) rr-rrrrrrrrrFrrrIr)rrrrrrrerrfrHrrHrrrr rrrr) rerErrHrrrrrrF)rrrrrrrerdrfrrrrrOrrSrrrr~rrrrrrn)rrxr rrrSrrS)rrxrrrr)rrrrHrrxrrxr}rxr~rnrrrr) rrrrrrr<rr=rrr)rrrrrrrrdrrrrr<rsr=rrrOrrxrrSrr~)rrrrrrrerdrfrrrrrOrrxrrSrrrr~)rrrrrrrerdrrrrr<rr=rrrOrrSrr~) rrrrr{rrWrr!r rrrrlr$rNrDrWrrSrrrurwr{r|rrrrlrkrrrrvrrs^          # + "     *  ? -rrrOptional[Sequence[int]]r^r rcsdfdd }|S) 1A closure containing math to read a given elementrrrrcsddur t|tksJt|tksJ}t|D]\}}}|dkr/|||}q |Sr )rr)rresultrstszr^rrrrrs z_fixed_indexer..indexerN)rrrrr)rrr^rrrrrVs rV INNER_FN_TYcs2eZdZUded<dfdd Zd ddZZS)!MultiOutputReductionr output_indexrrrr inner_fns)Union[INNER_FN_TY, Sequence[INNER_FN_TY]]rfrrrrOrrrSc sXtrftdkrd} nd fdd } tj||| |||||d | |_dS) Nr?rrr reduction_idxrtuple[OpsValue, ...]cstfddDS)Nc3s|]}|VqdSrrrrrrrrrrjz@MultiOutputReduction.__init__..loader..)rrrrrrsz-MultiOutputReduction.__init__..loaderr)rrrrrr)callablerru__init__r) rrrrrfrrrrrrrvrrrs     zMultiOutputReduction.__init__rr-rrrrrrrr c CsXt|j|j|j|||}t|ttfsJt |||j }t |p&d|||Sr) rjrrrrrerrrrrr)rrrrrrr'rrrrs  z$MultiOutputReduction.store_reduction)rrrrrrrfrrrrrOrrrrSrr) rr-rrrrrrrr )rrrrrrrrrrvrrs %rc@s"eZdZeejdfdddZdS)OnlineSoftmaxReductionNrrrrrrerdrfrr num_outputrrrSrrr1Sequence[Union[TensorBox, ShapeAsConstantBuffer]]c s<tfddt|D} | D]} | q| S)z> Create the reduction disregarding splitting. c 3s.|]}ttd| VqdS)rqN)rrrr output_idxrrrerfrrrrrr s" z0OnlineSoftmaxReduction.create..)rrr) rrrrrerfrrrrresultsr&rrrrs  zOnlineSoftmaxReduction.create)rrrrrrrerdrfrrrrrrrSrrrr)rrrrrSrrrrrrrs  rc@s<eZdZeejfdddZedddZedddZ dS)WelfordReductionrrrrrSequence[Callable[..., Any]]rfrrrrOrrSrrc s6dvsJtjjt}dfdd } |dkr-| d} | d} | d} | | | fS|d krWdfd d dkrLd| d| d fStfddDStjd|d\} }tj kro| |d kr| |Sfddt dD}|D]}| q|S)N)rprrXrrr~cs&dfdd }tj|tdS) Nrrrrkcs tSrr]r)rrXrrre3sz8WelfordReduction.create..const..inner_fnrrrrrkrrr)rXre)rrrfrZrconst2sz&WelfordReduction.create..constrr?rrcs&dfdd }tj|tdS) Nrrrrkcra)NcSrbrrcrrrrrLrzKWelfordReduction.create..copy..inner_fn..r)rre)rrrrreKrfz7WelfordReduction.create..copy..inner_fnrrr)rre)rrrfrrrcopyHsz%WelfordReduction.create..copyrpc3|]}|VqdSrrr)rrrrYr(z*WelfordReduction.create..)rrcs*g|]}tt| qSr)rrrr)rrrrfrrrrrrs z+WelfordReduction.create..r)rXrrr~)rrrr~) rlrrrirgrrrDrSrrlrr)rrrrrfrrrrrmeanm2weightrmrrr&r)rrrrrfrrrrr#sT     zWelfordReduction.createrrncCr)N)rrrrrvrrrrwrzWelfordReduction.default_valuerrxc  s$ttjjtd } | r9|dkr9dfd d } j||dt| dd t| d d f|d|dSt d t |t fdd|Dg|g||} | D]} | qadddtjjt|} | |}t |t fdd| D|gd|S)rrrrrrr'rrrkcs t|Srr])rrr'rrrrs z4WelfordReduction.create_multilayer..constantr*r?)rrrrfrrrrc 3s&|]}j|ddVqdS)r)r~N)r)rr)r}rrrrrrrs  z5WelfordReduction.create_multilayer..rrerrcSs|g||Srr)rrerrrrintermediate_loader_fnszBWelfordReduction.create_multilayer..intermediate_loader_fnc3s |] }t|dVqdS)rN)r rr)rrrrs  N)rrrrr'rrrk)rrrerrrrrk)rgrlrrrrrrlr r<rrrrrr{)rrrrrfrrrrrr intermediatesrr r)r}rrrrrrrrlsb        z"WelfordReduction.create_multilayerN)rrrrrrrfrrrrrOrrSrrr)rrrrrrrfrrrrrOrrxrrSrr) rrrrrSrrrNrwrlrrrrr"s x rcseZdZUded<ded<ded<ded<ded <d ed <d ed <ded<eddIdJfdd ZdKfdd ZdLd#d$ZdMd%d&ZdNd'd(Z dNd)d*Z dNd+d,Z dOd-d.Z dPd0d1Z dIdJd2d3Zeejfd4d5dQd=d>ZedRdGdHZZS)SScanr scan_rangesr=Callable[[tuple[Any, ...], tuple[Any, ...]], tuple[Any, ...]]rLzFCallable[[Sequence[_IntLike], Sequence[_IntLike]], Sequence[_IntLike]]rrSrrrtuple[torch.dtype, ...]dtypestuple[Callable[..., Any], ...]rFrurrrcDttjfdd|jDBtjfdd|jDBS)Nc3rgrr'rhrtrrr rjz,Scan.get_free_symbol_uses..c3rgrr'rhrtrrr rj)rurr:rkrrrrvrtrr s zScan.get_free_symbol_usesrc0t|jt|jt|jksJtdSr)rrfrrrurrrvrrr "zScan.__post_init__rr-r%Callable[[Sequence[_IntLike]], Never]rr scan_varsrr csR|||tfdd|jD}t|j|j|}t|p d|||jS)Nc3|]}|VqdSrrrrerrrr) r(z'Scan.store_reduction..r) rrrrjrrrLrr)rrrrrrrrrrr! s zScan.store_reductioncCr)Ncustomrrrrrr / zScan.get_reduction_typecCrr)rrrrrr!3 rzScan.get_reduction_sizecCrrrrrrrr6 rz Scan.get_sizecCrrryrrrrr}9 rzScan.get_pointwise_sizecCrr)rrfrrrrrr< rzScan.index_lengthrcC.||j}||jtj}|||}|fSr)rrfrr>rrrrrrrrrr?   zScan.inner_fn_argscC8||j}||jtj}|||}t|j||dSr)rrfrr>rrrHrerrurrrrrrrlE   zScan.inner_fn_free_symbolsT)can_fallback_to_atenrr+tuple[Callable[[Sequence[Expr]], Any], ...]axisrr;Sequence[Optional[Union[TensorBox, ShapeAsConstantBuffer]]]c sg d dd g tjtjs$dgtStdkr9tjtjs9dgtStjj} | t } ttksNJ| t | drg fddt tDS|jdd | d\} t | dkrtjjduptotdkotdk} | s|rdgtSd} nt d fd d  f ddt tD}|D]}|q|S)Nr?c&g|]}tj||dqSrrrrrrrrrrrrl zScan.create..r)rrrerpointwise_rangesrrL scan_numelz3.3.0rr scan_indexrrscHt|tks Jt|tksJg|d||dSrr)rr)rrrrrr  zScan.create..reindexcsBg|]}t d|| |d qS)) rrrrerrrfrrLrrrr)rrr) rLrrrrrrrr scan_typerrrr s* )rrrrrrs)rlrr)rBSCANrTUPLE_REDUCTIONrrirgrrLerrDrr#versionhip has_tritontriton_version SplitScanr)rrrrrrrLrrrrrrDsupports_splitrrr) rrLrrrrrrrrrrrrK sV         z Scan.createrrrerrrr rc s*d fdd } tj|||| ||d|d S) Nrrrrrkcs$g|d||dSrrrrrerrr s$z#Scan.num_splits..wrapper_fnr)rrrrerfrrr)rrrrrrk)rrD) rrrrerrrrLrrrrrrD s zScan.num_splitsr?rr)) rr-rrrrrrrr r4r2rFr)rrrrrrrrrrrLrrrSrrrr rr)rrrrrerrrrrrrrLrrr rr)rrrrrWrrrr r!rr}rrrlrrSrrrDrrrrvrr s6           arc@s eZdZdS)rN)rrrrrrrr srcseZdZUded<ded<ded<ded<ded <d ed <d ed <ded<ded<edd;d<fdd Zd=fdd Zd>d"d#Zd?d$d%Zd@d&d'Z d@d(d)Z d@d*d+Z dAd,d-Z dBd/d0Z d;d.c3rgrr'rhrtrrr rj)rurr:rkrrrrvrtrr s zSort.get_free_symbol_usesrcrr)rrfrrrurrrvrrr rzSort.__post_init__rr-rrrrrr csV|||tfdd|jD}t|j||j|j}t|p"d|||j S)Nc3rrrrrrrr r(z'Sort.store_reduction..r) rrrrjsortrrrrr)rrrrrrrrrrr s zSort.store_reductioncCr)Nrrrrrrr  r zSort.get_reduction_typecCrr)rrrrrr! rzSort.get_reduction_sizecCrrrrrrrr rz Sort.get_sizecCrrryrrrrr} rzSort.get_pointwise_sizecCrr)rrfrrrrrr rzSort.index_lengthrcCrr)rrfrr>rrrrrrr rzSort.inner_fn_argscCrr)rrfrr>rrrHrerrrrrl rzSort.inner_fn_free_symbolsrr'tuple[Callable[[list[Expr]], Any], ...]rrrc  s*g d dd g tjtjs$dgtStjj} | t } d} t j j o=| t | | } | sGdgtSttksQJ| t | drj fddttDSd  fd d  f d dttD}|D]}|q|S)Nr?rxcrrrrrrrr> rzSort.create..rr sort_indexrrscrrr)rr)rrrrrrH rzSort.create..reindexcsDg|]}ttd|| | d qS)) rrrrerrrfrrrrrrr)rrrr) rrrrrrrrrrrrrrM s,)rrrrrrs)rlrr)rBSORTrrrirgr@r6persistent_reductionsrrrrr)rrrrrrrrrrr sort_numel max_rblockis_persistent_kernelrrr) rrrrrrrrrrrrrr s0     z Sort.creater?rr)) rr-rrrrrrrr r4r2rFr)rrrrrrrrrrrrrrrrSrr rr)rrrrrWrrrr r!rr}rrrlrrSrrrrrrvrr s0          rcCs(z t|ddWdStyYdSw)NFfreezeT)r\rrrrrri s   rcCs@zt|dd\}}|r||WStyYdSwNFr)r\should_pad_strides pad_strides is_contiguousr)r_bufferr[rrr is_contiguous_storage_and_layoutq s  r rwant_contiguous stride_order'Optional[Sequence[Union[int, Integer]]]r rtuple[StorageBox, Layout]c Cst|trt|j|||||dSt|tr)t|j|||||d\}}||jfSt|trc|r[|r?||s>Jn|durK|j ||dn|durW|j ||dn| t||fSt|t rvt|j|d\}}||j fSt)z Try to simplify x into a StorageBox and a Layout. allow_padding only affect how we apply stride_order. When allow_padding is True, we have the freedom to add padding when applying the stride_order. rr rr rNr r)rrr\rZ StorageBoxrBufferr r r rrr`r[r) rrr rr rrr[bufferrrrr\} sR      r\cCs2zt|dd\}}||WStyYdSwr)r\is_stride_orderedr)rrr r[rrr"is_stride_order_storage_and_layout s   rrcCsrt|ttfr t|jSt|tr*|j}tjj |j t |j t }t|jp)|St|tr7|tjjvSdSr)rrr is_unalignedrZr`r[rlrrstatically_known_multiple_ofr^r^rr`rrunaligned_buffers)rr[has_unaligned_layoutrrrr s   rc@seZdZUded<eddEdFdd ZdGd d ZdHddZdIddZe dJddZ dKddZ dLddZ dMddZ dNd d!ZdOd#d$ZdPd(d)ZdQd*d+ZdRd-d.ZdSd/d0ZdTd2d3ZdQd4d5ZdQd6d7ZdUd9d:ZdVdd?ZdXdBdCZdDS)YBaseViewrrZFrurrrcC |j|SrrZrrrrrr  zBaseView.get_free_symbol_uses*Callable[[Sequence[Expr]], Sequence[Expr]]cCstd|)Nzmake_reindexer NYI on rVrrrrmake_reindexer rzBaseView.make_reindexerrc&|j|dfdd }|S)Nrrrr crrrrinnerrrrr rz&BaseView.make_indexer..indexer)rrrr )rZrr!rrrr#rr  zBaseView.make_indexerrcr")Nrrrrkcrrrrr#rrr rz$BaseView.make_loader..loaderr)rZrr!rrrr#rr r&zBaseView.make_loaderrcC |jSr)rZr rrrrr  zBaseView.dtypercCr(rrZrrrrrr rzBaseView.get_layoutrcCr(rrZr!rrrrr! rzBaseView.get_devicercCr rrrrrrr r zBaseView.get_origin_nodercCr(rrZrrrrrr rzBaseView.get_namercCrrrrrrrr} rzBaseView.get_pointwise_sizerrrcCrrrZrrrrrr rzBaseView.mark_reusecCr(rrZrrrrrr rzBaseView.has_exceeded_max_readsr-cCr(rrZrrrrrr rzBaseView.realizecCs|jdSrrZrrrrrr rzBaseView.realize_hintrxcCr(rrZrrrrrr rzBaseView.get_storage_numelcCr(rrZr"rrrrr" rzBaseView.is_externcCs$t|jts Jt|j|jSr)rrZrris_module_bufferrrrrr3 s zBaseView.is_module_bufferrcCr(rrZrrrrrr rzBaseView.get_read_namesrcCsFttddt||jWdS1swYdSr)rrrrJrrrrrrrr s$zBaseView.get_readscCs"|}t|tr|j}t|ts|Sr)rrrZ)rrrrrr% s   zBaseView.unwrap_viewrrcCs2|}ttd||}t||||dSr)rrrrrr rrrrrr$+ szBaseView.constant_to_deviceNr?rrr r9r8r.r/r6r+r;r2r=r1r4r)rGr*rEr>rI)rrrrrWrr!rrrQrrr!rrr}rrrrrr"r3rrrr$rrrrr s2                    rc@sFeZdZUded<eddd Zedd d Zdd dZdddZ dS)rrrrrnew_sizerHrcCstjj}dd|D}|}dgt|t|t|}t|t|ks(Jtt|D]:}||dkrE||dus>J||||<q.||dusTtjj||rUq.|j||||dddkshJdq.|S)zReplace `-1` with correct sizescSrNrrOrrrrr? rz.ExpandView._normalize_size..NrBrfallbackzKBroadcast failed in ExpandView({x.get_size()}, {new_size}) on dimension {i}) rlrrrrrris_size_one_or_falser)rr6rold_sizerrrr_normalize_size; s"  zExpandView._normalize_sizerc Cs|||}t|rWt|\}}t|t|j}|dksJtjjg|}t|j |jD]\}}| t j j |s<|ntjjq-t|j|jt|||j|j} t|| dSt||dS)NrrY)rZr)r;rr\rrrrrrrrrlrrr9r]rrrr^r_r`r) rrr6rarbskiprcrrrdrrrrV s,      zExpandView.createcCrrrrrrrrq rzExpandView.get_sizer cs6|}|jt|tdfdd }|S)NrrrcsRt|d}t|tksJttD]}|dkr&tjj||<q|Sr )rrrrrr)rractualr<rrr{ s  z*ExpandView.make_reindexer..reindexrrrr)rrZr)rtargetrrr=rr!t s   zExpandView.make_reindexerN)rrr6rHrrH)rrr6rHrrr2r5) rrrrrNr;rrrr!rrrrr7 s    rc@sFeZdZUded<eddd Zedd d ZdddZdddZdS) PermuteViewrsdimsrrrrrcs||}t|ttt|ksJt|r>t|\}tjjfdd|Dfdd|Dj j }t ||dSt ||dS)Ncg|]}j|qSrrrrbrrr rz&PermuteView.create..crCrrrrDrrr rrY)rZrB) _map_neg_dimsr:rrrr\r]rrr^r_r`rA)rrrBrardrrDrr s    zPermuteView.create list[int]csfddDS)Ncs$g|]}|dkr |nt|qSrr)rrNrBrrr s$z-PermuteView._map_neg_dims..r)rrBrrHrrF zPermuteView._map_neg_dimsrcsDt||jttt|jksJ|jfdd|jDS)Ncrrrrrrrr rz(PermuteView.get_size..)r:rFrBrrrZrrrrrr s   zPermuteView.get_sizer cs`ddt|jDfddtt|jDtttt|jks'Jd fdd }|S) NcSrrr)rrrBrrrr rz.PermuteView.make_reindexer..crrrrinvrrr rz.PermuteView.make_reindexer..rrrcsfddDS)Ncrrrrrrrr rz?PermuteView.make_reindexer..reindex..rrrJrrr z+PermuteView.make_reindexer..reindexr?)rrBrrr:)rrrrJrr! s zPermuteView.make_reindexerN)rrrBrrr)rBrrrGr2r5) rrrrrrrFrr!rrrrrA s    rAc@s8eZdZeddddd Zedd dZdddZdS) SqueezeViewNrNrrrNrrc sFt|rwt|\}}g}g}dur*ttsJtdkr(t|jks*Jtt|j|j D]0\}\}} durL|dkrK| || | q3|kr[| || | q3|dkscJdq3t |j |j |||j|j} t|| dSdurt|dd|DS|dksJt|fddt|DS)Nrr?zexpected squeezed size to be 1rYcSg|]}|dkr|qSrrrrrrr rz&SqueezeView.create..csg|] \}}|kr|qSrrrrrrNrrr )rr\rrrrrrrrrr]rrr^r_r`rrr) rrrNrarbr6rcrrrrdrrNrr s<      "zSqueezeView.createrr9tuple[list[int], Callable[[Sequence[Expr]], tuple[Expr]]]cs@dd|D}ddt|Dt|d fdd }||fS) NcSrOrrrrrrr rz(SqueezeView.squeezer..cSsg|] \}}|dkr|qSrrrPrrrr rQrrr tuple[Expr]csTt|tksJ|dtjjg}t|D]\}}|||<qt|S)N )rrrrrr)rrrrlengthnot_onerrr s " z%SqueezeView.squeezer..reindex)rrrrS)rr)rr6rrrUrsqueezer s zSqueezeView.squeezerrZr rcCstd)Nzuse SqueezeView.create())AssertionError)rrZrrrr rzSqueezeView.__init__)rrrNrrr)rrrrR)rZr rr)rrrrrrNrXrrrrrrM s & rMc@sXeZdZUded<ded<dddZdd d Zdd d ZeZedddZ dddZ dS) GenericViewrrr rrcCrr)rrrrrr! szGenericView.make_reindexerrcCsBddtt|jD}t||}ddtt|d|S)NcSg|]}ttj|qSr)rfr>rrrrrrr  z+GenericView.reindex_str..zlambda , r.)rrrrrrrr)r index_old index_newrrr reindex_str s  zGenericView.reindex_strcCs$||jd|jd|gS)Nsize=zreindex=)rrZrrarrrrr{ szGenericView.__str__rrr6rcCs||t||dS)NrZrr)r)rrr6rrrrr szGenericView.createcCrrrrrrrr rzGenericView.get_sizeNr5r;)rrr6rrr rrr2) rrrrr!rar{rrrrrrrrrZ s     rZc@sZeZdZedddZedd d ZedddZe ddddZe dd ddZ dS)!rrr rrcCs<t|}t|}tjjjj}|t|dr||}|Sr)rrPrlrrr evaluate_exprLt)rrrdrrrhandle_negative_index s   zView.handle_negative_indexrrr6rc s t|ts Jt||||\}tjj|r|Sd}t t dks1t t |dkr3d}d|vrGd fdd }||t ||d St |sM|rv|rXt |sXt |}t|dd \}}t|j|j|t||j|j}t||d S||}||t ||d S)NFrTrr rtuple[int, ...]cstdgtSr)rrrr:rr fake_reindex: rz!View.create..fake_reindexrc)r rY)rr rrg)rrrresolve_negative_sizerrlrrstatically_known_list_equalsrr2rr rrequire_contiguousr\r]rrrrr^r_r`r) rrr6unbacked_symbols_in_sizesrirarbrdrrrhrr( s4     z View.creater:tuple[list[Expr], list[Expr]]cCsdd|D}dd|D}t|}tt|D]}||dkr3tjj||<tt|t|||<nqtj j t|t|||fS)NcSg|] }tjj|qSrrlrrrirHrrrrZ z.View.resolve_negative_size..cSrorrprHrrrr[ rqrB) rrrrrOner;rgrlrr check_equals)r:r6rrrrrjV s  zView.resolve_negative_sizeNrH dense_dimrrc CsZz ||||}W|Sttfy,t|g}|||}|||}t||}Y|Swr)_dynamic_reshape_indexerrY IndexErrorrgr)rr:r6rtrflatrrrrrrg s    zView.dynamic_reshape_indexerr cstjjj}ddtt|Dtt|}t|}|duo,|t|dko,t|dk}|r?|dus5J||}| |g|r |r |}|\} } |dkrd t j j | | | fn| dkrn| |n|| ||kr | tjj | |n|| ||kr|| ||kr|\} } | | | } | | } || ||ks | tjj | |nL|| ||krt j j} |} t| | || |} || ||kr|} t| | || |} ||}|| ||kstjj | |nt|r |sG|r$|}tjj |d t j j |s|r8|\} } tjj | d|s'|durSt|dkrS}||ntt|ksbJd fdd }|S) zG Perform a reshape entirely by modifying indexing math cSr[r)rfr>VIEWrrrrr r]z1View._dynamic_reshape_indexer..Nr?rrrcsHt|tksJt|tftt|tfddDS)Nc3rgr)rhrH replacementsrrr rjzAView._dynamic_reshape_indexer..reindex..)rrrrrr view_exprryrr s$z.View._dynamic_reshape_indexer..reindexr?)rlrrrrrrrrrrrrrsrrr=rYreverseinsert)r:r6rtr stack_new stack_oldreordering_dense_dimold_dimsize_oldvarsize_newvar2 size_new2divisormodulus dense_exprrrr{rrux s                zView._dynamic_reshape_indexer)rr rr rr )rrr6rrr)r:rr6rrrnr)r:rHr6rHrtrrr)r:rr6rrtrrr ) rrrrNrfrrrjrrurrrrr s  - rcseZdZUdZded<d3fdd Zd4d d ZeZd4d d Zd5ddZ d6ddZ e d7ddZ d8ddZ d8ddZd9ddZd:dd Zd;d!d"Zd3d#d$Zed %dd?d.d/Zd@d1d2ZZS)Ar`z*Pretend our storage has a different layoutrr[rrcs2tt|jtrt|d|jdSdS)NrZ)rurrrZrrrrrrvrrr s  zReinterpretView.__post_init__rcC||j|jgSr)rrZr[rrrrr{ s zReinterpretView.__str__cCr(rr,rrrrr rzReinterpretView.get_namercC|jjSr)r[rrrrrr! rzReinterpretView.get_devicercCr rrrrrrr r zReinterpretView.get_origin_nodercCrr)r[rrrrrr rzReinterpretView.dtypercC t|jjSr)rr[rrrrrr rzReinterpretView.get_sizecCrr)rr[rrrrrr@ rzReinterpretView.get_stridercdfdd }|S)NrrrrkcsFj}t||}jjjjkr!t|jjjS|Sr)r[rrjloadrrrZto_dtype_bitcast)rr tmp_loaderrrrr s z+ReinterpretView.make_loader..loaderrrrrkrr'rrrr szReinterpretView.make_loaderrcCr(r)r[rrrrrr rzReinterpretView.make_indexercCrrr[rrrrr rzReinterpretView.get_layoutcCr rrrrrrr  r zReinterpretView.freeze_layoutFrurrcCs*t|jj|t|jj|Bt|jj|BSr)r(r[rrr^rrrrr s   z$ReinterpretView.get_free_symbol_usesNrrcCs@tjjj|j|jj|jj|jj|dur|j ntjjj |jj dSr) rlr wrapper_codecodegen_reinterpret_viewrZr[rrr^ writelinerrrrrr sz!ReinterpretView.codegen_referencercCrr rrrrrr' r zReinterpretView.num_readsr)r;r6r+r.r2r8r9r/r?rHrr5rF)rrrrKrrr{rrr!rrQrrr@rrrr rWrrrrrrrvrr` s,             r`c@sXeZdZUdZded<edd d Zdd d ZeZe dddZ dddZ dddZ dS) DtypeViewz(Pretend our storage has a different typer target_dtyperr new_dtyperrcCsHt|rt|\}}t|j||j|j|j|j}t||dSt ||dS)NrY)rZr) rr\r]rrrr^r_r`r)rrrrarbrdrrrr1 s   zDtypeView.creatercCrr)rrZrrrrrr{@ rzDtypeView.__str__cCrr)rrrrrrE zDtypeView.dtypercCr(rrZrrrrrrI rzDtypeView.get_sizercsjdfdd }|S)Nrrrrkcst|jjjSr)rjrrrZrrr$rrrrO sz%DtypeView.make_loader..loaderrrZrr'rrrrL s zDtypeView.make_loaderN)rrrrrrr;r.r2r8) rrrrKrrrr{rrQrrrrrrrr+ s     rc@s.eZdZedd d Ze ddddZdS) SliceViewrrrNrstartendrtuple[int, int]cstjj||tdd||fDrtjtjnjj dfd d dfdd }||dd}|||}||fS)zz Normalize start and end such that both are in the range [0, x.get_size()[dim]] and start <= end. cs|]}t|VqdSr)r2rHrrrra r(z0SliceView.normalize_start_end..rr lowerrupperrcs<||r|n||}||r|}|S||}|Sr)statically_known_geqrK)rrr clamped_lower clamped_full)max_funcmin_funcrrrclamph s z,SliceView.normalize_start_end..clamprXUnion[int, None]r~Union[Expr, int]cs$|dur|S|}|||Sr)rf)rXrrr~)rrdim_sizerr clamp_wraps s  z1SliceView.normalize_start_end..clamp_wraprN)rr rrrrrr ) rXrrrrrr~rrr) rlrrrrrMinMax evaluate_min evaluate_max)rrrNrrrr)rrrrrrrnormalize_start_endV s   zSliceView.normalize_start_endr?Tsteprrc stttsdksJzdkr"|dkr"dkr"|WSWn ty,Ynwt||r?|||\}t|d<t |r}t |\}}t|j } | | <t |j |j| |j|j |j} t|| dSd fdd } t|| d S) Nrlr?rYrrrcsDt|tksJd|dt|}||<|S)Nz wrong ndim rT)rrrrNr6rrrrr s$z!SliceView.create..reindexrcr?)rrPrr TypeErrorrrrr<rr\rr]rrr^r_r`r) rrrNrrrrrarbrcrdrrrrr s8       zSliceView.createN) rrrNrrrrrrr)r?T)rrrNrrrrrrrrrrr)rrrrrrrrrrrU s )rc@sFeZdZUded<ded<dddZdd d Zdd dZdddZdS) BaseConstantrrrrrrcCrNrrrrrrr r zBaseConstant.get_sizercCrrr|rrrrr! rzBaseConstant.get_devicercCr rrrrrrr r zBaseConstant.get_origin_nodercCr^rr9rrrrr rzBaseConstant.get_readsNr2r6r+rE)rrrrrr!rrrrrrr s    rc@sDeZdZUded<ded<ded<dd d Zdd d ZdddZdS)Constantr r'rrrrrrcr)NrrrrkctjjSr)rjrr'rrrrrr r`z$Constant.make_loader..loaderrrr'rrrr zConstant.make_loaderr-cCr rrrrrrr r zConstant.realizercCt|j|j|dS)N)r'rr)rr'rrrrrr$ rzConstant.constant_to_deviceNr8r4rI)rrrrrrr$rrrrr s   rc@s:eZdZUded<ded<ded<dd d Zdd d ZdS)IndexingConstantr rrrrrrrcr)Nrrrrkcrr)rjrQrrrrrrr r`z,IndexingConstant.make_loader..loaderrrr'rrrr rzIndexingConstant.make_loaderrcCr)N)rrr)rrrrrrrr$ rz#IndexingConstant.constant_to_deviceNr8rI)rrrrrr$rrrrr s  rcCspd}d}ttt||D](\}}|dkrq tjj||s)tjj||s)dS|td|9}||9}q dSNr?FT) reversedrrrlrrrLrr)rrJexpected_strideexpected_stride_maxryrrris_contiguous_strides_for_shape s rcCs tj|jSr)r@padding_alignment_bytesitemsizerrrrget_align_for_dtyperrc@s2eZdZdZdddZdddZ ddd dZdS)r0zxAbstract base for Layout, MultiOutputLayout, NoneLayout. Represents the memory layout of the output of an Operation.rrcCrrrrrrrr!rzOutputSpec.get_devicercCrrrrrrr storage_size rzOutputSpec.storage_sizeFrurrcCrrrrrrrrr zOutputSpec.get_free_symbol_usesNr6rFr?rH)rrrrKr!rrrrrrr0s   r0c@s8eZdZdZdeddfdMddZedNddZejdOddZedNddZ e jdOddZ edPddZ e jdQddZ dRd d!Z e Z dSd"d#Z dTd%d&ZdUd'd(ZedVd,d-ZdUd.d/ZdWd2d3ZdUd4d5ZedXd7d8ZdYd9d:ZdUd;d<ZdZd>d?Zd[dAdBZd\dEdFZdPdGdHZed d]d^dKdLZdS)_rzo Layout base class Carries tensor meta-information including offset and whether it is pinned. NrFrrrrrrrOptional[Sequence[Expr]]r^r r_rrrcCs|dur t|}||_||_t|t|ks!Jd|d|tdd|Ds,J||_||_||_||_ |j rC|jj dksEJdSdS)Nrb , stride=css|] }t|ttfVqdSr)rr rrrrrr+z"Layout.__init__..r3) rrrrrr_size_stride_offsetr_r)rrrrrr^r_rrrrs $zLayout.__init__cCrrrrrrrr3rz Layout.sizer'cC ||_dSrrrr'rrrr7r)cCrrrrrrrr;rz Layout.stridecCrrrrrrrr?r)cCrrrrrrrr^Crz Layout.offsetcCrrrrrrrr^Gr)rc Csd}|jdkr d|j}|jjdurdnd|jj}d}|jr'd|j}t|jd|jj|d|jd|jd |j||d S) Nrrz , offset=:z , is_pinned=z('z', z, size=rr/) r^rrr_rrrrr)rr^device_index_str is_pinned_strrrrr{Ks"   "zLayout.__str__cCrrr|rrrrr![rzLayout.get_devicercCsPtjtjt|jt|j|j|j|j dWdS1s!wYdS)N)rr pin_memory) rl fake_moder#r$rZrrrrr_rrrr get_example^s$zLayout.get_examplecCst|j|jSr)rrrrrrrr hrzLayout.is_contiguousrJrHrScCsVt|}|dvs|ddkrdSt|t||D]\}}}|dkr(||kr(dSqdS)N)rr?FT)rrr,)rJrSndimleftrightrrrris_channels_last_contiguousks z"Layout.is_channels_last_contiguouscCsJt|jtttt|j|jD]\}}}|dkr"||kr"dSqdSr)rrrrrrr)rrrrrrr is_transposedyszLayout.is_transposedrrcstjtks JddtjD}fdd|D}fdd|Dddd }|d gt}ttD] }||||<q=ttd D]'}||||d k}t|tsrtjj j ||||d kd d }|rwdSqPd S)NcSs*g|]\}}tjjj|dddkr|qS)rr7r?)rlrrr)rrrNrrrrs z,Layout.is_stride_ordered..crCrrErrrrrrcrrrrrrrrrarrrrcst|fdd|DS)Ncg|]}|qSrr)relement sorted_arrrrrrzDLayout.is_stride_ordered..sorted_indices..)r)rrrrsorted_indicessz0Layout.is_stride_ordered..sorted_indicesrBr?T)size_obliviousF)rrrr) rrrrrrrrlr _shape_envrd)rr non_1_indicesrrstride_orderedrexprr)rrrrs*  zLayout.is_stride_orderedcCs:dgtttdt|jd}t|g|}||SNrr?)rrrrrrrrrris_channels_last_stride_ordereds" z&Layout.is_channels_last_stride_ordered in_stridescs|t|}t|dkr |Stjst||r|St}t|dr)|j ddr)|Sttj dr3tj j nddfd d rKt fd d|DrK|St|}t|}ddtt|D}d||d<d}t|ddddD]?\} } || d} || || } t| ttjfo| tjko| |dkpt| tjotj} | || <| rt| |||| <d}qq|s|Stjd7_|S)z The padding does not change stride order but makes sure all strides larger than the threshold are multiple of align. rrmdislike_paddingFrNrsympy.Expr | intrrcs4durdSt|tjsdStfdd|jDS)NFc3s|]}|VqdSr)is_unbacked_symintrrrrrrjzILayout._pad_strides..contains_unbacked_symints..)rrr rr1rrrrcontains_unbacked_symintss  z6Layout._pad_strides..contains_unbacked_symintsc3rrrr)rrrrr(z&Layout._pad_strides..cSrrrrrrrrrz'Layout._pad_strides..r?)rT)rrrr)rrr@pad_channels_lastrrrlget_current_noderrmrrrrrrrrrrrr!padding_stride_thresholdr pad_dynamic_shapesrXr&num_comprehensive_padding)rrraligncurrent_fx_noderr new_stridespaddedrankrprev_idxrrequire_paddingr)rrr _pad_stridessP      zLayout._pad_stridescCs>t|ts Jt||jdusJ||j|j|j|_dSr)rrrrrrrrrrrr szLayout.pad_stridescCstjot|tSr)r@comprehensive_paddingrrrrrrrr`zLayout.should_pad_stridesr]cCs<t|tr|S|r|t|j|j|j|j|j|j Sr) rr]rr rrrrr^r_rrrras_fixeds zLayout.as_fixedrcCs(tjsJdt|jd|S)Nzconvert z to FixedLayout first)rrrrrrrrrrrs zLayout.make_indexerotherrcCsRt|to(|j|jko(|j|jko(|j|jko(|j|jko(|j|jko(|j|jkSr)rrrrrrr^r_)rrrrr__eq__s       z Layout.__eq__cCt|j|j|jSr)r)rrr^rrrrrrzLayout.storage_sizerurcCs$t|j|t|j|Bt|j|BSr)r(rrr^rrrrrs   zLayout.get_free_symbol_uses)rrrrrrrrr^r r_rrrr2r'rrrr3r'r rrr;r7)rrr1)rJrHrSrHrr)rrrr)rrrrrrrrr))rr]r9)rrrrr?rH)rrrrKr!rrQrsetterrr^r{rr!rr rNrrrrrr rrrrrrWrrrrrrsN              #  D     rc@seZdZdZdddZdS)r]z A Tensor layout we cannot changerrcCr)r)rVrrr^rrrrr*rIzFixedLayout.make_indexerNr9)rrrrKrrrrrr]'sr]cseZdZdZdZedBddZedCd d ZedDd dZedEddZ edFddZ e dGddZ e j dHddZ e dGddZej dHddZe dId d!Zej dJd"d!Z dKdLd&d'Z dKdMd)d*ZdNd+d,ZdOd-d.ZdPd0d1ZdQd5d6Z 7 dRdSfd@dA ZZS)Trz| A Tensor layout that we are allowed to change Assumption: layout change should NOT add or remove free symbols FsizesrrrscCsNt|dkrgStjjg}t|ddD] }|||dqtt|S)Nrr?rB)rrrrrrrr)rreversed_stridesrrrrr9s   z!FlexibleLayout.contiguous_stridesrcCs\ttt|t|ksJ||ftjj}dgt|}|D] }|||<|||}q|S)z Create a stride based on the order the dimensions should be filled in. In this format, channels last would be: [1, 3, 2, 0] N)r:rrrrrr)rr next_striderSrrrr fill_orderedBs$zFlexibleLayout.fill_orderedrcCs0ttt|t|ksJt|}t||S)z Create a stride based on the sorted order of a permuted range. In this format, channels last would be: [3, 0, 2, 1] )r:rrrrr )rrrrrrrSs zFlexibleLayout.stride_ordered memory_formattorch.memory_formatcCsP|tjkr t|tS|tjkrt|tS|tjkr t|St d|t )aq Create a stride based on a memory format. Memory format is translasted into a stride order, so channels_last is the same as: FlexibleLayout.stride_ordered(sizes, [3, 0, 2, 1]) This interface does not support memory_format `torch.preserve_format` which should be used to deduce a format from another source z>stride_ordered_for_memory_format, unsuppored memory_format: %s) r# channels_lastrrNHWC_STRIDE_ORDERchannels_last_3dNHWDC_STRIDE_ORDERcontiguous_formatrr1rr)rr rrr stride_ordered_for_memory_format_s      z/FlexibleLayout.stride_ordered_for_memory_formatrrHcCsDt|t|ks Jdd|D}ttt||jd}t||S)z Create a stride that has the same stride order as given stride For example, if given stride is [1000, 1, 100, 10], the fill order should be [1, 3, 2, 0] cSror)rlrrr?rHrrrrrqz/FlexibleLayout.same_ordered..r)rrr __getitem__rr )rrrrrr same_orderedzs  zFlexibleLayout.same_orderedcCrrrrrrrrrzFlexibleLayout.sizer'rcC|d|||_dS)Nr)!assert_free_symbol_uses_unchangedrrrrrr  cCrrrrrrrrrzFlexibleLayout.stridecCr)Nr)rrrrrrrrr cCrrrrrrrr^rzFlexibleLayout.offsetcCr)Nr^)rrrrrrr^rr rr]cCsH||j|}|r|r|||j|j}t|j|j|j||j|jSr) rrrrrr]rr^r_)rrr rcrrras_stride_orders zFlexibleLayout.as_stride_orderrcCs>|}|r|r|||j|j}t|j|j|j||j|jSr)rrrrr]rr^r_)rrr rcrrras_exact_stridess zFlexibleLayout.as_exact_stridescCD||j|}|r|||j|j}t|j|j|j||j|jSr) r rrrrr]rr^r_)rrrcrrr as_fill_orderzFlexibleLayout.as_fill_ordercCrr) rrrrrr]rr^r_)rrrcrrr as_same_orderrzFlexibleLayout.as_same_order$dict[tuple[str, bool], sympy.Symbol]cCs<i}dD]}dD]}||f}ttt|||||<qq|S)N)rrr^TF)r:r(r)rinitial_free_symbolsrrurrrrget_initial_free_symbol_usess z+FlexibleLayout.get_initial_free_symbol_usesrrr3cCsFdD]}|j||f}tt||}||ks Jd|d|qdS)Nr z)Expected free symbols unchanged, but got z vs )r!r:r()rrr'ruold_free_symbolsnew_free_symbolsrrrrs z0FlexibleLayout.assert_free_symbol_uses_unchangedNrrrrrrrr_cs@|r t||}nt|}tj|||||d||_dS)Nr_)rr rrurr"r!)rrrrrr_rSrvrrrs  zFlexibleLayout.__init__)rrrrs)rrrrrrs)rrrrrr)rrr r rr)rrrrHrrr2rr3rr?)rrr rrr])rrHr rrr])rrrr])rrHrr])rr)rrr'r3rrr) rrrrrrrrr_rrr)rrrrKrrNrr rrrrQrrrr^rrrrr"rrrrrrvrr/sH               rcsLeZdZdZdfdd Zdd d Zdd d Zed ddddZZ S)NonOwningLayoutz,Is a view into the storage of another tensorviewUnion[BaseView, TensorBox]rrcs,|}t|j|j|j|j||_dSr)rrurrrrrr')rr'r[rvrrrs zNonOwningLayout.__init__rcC |Sr)rrrrrrrrzNonOwningLayout.make_indexerrcCs4|jj}|dkr dSddlm}tjj||S)NrTr?) ALIGNMENT) r'rr^utilsr*rlrrr)rr^r*rrrmaybe_guard_aligneds  z#NonOwningLayout.maybe_guard_alignedFrurcCsVt|jtsJ|jj}t|tsJt||j}t|ts%Jt||j|Sr) rr'r`rZrrrr[r)rrubox input_bufferrrrrs  z$NonOwningLayout.get_free_symbol_uses)r'r(rrr9r1r?rH) rrrrKrrr,rWrrrrrvrr&s r&c@eZdZdZdS)CommBufferTypesymm_memN)rrrSYMM_MEMrrrrr0(r0cs4eZdZUdZded<ded<d fdd ZZS) CommBufferLayoutax A layout that signifies the buffer is a comm buffer. In terms of striding, the layout is identical to `FixedLayout`. Buffers with this layout do not participate in in-place reuse - it can be neither the source nor the target for in-place reuse. For detailed motivation and usage of this layout, see NOTE [lowering-time collective optimization]. r0comm_buffer_typer group_namer[rcsVt|ts td|d|}tj|j|j|j|j |j |j d||_ ||_ dS)NzJA `CommBufferLayout` can only be initialized with a `FlexibleLayout` (got z).rrrrr^r_)rrrYrrurrrrrr^r_r5r6)rr[r5r6fixedrvrrr;s"  zCommBufferLayout.__init__)r[rr5r0r6r)rrrrKrrrrrrvrr4,s  r4c@sdeZdZUded<ejdddZded<ejdddZded <dd d ZdddZ dddZ dS) NoneLayoutrrcCdgSrrrrrrr_rzNoneLayout.default_factoryrGrcCr:rrrrrrr`rrrrcCrrrrrrrrbr zNoneLayout.storage_sizer0cC|Srrrrrrrer zNoneLayout.as_fixedcCrrr|rrrrr!hrzNoneLayout.get_deviceNrFr0r6) rrrrrLrMrrrrr!rrrrr9Ts    r9cseZdZd#fdd Zed$dd Zejd%d d Zd&ddZd'ddZd(ddZ e d)d*ddZ d+ddZ d,d!d"Z ZS)-MutationLayoutSHOULDREMOVEr@rrrcs@t|||d||_|}tj |dSr) rurrr rr@ get_bufferrrlrmark_buffer_mutated)rr@rrvrrrms z#MutationLayoutSHOULDREMOVE.__init__rcCrr) real_layoutrrrrrrxr)z!MutationLayoutSHOULDREMOVE.strider'rcCr rrrrrrr|rr cCr)r)rArrrrrrrz'MutationLayoutSHOULDREMOVE.storage_sizercs2dfdd |j}t|tsJt||S)Nr@r rcsBt|tr |jSt|tr|St|tr|jS|Sr)rr>r@rr MutableBoxrZ)r@ unwrap_viewsrrrDs      z;MutationLayoutSHOULDREMOVE.get_buffer..unwrap_views)r@r rr )r@rrr)rrrrCrr?s z%MutationLayoutSHOULDREMOVE.get_bufferrcCs|j}t|ts J|Sr)r?r[rr)rr[rrrrAs z&MutationLayoutSHOULDREMOVE.real_layoutFsrcdst unsafe_aliasrc Cs|tj|t|tr|j}||sAt j | | | ddt||Dd}t|ttfs>J|j}|t|dsNJ|t|jjts]Jt|jjt||j_|jS)NcS g|] \}}tjj||qSrrlrrcheck_equals_and_simplifyrrrrrrrz;MutationLayoutSHOULDREMOVE.realize_into..rrZ)rrlrr@rrrrZrrrr!r rrrrrBrr[rrr>)rrErFrGrrrr realize_intos*   z'MutationLayoutSHOULDREMOVE.realize_intorcCr=rrrrrrrr z#MutationLayoutSHOULDREMOVE.as_fixedrcCr(r)r@rrrrrrrz'MutationLayoutSHOULDREMOVE.make_indexer)r@rrrr2)r'rrrr3)rrr/r?)rErrFrrGrrr)rrr9)rrrrrQrrrr?rArrMrrrrrrvrr>ls      'r>csleZdZUded<ded<d]fdd Zd^d d Zd_d dZd`ddZdaddZdbddZ e dcddZ ddddZ dedd Z dfd"d#Zdgd%d&Zdhd'd(Zdid*d+Zdjd-d.Zd]d/d0Z 1dkdld5d6Zdmd7d8Zdnd:d;Z 1dkdod=d>Zdjd?d@ZdpdBdCZdqdrdGdHZd]dIdJZdsdLdMZdsdNdOZdtdQdRZed 1dkdudUdVZ dvdWdXZ!dwdYdZZ"djd[d\Z#Z$S)xrr-rr0r[rrcst|dddSrX)rurrrrvrrrs zBuffer.__post_init__rcCr)r)rrrrrrrrzBuffer.make_indexerrcCs|jsJ||jSrrrrrrrrYzBuffer.get_name!Union[torch.Tensor, sympy.Symbol]cCs&t|jtr |jStt|jjr)rr[rrrrrrrrrrs  zBuffer.get_examplercCr)r)rr!rrrrr!rzBuffer.get_devicercCr rrrrrrrr zBuffer.get_defining_oprcCrr)rrrrrrrr)z Buffer.dtypercCg|jSr)rrrrrrrrzBuffer.get_sizerscCrOr)rrrrrrr@rzBuffer.get_strider cCrr)rr^rrrr get_offsetrzBuffer.get_offsetrcCs"t|jtr |jStt|jjr)rr[rrrrrrrrrs zBuffer.get_layoutcCrrrrrrrrrzBuffer.get_output_specrcCrr)rrrrrrrzBuffer.get_storage_numelrcCrr)rr_rrrr get_is_pinnedrzBuffer.get_is_pinnedcCs0t|jtrt|jts|j|_dSdSdSr)rr[rr&rrrrrr s zBuffer.freeze_layoutFrrr cC0t|jts Jt|j|jj||d|_dSNr)rr[rrrr rrrr sz&Buffer.freeze_layout_with_stride_ordercC,t|jts Jt|j|j||_dSr)rr[rrrrrrrr z$Buffer.freeze_layout_with_fill_orderrcCrTr)rr[rrrrrrrrrUz$Buffer.freeze_layout_with_same_orderrcCrRrS)rr[rrrrrrrrs z'Buffer.freeze_layout_with_exact_stridescCrrrrrrrrrzBuffer.is_zero_elementsrcs*r ttdSdfdd }|S) Nrrrrrkcs}tjp d||Sr)rrjrrrrrrrr$sz"Buffer.make_loader..loaderr)rr rr r'rrrrszBuffer.make_loaderNrrcCrrrrrrrr*rzBuffer.codegen_referencecCr rrrrrrr-r zBuffer.decide_layoutrcCt|jtr |jjgSdSr)rr[r&r'rrrrrr'0 z#Buffer.get_inputs_that_alias_outputcCrXr)rr[r>r@rrrrrr%5rYzBuffer.get_mutation_namesrcCst|gSr)r:rrrrrr:rzBuffer.get_read_namesrurcCr^rr9rrrrr=zBuffer.get_free_symbol_usescCr^rr9rrrrr_CrzBuffer.get_unbacked_symbol_defscCr rrrrrrrFr zBuffer.realizecCrrrrrrrshould_allocateIrzBuffer.should_allocater)r9r;)rrNr6r,r.r2)rrsr3r/r0rFr1r?r@rA)rrrr)rrr rrrr8rr5rJr*rHrbr4)%rrrrrrrrr!rrQrrr@rPrrrrQr r rrrrrrrr'r%rrWrr_rr[rrrrvrrsL                         rc@s0eZdZd ddZd ddZejZdd d Zd S)OperationBufferrr\cCs|gSrrrrrrr]QrzOperationBuffer.get_outputsrRcCr=rrrrrrrTr zOperationBuffer.get_defining_oprcCst|t|dSr)rrrRrrrrrZs zOperationBuffer.__post_init__NrarrRr))rrrr]rrRr&rrrrrr\Ns   r\c@eZdZdddZdS)rrrcCrr rrrrrr`r zInputBuffer.num_readsNrF)rrrrrrrrr_rc@r/) DonatedBufferaY Represents a donated buffer which is a saved tensor that is not alias to any fwd inputs, fwd user outputs, and bwd outputs. We generally cannot inplace reuse the input tensor memory during backward since it might be used in another function. However, donated buffer can be inplace reused during backward to save memory. N)rrrrKrrrrr`dr3r`c@s.eZdZUdZded<d ddZdd d ZdS)rNrrrrcr)Nrrrrkcs,}ttjj||Sr) rrrjrrlr constant_namerrrVrrrrrs z*ConstantBuffer.make_loader..loaderrrr'rrrrqszConstantBuffer.make_loaderrrrcCsttj|||jdSN)rr[)rrlrrarr[rrrrr${sz!ConstantBuffer.constant_to_devicer8rI)rrrrrrr$rrrrrns   rc@sLeZdZdddZed ddd d ZddddZdddZdddZd S)NoneAsConstantBufferrrcCr^rr9rrrrrrzNoneAsConstantBuffer.get_readsFrurrcCr^rr9rrrrrrZz)NoneAsConstantBuffer.get_free_symbol_usesNrrrcCs tjjjSr)rlrrnone_strrrrrrrz&NoneAsConstantBuffer.codegen_referencer0cCs tddSNr|)r9rrrrrrz$NoneAsConstantBuffer.get_output_speccCrrrrrrrrr z&NoneAsConstantBuffer.has_tensor_outputrEr?rHrr5r0r1) rrrrrWrrrrrrrrrcs   rcc@sBeZdZUded<ed dddd ZddddZdddZd S)rr rFrurrrcC t|j|Sr)r(rrrrrr z*ShapeAsConstantBuffer.get_free_symbol_usesNrrrcCstjj|jSr)rlrrcodegen_sizevarrrrrrrr`z'ShapeAsConstantBuffer.codegen_referencecCrrrrrrrrr z'ShapeAsConstantBuffer.has_tensor_outputr?rHrr5r1)rrrrrWrrrrrrrrs  rcseZdZUdZded<dZded<eejdQd d Z dRd d Z dSddZ dTddZ dUddZ dVddZed dWdXddZdYfd d! ZdZd"d#Zd[d%d&Zd\d(d)Zd]d+d,Zed^d.d/Z 0 0d_d`d6d7Ze 0dadbdAdBZdcdDdEZdRdFdGZdZdHdIZdZdJdKZdddOdPZZS)erzb Represents a buffer that is computed during kernel execution rather than being an input. rcrZFzClassVar[bool]_force_realizerIterator[None]ccs*tj}z dt_dVW|t_dS|t_wNT)rri) old_valuerrr force_realizes zComputedBuffer.force_realizer-cCs(|jdur|jSt|jdr|jjSdS)z Returns self.name if it exists, otherwise returns the name of the data node if that exists. If neither exist, returns None. Nr)rrrZrrrrget_computed_buffer_names  z'ComputedBuffer.get_computed_buffer_namercCr(rrZrrrrrrrzComputedBuffer.num_readsrcCr(rrZrrrrrrrzComputedBuffer.get_readsrcCr(rr4rrrrrrzComputedBuffer.get_read_namesrcCst|jttttfstjtttdSt t dd,|j r7t ||j|jWdSt ||jWdS1sKwYdS)Nrwrites index_exprsrT)rrZrrrrrA ReadWritesr:rrrr rJget_store_functionr}r!rrrrrrs& $zComputedBuffer.get_read_writesrurrcCsD|j||j|B}|r t|tr |||O}|Sr)r[rrZhas_store_functionrrurLr)rrurrrrrs  z#ComputedBuffer.get_free_symbol_usesrcs<|s|jtjjvr|dkr|js|jSt Sr) r rrlrmutated_buffersrrirZrrurrvrrrs   zComputedBuffer.make_loadercCst|jttttfSr)rrZrrrrrrrrrv rz!ComputedBuffer.has_store_functionCallable[..., None]cCs`|}t|jtttfrt|jj |j |St|jt s'Jt |jt|jj |j |Sr)rrrrrZrrrr rrrrrr%rrrru s z!ComputedBuffer.get_store_functionOptional[list[int]]cst|jtrYt|j|j\\}}|j }t dd|Ds&Jfdd|D}|rYt|jt t frA|j |n|fdd|D}ddlm}|||SdS) al If our layout is still flexible, try to determine the stride order based on stride orders of reads. TODO(jansel): A better algorithm here would look at downstream consumers of this value and try to do global graph-level layout optimization. This is also something just begging to be autotuned. css"|] }t|tjtjfVqdSr)rrAStarDep MemoryDeprrrrr$  z0ComputedBuffer.get_fill_order..cs.g|]}t|tjrt|jddDqS)cSsi|] }|dkr|tjjqSrrc)rvrrrr)z..)rrAr{rhrr)rrrr(s  z1ComputedBuffer.get_fill_order..csg|] }tjj|qSrrlrrr4rr)r"rrr3sr?pick_loop_orderN)rr[rrAr2rZr}r!rrrrrr schedulerrr)r index_varsrrstride_lengthsrr)r"rrrs*     zComputedBuffer.get_fill_orderrcCs6t|jtr|}|r||dS|dSdSr)rr[rrrr rrrrr<s  zComputedBuffer.decide_layoutMtuple[tuple[list[Expr], list[Expr]], LoopBody, tuple[list[Expr], list[Expr]]]c Cstj|j|jdd\}}ttd|t | | r$|n|dd|g|R}Wdn1s:wYg}g}g}g}| D]+\}} ||dvrd|rYJ| || | qK||dvslJ| || | qK||f|||ffS)Nqr{rr?r)rAr2rZr}r!rrrr!rLrur itemsr) rr var_rangesrr reduce_vars index_size reduce_sizer}rrrrget_default_sizes_bodyDs2       z%ComputedBuffer.get_default_sizes_bodyNextra_indexing_constraints*Optional[tuple[dict[Any, Any], list[Any]]]recompute_sizes_body_funcOptional[Callable[..., Any]]8tuple[tuple[list[Expr], list[Expr]], Optional[LoopBody]]cs\\}}}\}}|r|||f|||f\\}}}\}}g|j|durut|tr4t|dks6J|\}} t|tsEJt|t| tsPJt| t dd| Ds[J|j } | |kshJ| |ffdd| D} | 7g| t j tjs|dfdd } ||} tt ptj } | || || \}}}| || || \}}}tj||dd\\}}}t|||||g|||}||f|fS)an This is a main place where we do loop transformations in a backend-agnostic way. Here we: 1) Remove any 1 dimensions 2) Fuse contiguous dimensions together 3) Reorder dimensions based on stride orders Optional argument extra_indexing_constraints can be used to append additional indexing expressions to existing ones derived from buffer's body. This can be useful to fuse scheduler nodes with compatible ranges, e.g. (s0*s1*...,) and (s0, s1, s2, ...) on CPU by preventing indexing simplifications and obtaining index/reduce ranges for the scheduler node compatible with other nodes. Optional argument recompute_sizes_body_func can be used to recompute sizes and body on the default body. This can be useful to append additional loop transformations. Nrcs|]}t|tVqdSr)rr )rfrrrrrjz6ComputedBuffer.simplify_and_reorder..csg|]}|vr|qSrrrh)index_formulasrrrz7ComputedBuffer.simplify_and_reorder..x_varsSequence[sympy.Symbol] support_varsrrsimplify_loopsrrdtuple[list[int], Callable[[Sequence[int]], Sequence[int]], Callable[[Sequence[int]], Sequence[int]]]c s\|||\}}}||}|r'tjj||t||\}}}t||}n|}|||fSr)_apply_loop_reorderingrlrr_simplify_loopsrEr) rrrrreindex0rr_prunerr memory_addrsrrrsimplify_and_reorders     zAComputedBuffer.simplify_and_reorder..simplify_and_reorderpr{) rrrrrrrrrr)rindexing_exprsrrrrrrrrrget_write_exprsrlrr)rBPREFER_STORE_LOOP_ORDERextendget_read_exprsrcr1r@loop_ordering_after_fusionrAindex_vars_no_squeezerL)rrrrrrrrextra_indexing_rangesextra_indexing_exprexpected_var_rangesrrshould_merge_loops iter_ranges iter_reindexr reduce_rangesreduce_reindex iter_varsrrrrresx       z#ComputedBuffer.simplify_and_reorderrrrrrrlist[sympy.Expr] priority_idxrc sddlm}|dur g}z*fdd|D}t|t|kr)t|dtks+Jtt|||}Wn tyVtjrLt dt t |tt t}Ynwfdd|Dt |t|fS) zU Shuffle the order of loops around to hopefully improve performance. r?rNcsg|] }tjj|qSrrr)rrrrrz9ComputedBuffer._apply_loop_reordering..rz%Did not simplify complex index: %s %scrrrr)rrrrr)rrrrr Exceptionr@rr1warningrrrrr)rrrrrrrSrr)rrrrrs,     z%ComputedBuffer._apply_loop_reorderingrcCr(rrZr!rrrrr! rz!ComputedBuffer.get_reduction_sizecCr(rrZr rrrrr  rz!ComputedBuffer.get_reduction_typecCr(r)rZrrrrrr#rzComputedBuffer.is_no_opcCrrkrrrrrr[r zComputedBuffer.should_allocaterrrcCr)rrZr$rrrrr$rz!ComputedBuffer.constant_to_device)rrjr4rFrEr*rDr?rHr8r1)rrx)rryr))rrNN)rrrrrrr) rrrrrrrrrryrrr2rI) rrrrKrrirNrOrPrmrnrrrrrWrrrvrurrrVrrrr!r r#r[r$rrrrvrrsB         ' " | '   rcsfeZdZdZd#fd d Zd$d dZd%d&ddZd'ddZd(ddZd)ddZ  d*d+d!d"Z Z S),rzt Represents a Triton (in the future other type) of template operator that we can fuse an epilogue onto. r[r0rqrrmake_kernel_renderrrrcs@tjd|dt||_||_tj||_ tj |dSrb) rurrunwrap_storagerqrrlrregister_bufferrregister_operation)rr[rqrrvrrr s  zTemplateBuffer.__init__rcCs |jddS)NT normalize)rJrrrrr,rzTemplateBuffer.get_read_writesFrrc s||d fdd }tj||d|d }|jD]:ttt fs/Jt tj t s.dummyrrcs$t|dksJt|Sr)rrjrrrS)rrwrrrAs)rrrrrr )rrrrArJrrqrr`rrr[rr)rrrdepsr)rrwrrrJ/s"      z"TemplateBuffer.extract_read_writesrcCstjjSr)rrrrrrrrr!Krz!TemplateBuffer.get_reduction_sizer-cCr rrrrrrr Nr z!TemplateBuffer.get_reduction_typecCrrkrrrrrr[Qr zTemplateBuffer.should_allocateNrrr.)rurroutputsr#rj higher_orderflex_attentionflex_attention_backwardrlr current_noder@rrqrrr!r:r subgraph_inps subgraph_outs)rr[rqrrr allowed_setrrvrrrcs(  "  zTritonTemplateBuffer.__init__Frurrcst|}|jr |jng}|jr|jng}|D]%}t|tjr)|t||qt|t r7|||q|dus=Jq|D]}t|t rP|||q@|dusVJq@|Sr) rurrrrrr updater(r)rruresrrrwrrvrrrs    z)TritonTemplateBuffer.get_free_symbol_usesr\cCrrrrrrrr]rz TritonTemplateBuffer.get_outputsrcCrr)rrrrrget_allowed_prologue_inpsrz.TritonTemplateBuffer.get_allowed_prologue_inpsrcCsd|jd}|S)NzTritonTemplateBuffer(layout=r/r)rrrrrr{szTritonTemplateBuffer.__str__r) r[rrqrrrrrrrrrrr?rHrar*r;) rrrrrWrr]rr{rrrrvrrbs,  rcsreZdZdZd%fd d Zd&ddZd'ddZd(ddZd'ddZd'ddZ d)ddZ d*d!d"Z d'd#d$Z Z S)+ ChoiceCallera. Represents a possible choice used in autotune_process.py. During autotuning, self.benchmark() is first called to get benchmark result, and if this choice is selected, self.output_node() is called to get the output_node. Children classes: TritonTemplateCaller, CUDATemplateCaller. rrrr\r[r descriptionrrcs&t||_||_||_||_dSr)rurrr[rr)rrrr[rrvrrrs  zChoiceCaller.__init__rr rrrcsL|ttd}tjrtfddfi|Stjd|ifi|S)N)warmuprepcsSrrralgorrrrsz(ChoiceCaller.benchmark..r) to_callableautotune_warmup autotune_repr@/profile_bandwidth_with_do_bench_using_profilingr\rQ benchmark)rrrbenchmark_configsrrrrszChoiceCaller.benchmarkcCrUrrVrrrr call_namer zChoiceCaller.call_namerdcCrUrrVrrrrrr zChoiceCaller.to_callablecCr)z Hash key for the underlying kernel. By default, we assume there are no runtime params, so kernel hash key defaults to choice caller's hash key. )hash_keyrrrrkernel_hash_keyszChoiceCaller.kernel_hash_keycCrUrrVrrrrrr zChoiceCaller.hash_keyr~cCrUrrVrrrrror zChoiceCaller.output_node.)rur_choice_timings_fn_choice_timingsoriginal_inputsr_output_plannable_make_kernel_renders)rr[rqrrrrvrrrs zMultiTemplateBuffer.__init__rcCr)z^ Are all possible choices TritonTemplates or Extern Kernels with out variants )rrrrroutput_plannablesz$MultiTemplateBuffer.output_plannableN hint_overriderdict[ChoiceCaller, float]cCs$||jvr |||j|<|j|Sr)rr)rrrrrchoice_timings"s  z"MultiTemplateBuffer.choice_timingscallerrrjccsZt|tjjjsJt||j|jksJ|j}||_z dVW||_dS||_wr) rr#rrTritonTemplateCallerrr[rr)rrrenderrrrswap_as_triton_caller)s  z)MultiTemplateBuffer.swap_as_triton_callercCsRt|tjjjsJt|||jjksJ| |jj ks"J| |_ dSr) rr#rrrrrr[rr@rrr)rrrrrfinalize_as_triton_caller7s z-MultiTemplateBuffer.finalize_as_triton_callertuple[ChoiceCaller, float]cCs&|j|d}t||jd}|||fS)N)rr)rrr)rrtimings min_choicerrrget_min_choice?s  z"MultiTemplateBuffer.get_min_choicecallers-dict[Optional[int], TritonTemplateCallerBase]cCs0|D] \}}||j|<q|jd|_dS)z;Finalize with multiple callers for different hint overridesN)rrrr)rrrrrrrfinalize_as_triton_callersFsz.MultiTemplateBuffer.finalize_as_triton_callers) r[rrqrrrrrrrrrrr1r)rrrr)rrrrj)rrrr)rrrr )rrrr)rrrrKrrQrrrOrPr r rrrrrrvrrs     rcs2eZdZdfdd ZdddZdddZZS)CUDATemplateBufferr[rrqrrrCallable[_P, _T]workspace_sizertemplaterpsupports_epilogue_fusionrrrcs&t|||||_||_||_dSr)rurrrr)rr[rqrrrrrvrrrRs  zCUDATemplateBuffer.__init__cCs|jdur|jSdSr)rrrrrr`arz%CUDATemplateBuffer.get_workspace_sizecCs$|D] }t|ddqdSr)r]rjrr)rrhrrremulate_store_fnds z#CUDATemplateBuffer.emulate_store_fn)r[rrqrrrrrrrrprrrrrFr))rrrrr`rrrrrvrrQs rcs,eZdZdfd d Zdfdd ZZS)CppTemplateBufferr[rrqrrrrrrprr rrcs&t|||||_||_d|_dSr)rurrrr)rr[rqrrrrvrrrjs zCppTemplateBuffer.__init__cspt|jtr3t|jtsJt|j|jd}t|ts#Jt||j}t|ts1Jt||St Sr) rr[MultiOutputLayoutrrrrrrur)r first_outputr[rvrrrws   zCppTemplateBuffer.get_layout) r[rrqrrrrrrprr rrr/)rrrrrrrrrvrris rcs0eZdZdZ ddfdd ZdddZZS)CuteDSLTemplateBufferz Buffer for CuteDSL (CUTLASS Python DSL) template kernels. Similar to other template buffers but specialized for CuteDSL operations. Nr[rrqrrrrrr rrrrcst||||_|_g_|dur?tjdts'Jtjdjd jfdd|D7_dSdS)Nrcrrrrrrrrrz2CuteDSLTemplateBuffer.__init__..) rurrrrrrqrrr!)rr[rqrrrrvrrrs"zCuteDSLTemplateBuffer.__init__r\cCrrrrrrrr]rz!CuteDSLTemplateBuffer.get_outputsr) r[rrqrrrrrr rrrrra)rrrrKrr]rrrrvrrs  rr)Sequence[Union[IRNode, Sequence[IRNode]]]TypeIs[Sequence[IRNode]]cCstdd|DS)Ncsrrrrr\rrrrrjz#is_node_sequence..)r)rrrris_node_sequencerLr c@szeZdZUded<d!ddZd"d d Zd#d dZed$ddZe d%ddZ d&ddZ d'ddZ e d d(d)ddZd S)*rrrqrrrrcCs |j|}t|ts J|Srrqrrr)rrinputrrr input_names zInputsKernel.input_namercsttj}tj|jD]#}t|tr |fdd|Dq t|tr&q | | q ttjfdd| D}tj ||tdS)Nc3|] }|VqdSrrWrHrzrrrrz/InputsKernel.get_read_writes..c3r$rrWrr%rrr  rq) r:rArGrzrqrrrrrrr]rt)rrr"rrrr%rrs    zInputsKernel.get_read_writesrcCrrrrrrrrrzInputsKernel.get_readsrrcCs~t|tr|j}t|tr|j}t|trt|tst|}t|tr)||St|t r0|St|t tfs=Jt ||Sr) rrrZrrr`r realize_inputunwrap_storage_for_inputTorchBindObjectrrrrrrrr(s      z%InputsKernel.unwrap_storage_for_input%list[Union[IRNode, Sequence[IRNode]]]cCs@g}|D]}t|trdd|D}nt|}||q|S)NcSrNr)rr(rrrrrrz/InputsKernel.unwrap_storage..)rrrr(r)rq inputs_newrrrrrs   zInputsKernel.unwrap_storagercCrrkrrrrrr"r zInputsKernel.is_externcCrr rrrrrrr zInputsKernel.num_readsFrurcCsNttj}|jD]}t|tr|||O}q |D] }|||O}qq |Sr)r:rr"rqrrr)rrurrw inner_inprrrrs   z!InputsKernel.get_free_symbol_usesN)rrrrrDrErrrr)rqrrr+r1rFr?rH)rrrrr#rrrr(rNrr"rrWrrrrrrs       rc@s eZdZd ddZd ddZdS) NopKernelrrcCrrkrrrrrr#r zNopKernel.is_no_oprcCr^rr9rrrrrrzNopKernel.get_readsNr1rE)rrrr#rrrrrr/s r/c@s^eZdZdZeddd Ze ddddZed ddddZed ddZ d!ddZ d S)" ConcatKernelzn There isn't actually a real kernel for concat, we just change the storage for the upstream data. rqrrrNrrrc Cs|d}|d}t|d}dg}||g}d|kr)t|ks,JJtdt|D]Z}||} |||t| t|ksLJ|||ksVJ|||ks`Jtt|D]} | |krw|| | | || <qftjj || | | || <qf|||q3t |} t jrt| ||dj} tt|D]!}||} t| r| } t| trt| j| jrt|} nqtdd|D}tjjjd}t|tsJt||durtdd|Drt|} tdd|D}|dusJtdt|||| |d gd }t |}g}t!|D]{\}}t|t"t#fs.Jt||$|t%j&||||||dd }t|t'sLJt|t|j(tsZJt|j(|j(|t|j)t"rm|j)*}n|j)}t|t r|+r|}durt,|jrt-|s||.qt|dkrtj/|t0j1rtj2|tj3||_4|5|j(|_(tj6||S) z6 Create the concat kernel from inputs rr?csrr)rrHrrrr8r(z&ConcatKernel.create..FcssB|]}d|jvo|jdjtjdp|jdjtjdVqdS)rXr N)rmr r#rrrargrrrr<s  css"|] }t|o |jVqdSr)rrr_rHrrrrFs N)rrrrr_rr[rq)r)7r!r rrrrrrlrrrJrrr@rrrrrrrr]rrrr,rrrrrr0rrrrBrMrrrrqrZrrrcrbr&r)rBFOREACHregister_operation_listrrrr)rrqrNrrr6 offsets_start offsets_endr input_sizerB output_striderr[any_input_is_storage_and_layout fx_node_argsr_ concat_kernelkernelop_namesrwr.input_unwrappeddevrrrr s               zConcatKernel.createNrErrFrrcCst|tr ||j|St|ttfsJt|t|jtrPt|jjt r*|jj s,dS|dur2dSt | t | ks@dSt ddt| | DSt|jdoct|jjtoct|jt S)NFTcsrUrrVrWrrrrrXz=ConcatKernel.can_realize_into_without_copy..r[)rrcan_realize_into_without_copyrZrrrrr[r]rrr@rrrrExternKernelAlloc)rrErFrrrrB{s*      z*ConcatKernel.can_realize_into_without_copyFrurcCs t||Sr)r/rrrrrrrgz!ConcatKernel.get_free_symbol_usesc Cst|tst|rt|\}}t||d}t|ts Jt|t|tr,||j|St|trL| t |jds=J| ||rLt ||j_ |jStj|||ddt||Dd}|||S)NrYr[cSrHrrIrKrrrrrLz-ConcatKernel.realize_into..r)rr`rr\rrrMrZrrrrBr&r[rrr!r rrr)rrErFrar[pwrrrrMs,        zConcatKernel.realize_intocCrrkrrrrrr[r zConcatKernel.should_allocate)rqrrrNrrrr)rErrFrrrr?rH)rErrFrrrr1) rrrrKrrrBrWrrMr[rrrrr0s n#  r0cseZdZUdZdZded<ejedZ ded<dZ d ed <dZ d ed <dZ d ed <eje dZded<dZded<dZded<ejedZded<dZded<ejedZded<eje dZded<       ddfd$d% Zdd'd(Zdd*d+Zdd,d-Zdd.d/Zdd2d3Zdd4d5Zddd6d7Zdd8d9Zdd;d<Zedd@dAZ e!ddGdHZ"e!ddJdKZ#e!ddLdMZ$e!ddNdOZ%e!   PdddWdXZ&e! PdddZd[Z'e! Pddd]d^Z(e!dd_d`Z)e!ddadbZ*e!ddcddZ+e!ddedfZ,ddgdhZ-ddidjZ.dddndoZ/ddpdqZ0ddsdtZ1dddvdwZ2ddxdyZ3ddzd{Z4dd|d}Z5dd~dZ6dddZ7dddZ8e9d PddddZ:dddZ;e;Z<Z=S)rz A class that represents Kernels which are not directly lowered to Inductor Loop Level IR, such as custom operators, or aten operators which we fallback to. rr constant_argsr;dict[str, Any]rNOptional[ReinterpretView] output_viewr-python_kernel_namecpp_kernel_name Iterable[str]ordered_kwargs_for_cpp_kernelOptional[_OpOverloads] op_overloadzOptional[list[dict[str, Any]]]arg_propertieszdict[str, dict[str, Any]]allarg_propertiesz#Optional[dict[str, dict[str, Any]]]kwarg_propertiesz"dict[sympy.Symbol, pytree.KeyPath]unbacked_bindingszlist[MutationOutput]mutation_outputsrr[r0rqrOptional[dict[str, Any]]rrc sntj|||d||_|r|ni|_||_| |_||||| |_| i|_ g|_ t j j|_dSNr4)rurrErrHrNset_cpp_kernel_nameset_python_kernel_namerLcollect_arg_kwarg_propertiesrRrSrlrrfx_node) rrr[rqrErrHrIrJrLrNrvrrrs    zExternKernel.__init__r\cCs |g|jSr)rSrrrrr]rzExternKernel.get_outputsrcCr^rr9rrrrr_rz%ExternKernel.get_unbacked_symbol_defscCst|jtjjrdd|jjjDn ddtt|j D|_ t|jtjjr1dd|jjjDni|_ t|jtjjrW|j sJdd|jjjD|_ dd|jjjD|_ dSg|_ dS)NcSs$g|]}|js|j|j|jdqS))rrrw) kwarg_onlyr real_typerwrHrrrrsz=ExternKernel.collect_arg_kwarg_properties..cSsg|]}iqSrrrrrrrrcSsi|] }|j|j|jdqS))rrw)rr[rwrHrrrrrz=ExternKernel.collect_arg_kwarg_properties..cSg|]}|jr|jqSrrZrrHrrrr& cSsg|]}|jr|qSr)rZrHrrrr)s )rrNr#_ops OpOverload_schema argumentsrrrqrOrPrL schema_kwargsrrrrrX s*    z)ExternKernel.collect_arg_kwarg_propertiescCs$t|jtr||dSdSr)rr[rapply_constraintr rrrrr/s  zExternKernel.decide_layoutwrapperrqcCs$t||\}}|r||dSdSr)r_ make_comment)rre origin_str_detailed_origin_strrrrcodegen_comment4szExternKernel.codegen_commentcCrUrrVrrerrrcodegen9r zExternKernel.codegencCs||_tjjrt|jtjjsdS|j}|jdurB|j dkr;|j dkr+|j ddn|j dd}d|d|_dS|jj|_dSdS)Natenr~.rrz at::_ops::z::call)rJrlr cpp_wrapperrrNr#r_r` namespace _overloadnamerrreplacerar)rrJr>opnamerrrrV<s      z ExternKernel.set_cpp_kernel_namecCsd||_|dur dS|j}|durdSt|tjjr"d|j|_dS|jddd|j|_dS)Nztorch.ops.higher_order.._ops..ops.rm) rIrNrr#r_HigherOrderOperatorrrrq)rrIr>rrrrWTsz#ExternKernel.set_python_kernel_namercCsddlm}|}r|jntjj}tjjr!|jdusJ|jStjj rDt tjj |s4Jttjj |j dus;Jtjj |j |S|jdusKJ|jS)Nr?) CppWrapperCpu)codegen.cpp_wrapper_cpurvr!rrlr device_type fx_wrapperrIrnrrrJget_c_shim_func_name)rrvdrrrrget_kernel_namecs zExternKernel.get_kernel_namerrr~cCs:tj||||||d}||S)N)rrrerfrr~) rrr!r rrrrr)rrDrrr copy_inputvszExternKernel.copy_inputr>rzrr ituple[Any, list[Any], list[Any], Callable[[Any, Any], Any], Optional[dict[sympy.Symbol, pytree.KeyPath]]]cs||d}t|\}gg}g}|D].}t|to"t|t dr.||qt|tr=tjj j j |dd}||qdfd d } fd d |D}|D] } t | rbt | d dqVg} |D]n} t| ts| tjjvr| tjj| qgt| ts| tjjvr| tjj| qgt| tr| | qgt| tjjjr| jj} | jjdkr| dusJ| tjj| qg| t| d dqg| | |\} }|| i|}d}tjj }r*tjj !d}t"}tjj#tj$j%j&kr |d}t'tj}|t(|tj|Wdn 1swYt)|||}t|t*t+fs5|gn|}|D]'}t|tj,r_|j-r_d}tjjj !dd}r[|d|}|tj_.q9|||| |fS)NrrB)rmnew_tensor_argsrnew_non_tensor_argsrtuple[list[_T], dict[str, _T]]csdg}t|}t|}D]}|r|t|q |t|q t|}|dg|difS)Nrr)iterrnextpytreetree_unflattenr)rrr it_tensorsit_non_tensors is_tensorr) args_spec is_arg_tensorrrunflatten_argss z3ExternKernel.process_kernel..unflatten_argscrrr'rHrrrrrz/ExternKernel.process_kernel..Trr4)r rXr?zEsparsity not handled. Please file issue for sparse inference weights.rz Found from : )rrrrrr)/r tree_flattenrrrGeneratorStater rlrrrcreate_symintnoderr\rr constantstorchbind_constantsr) get_valuer#rirrrrr4default_generators clone_stater rrrmrr r@_higher_order_opseffects with_effectsr/r4r0rrTensor is_sparsedisable_cudagraphs_reason)rr>rr binded_args args_flat tensor_argsnon_tensor_argsr3rr example_args device_indexnew_args new_kwargsexample_outputrRr node_meta_valctxexample_out_lir&msgrr)rrrrprocess_kernels           zExternKernel.process_kernelr`c Cspt|ts Jt|t|tr|S|}tj|}|dus$J| }|dur]d|j vr]t|tt t fr]t|j tr]|j djtjdsS|j djtjdr]|t|n|tj|dd\}}|d}||}tjj||}tjj||} tjj||} t|| | } || krt d| | |t!t|j"t#|$|%|| | dd d S) z In order to pass this to an extern kernel we need a ReinterpretView not a View. This allows us to avoid some unneeded copies. NrXr1rr{rz@convert_to_reinterpret_view failed: stride=%s offset=%s index=%sFr7rY)&rrrr`rrlrr?rrrmrrBr[rr r#rrrr,rr rAr2rrr3 stride_vars offset_varrdr1rrrZr]rr ) rr x_unwrap_viewr$x_unwrap_view_fx_node index_argsrrrrSr^expectedrrrconvert_to_reinterpret_viewsj         z(ExternKernel.convert_to_reinterpret_viewcCs|durtSt|ttjjjtfrt|dSt|t r-t j t j|j||dSt|tr4|St|tr?||jSt|trPt||j|dSt|tro|t|roz||WStynYnwt|trz||St|ttfr|S| |S)Nr)rrrY)!rcrr rrrrrrrrlradd_tensor_constantr#rRr'r r!rrr'rZr`rrrrrrrr NonTensorObjr}r*rrrr'Cs<            zExternKernel.realize_inputcCsDt|rt|dkr|S|D] }|dkr|Sq||Sr)rrr@r})rrrrrrrequire_stride1ds  zExternKernel.require_stride1Frrrrr rc s|dus dus JdvrsStrttrS|rGt|o-tj }tdd|r@t t j j jn||dStddd|dStttfr}|re|srr}tjr}dur{tSSt}trt|}trtdt|tr|r||srt|jrSttr|r|sLjrtjrɈSttrtjtrtjtst}rt|drt|jtsz!|j_|r|j ||dWSr|j!|dWSWn t"yYnwd} } durOt j j fd d t#t$D} | D] } t%j&j'(| d d qA|)tdd||d|rkt|siJS| r| durxduszJt%j&j'*| tSS) N)rr?TF)rr rr rzHthe MutationLayoutSHOULDREMOVE's real layout shouldn't be FlexibleLayoutrZrcs4g|]}|dr|dr|qS)rr)rLrrrrrrrrrsz0ExternKernel.require_strides..rr?)+rrrrrrr2rr\rrlrrsize_hints_or_throwr]r&rrQrrer>rArYrrrZrr`rrrCrrequire_stride_orderrequire_exact_stridesrrrr#rloweringslice_r}rP) rrrrr use_current_stride_ordermutation_layoutrAr expanded_dims orig_sizerNrrrrequire_stridesns                 zExternKernel.require_stridesrHcC|j|||dS)N)rr r)rrrr rrrrsz"ExternKernel.require_exact_stridesrcCr)N)rr r)rrrr rrrrsz!ExternKernel.require_stride_ordercC ||tSr)rrr*rrrrequire_channels_last rz"ExternKernel.require_channels_lastcCrr)rrr*rrrrequire_channels_last_3d$rz%ExternKernel.require_channels_last_3dcCs,ddd}||r |S||t|S)Nrrrrc Ss@z|}Wn ttfyYdSw|tjjvotjj|jSr)rAttributeErrorrrlrr is_mkldnn)rrrrris_mkldnn_tensor*s  z9ExternKernel.require_contiguous..is_mkldnn_tensorrrrrrrrr)rrrrrrrl(s  zExternKernel.require_contiguouscCs||t|Srrr*rrrrequire_contiguous_strides:sz'ExternKernel.require_contiguous_stridescCr rrrrrrrdBr zExternKernel.apply_constraintcCst|ts Jt|t|tst|}|jsJdt|}t|j}||krStd|j||t ||D]}|j|d}| ||vrJ||n|j|dq7|S)Nz/ExternKernel.arg_properties should not be emptyzv%s has %d unprovided positional arguments. Will check if they are in the keyword arguments or will use default values.rrw) rrrrrOrr1rrNrr)rrrn_args n_pos_argsrarg_namerrrfill_non_provided_argsEs(   z#ExternKernel.fill_non_provided_argsrmr}rc Cstjjrog}d}|r"|jr"t|jt|ksJddd|jD}t|jD]E\}}|durF|dus5J|||}|rC|dnd}nt|j|}|jr_|t|jkr_|j|dnd}| tjj ||q'|Sdd|jDS)NzDnames passed to codegen_const_args does not match self.constant_argscSsi|]}|d|qSr)rr2rrrrur]z3ExternKernel.codegen_const_args..rcSrorrlrrval_to_arg_strrrrrrrrqz3ExternKernel.codegen_const_args..) rlrrnrOrrErrrqrrr) rrmrname_to_arg_propertiesrrproptype_rrrrcodegen_const_argsis2  zExternKernel.codegen_const_argscCstjjr|jdur|g|j|j|j}d}n|j}d}g}t|D]4\}}tjjrN|j r6|t |j ks:Jd|j | d}| tjj ||q$| tjj |q$|rb|||S)NFTz-Invalid access to ExternKernel.arg_propertiesr)rlrrnrNrrqrErrrOrrrrrrr)rrqneed_codegen_constant_argsrrrrrrr codegen_argss&zExternKernel.codegen_argsrcKsT||vr ||S||jvr|j|S|j|}dur#|dSt|d)zGiven an argument name, queries for values in (in order): 1. any provided kwargs for this function. 2. the class self.kwargs member. 3. any available default arguments in self.allarg_properties.Nrwz not in self.allarg_properties)rrrPrY)rrrr3rrrget_kwargs_values    zExternKernel.get_kwargs_valueskip_outcCstjjrO|jdurt|jdkrgSg}|jD]5}|r |dkr q||}t|t r0| |q|j dus7J|j |i d}| tjj ||q|Sdd|jD}|S)NrrrcSs(g|]\}}|dtjj|qSror)rkr}rrrrsz/ExternKernel.codegen_kwargs..)rlrrnrNrrcrLrrr rrPrrrrr)rrrrr}rrrrcodegen_kwargss$     zExternKernel.codegen_kwargscCsT|jdur&|jj}t|dd}|dd}|ddd}|d|}|Sd}|S) Nrunknown_namespacersrtrmr?r unknown_op)rYr@rrqrsplit)rr@ op_namespaceop_namerrr get_op_names   zExternKernel.get_op_namec Cstjr=tjjs?t|dkrdStjj|}tjj| }| }| d| d|d|d|d dSdSdS)Nrzassert_size_stride(r^r/) r@ size_assertsrlrrnrgrrcodegen_shape_tupler@rrr)rrerrrrrrcodegen_size_assertss z!ExternKernel.codegen_size_assertsc Csttjr6tjjs8|}|tjjv}|}|r(|d|dt d|ddS|d|d|ddSdSdS)Nzassert_alignment(r^r/z # buffer z (op: z) is assumed to be not aligned) r@alignment_assertsrlrrnrrrrr`)rreralignedrrrrcodegen_alignment_assertss z&ExternKernel.codegen_alignment_assertscCs@tjjrtjjr dS||}|d|d|ddS)zc Track outputs of fallback operators if config.test_configs.track_memory_lifecycle Nz track_tensor(z, 'z')) r@ test_configstrack_memory_lifecyclerlrrn"write_memory_track_allocation_oncerr)rrerrrrcodegen_memory_trackings z$ExternKernel.codegen_memory_tracking'tuple[list[Sequence[Expr]], list[Expr]]cCs|}|}|gg|fS)zD get output sizes and strides, for template_codegen )rr@)rrrrrrget_group_strides zExternKernel.get_group_stridetuple[Expr, Sequence[Expr]]c stjj|}|}fdd|D}ddtt|Dttt||jdd}ddt |DfddttD}fd d|D| }|}tjj ||g\}}} t d \} t t|fd d|D} tt|| }|t|fS) zC Manually get canonicalization of the output index crr)rrH)rrrr rz-ExternKernel.canonicalize..cSsg|] }td|qS)r{)rerrrrrrqT)rr}cSrrrrrrrrrz-ExternKernel.canonicalize..crrrrrrrrrcrrrr)rrrrrccrrrrH)add_varrrrr)rlrrrr@rrrrrrrrKrrrhrrPr) rrrS index_orderrrr new_sizesrrr replacementr)rrrrr canonicalizes$   zExternKernel.canonicalizerucCsP|rtnt}t||}|jD]}|||O}q|jD]}|||O}q|Sr)maybe_free_unbacked_symbolsmaybe_free_symbolsrrrErr)rrumaybe_get_symbolsrr3rrrr#s   z!ExternKernel.get_free_symbol_usescsPtdd}d|g}|fddtD7}|dj|S)NrIzpython_kernel_name=cs$g|]}|jdt|jqSro)rr)rrMrrrr8sz(ExternKernel.__str__..rr)rrLfieldsrrr)r kernel_namerrrrr{3s   zExternKernel.__str__rNNNNrN)rr-r[r0rqrrErrrTrHrGrIr-rJr-rLrKrNrMrrrarbr)rerqrrrrJr-rr)rIr-rrr;)rrrr~)r>rzrr rr rr~)rrrr`r.)NNF) rrrrrrr rrrr?)rrrrHr rrr)rrrrr rrr)rrrrFrr)rmr}rrrr)rrrr rr )rrrr)rr)rrrH)>rrrrKrErrLrMrrrHrIrJrrLrNrOrPrQrRrSrr]r_rXrrirkrVrWr|rNr}rrrr'rrrrrrrlrrdrrrrrrrrrrrrWrr{rrrrrvrrs            %       y E   $       $ !      rcsBeZdZdddZ       dd fdd Zd!ddZZS)"ExternKernelOutrerqrrcC||dSr)generate_extern_kernel_outrjrrrrkDrzExternKernelOut.codegenrNr[rrqrrrErrrTrHrGrIr-rJrLrNrMc s`||} t| tsJt| td|| ||pid|||| tj||_ tj |dSr) rrrrrurrlrrrr) rr[rqrErrHrIrJrLrNunwrapped_inputsrvrrrGs  zExternKernelOut.__init__rcCrrkrrrrrr[dr zExternKernelOut.should_allocaterr)r[rrqrrrErrrTrHrGrIr-rJr-rLrrNrMrrr1)rrrrkrr[rrrrvrrBs rceZdZd fdd ZZS) RandomSeedscountrrrrrcsFttj}tjt|tj|gdg|j|j|ggddtj j ddS)Nrzaten.randint.low_outzat::_ops::randint_low_out::call)r[rqrErIrJrN) r#rrrRrurr]rrrlrandintlow_out)rrrlimitsrvrrris  zRandomSeeds.__init__)rrrrrrrrrrrrrrvrrhrcsJeZdZdddZ      dd fdd Zd!ddZd"ddZZS)#rCrerqrrcCrr)generate_extern_kernel_allocrjrrrrk}rzExternKernelAlloc.codegenrNr[r0rqrrrErrrTrIr-rJrLrNrMc sp||} tdd| DsJtd|ttt| ||pid|||| g|_tj ||_ tj |dS)Ncsrrrrrrrrrjz-ExternKernelAlloc.__init__..) rrrurrrrrrlrrrr) rr[rqrErrIrJrLrNrrvrrrs"  zExternKernelAlloc.__init__rcCrrrrrrrr[r z!ExternKernelAlloc.should_allocatecCrUrrVrrrrrdr z"ExternKernelAlloc.apply_constraintr)rNNNrN)r[r0rqrrrErrrTrIr-rJr-rLrrNrMrrr1r))rrrrkrr[rdrrrrvrrC|s  rCcsJeZdZdZdfd d Zdd d ZdddZdddZdddZZ S)rzP An output buffer that represents the mutation of a pre-existing buffer r[r0 mutated_noder mutating_noderRrrcsDtjd|d|}tj||g|_||_tj||_ dSrb) rurrrlrr@mutation_namesr rr)rr[r r mutated_node_namervrrrs  zMutationOutput.__init__cCrr)r rrrrrrzMutationOutput.get_defining_oprcCrr)r rrrrr%rz!MutationOutput.get_mutation_namesrcCrrrrrrrr[r zMutationOutput.should_allocaterrcCs |}dddd|DDS)NcSg|]}|dur|qSrrrrrrrs z7MutationOutput.get_mutation_buffers..cs|] }tj|VqdSr)rlrtry_get_bufferrqrrrrrz6MutationOutput.get_mutation_buffers..)r%)rr rrrget_mutation_bufferss z#MutationOutput.get_mutation_buffers)r[r0r rr rRrrr]rJr1rrr) rrrrKrrr%r[rrrrrvrrs  rcs`eZdZUdZiZded<edd d Zedd d Zdfdd Z dddZ dddZ Z S) TMADescriptorad An IR node representing a generic host-side TMA descriptor in the Triton API Mostly useful for user-defined Triton kernels relying on host-side TMA; but can, in principle, be used for Inductor's Triton templates, too. See TMADescriptorExperimental and TMADescriptorStable for the two implementations (the old API and the new API) zdict[Any, TMADescriptor]_CACHErRrtma_metatuple[str, tuple[Any, ...]]rcCsTt|dksJ|ddkrt|g|dRS|ddks Jt|g|dRS)Nrr experimentalr?r)rTMADescriptorExperimentalTMADescriptorStable)rrRrrrr _create_impls  zTMADescriptor._create_implcCs2t||f}||jvr||||j|<|j|Sr)idrr)rrRrrrrrrs   zTMADescriptor.createrqrrErcsVtdtt||dttt|t|d||_ t j ||_ t j |dS)NrY)rurr&r`rrrrrrRrlrrrr)rrRrqrErvrrrs zTMADescriptor.__init__rerqcCrr)generate_tma_descriptorrjrrrrkrzTMADescriptor.codegencCrr)rRrrrr get_tensorrzTMADescriptor.get_tensor)rRrrrrr)rRrrqrrErrrrr>) rrrrKrrrrrrrkrrrrrvrrs     rcs&eZdZdZ ddfd d ZZS)rz the new host-side TMA Descriptor API: (the ones obtained via create_{1d,2d}_tma_descriptor calls). See also TMADescriptorStable for the new API. NrRrrBlist[Union[int, torch.SymInt]] block_dims element_sizerrrcst|dvsJt|t|ksJ|dur|j}||_||_||_t|j|_|g}g|j|j|j}tj|||ddS)N)r?rrRrqrE) rr rrBr r!rrur)rrRrBr r!rqrErvrrrs*   z"TMADescriptorExperimental.__init__r) rRrrBrr rr!rrrrrrrKrrrrrvrrs rcs"eZdZdZdfdd ZZS) rz the new host-side TMA descriptor API (the ones obtained via TensorDescriptor.from_tensor). See also TMADescriptorExperimental for the old API. rRr block_shapercs||_tj||g|ddS)Nr")r$rur)rrRr$rvrrr8s  zTMADescriptorStable.__init__)rRrr$rr#rrrvrr0srcs(eZdZdfd d ZdddZZS)SubgraphBufferr[rrr\rfrgexample_inputs list[Any] subgraph_namerc s,td||||_||_tj||_tj|tj |j|||_ t |j s,Jt |j }|D]}||j j|j<|j j|jq3dd|D|_ddlmm}t|j 0|jdddd|j j|jWdn1swwYWddSWddS1swYdS)NcSsg|]}|jqSrr)rsym_varrrrrZsz+SubgraphBuffer.__init__..rFATEN) max_autotunemax_autotune_gemmmax_autotune_gemm_backends)rurrfr&rlrrrr make_subgraphsubgraphr rqrxrgraph_input_namesr sym_inputstorch._inductor.configrr@set_graph_handlerrrun) rr[rrfr&r(r1sym_inpinductor_configrvrrrCs0  "zSubgraphBuffer.__init__rerqrrcCsRGddd}t|jsJdd|jD}|||jg|j||jgdS)Nc@r^)z,SubgraphBuffer.codegen..CodegenGraphrrrcSs||_|j|_dSr)rr)rrrrrris z5SubgraphBuffer.codegen..CodegenGraph.__init__N)rrr)rrrrrrrr CodegenGraphhr_r7cSg|]}|qSrrrr&rrrrnrz*SubgraphBuffer.codegen..)r rq'codegen_subgraph_with_flattened_outputsr/r1r)rrer7 outer_inputsrrrrkgs zSubgraphBuffer.codegen) r[rrr\rfrgr&r'r(rrrrrrrkrrrrvrr%Bs$r%csneZdZd!ddZed"dd Zed d#d$fdd Zd%ddZd&fdd Z d'ddZ d(dd Z Z S))UserDefinedTritonKernelr(tuple[Kernel, Any, list[str], list[str]]csddlm}ddlm}||jg}g}g}t|retdr0|fddj Dn tds7J|j tdrRj D] }| j j|qEn td sYJ|jj}j |||fS) Nr) Autotuner)kernel_side_table restore_idxc3s|] }jj|VqdSr)r arg_namesrr>rrrr&zBUserDefinedTritonKernel.get_kernel_and_metadata.. restore_value reset_idx reset_to_zero)triton.runtime.autotunerr@*torch._higher_order_ops.triton_kernel_wraprA get_kernel kernel_idxrrrrBrErFrrrCrGconfigs)rr@rArLrestore_value_argsreset_to_zero_argsrrrDrget_kernel_and_metadataws,           z/UserDefinedTritonKernel.get_kernel_and_metadatarerqrc sddlm}\}}}||j||j\}}}fddjD} tdr1tds7Jtt fddj D} g} g} g} g}t | tt d |D]w\}}|| vre|reqY| |||t|tr| || |qYt|ttttjfr| || t|qY|| vr| d | tqY|d ur |r| d | tqY| |qYtd t|d |||j|| | || |djj d d S)YOverrides the parent member. See https://github.com/pytorch/pytorch/issues/151692r)triton_version_uses_attrs_dictcsi|]}||qSrrrrrrrrr]z3UserDefinedTritonKernel.codegen..rC constexprsc3s|]}j|VqdSr)rCrrDrrrrjz2UserDefinedTritonKernel.codegen..rrBNzUnsupported arg type: r.T) arg_typesraw_argsraw_keys triton_metar6roriginal_fxnode_name)!torch._inductor.utilsrQrO!define_user_defined_triton_kernelrgridrLrrr:rTrJchainrrrepeatrrrrr rrrrr rrrigenerate_kernel_callr!rYr)rrerQrLrMrNnew_namerXextra_launch_args named_argsconstexpr_namesrrUraw_keys_filteredraw_args_filteredrr3r)r>rrrks              zUserDefinedTritonKernel.codegenFrurrcst|t|j|BSr)rurr(r\rrvrrrs z,UserDefinedTritonKernel.get_free_symbol_usescCr^rr9rrrrr_rz0UserDefinedTritonKernel.get_unbacked_symbol_defsrKrr\r tma_descriptor_metadatarF kernel_argscszg}i}g}D]0\}} t| tr1t| } ||vr't| ||} || | ||<q || | ||<q t |dksCJ|d _ t|t sUJt |tdtj d|t|||_|_\} } } } t| dszJfdd| jD_ddlm}t | dkr| djni}fdd|| i||D_fddjD_tjdS) Nrr|rCcsg|]}|vr|qSrrr2rgrrr&rz4UserDefinedTritonKernel.__init__..)identify_mutated_tensorscrrrrrrhrrr-scs g|] }ttjd|qSr)rr9rrrrrr4s) rrrrr(r'rrrrr!rrrrurr9rrKr\rOrrCrLrIrir mutable_argsrSrlrr)rrKr\rfrgrqrrErr}r&r>rLrriautotuned_kwargsrv)rgrrrsP          z UserDefinedTritonKernel.__init__r\cCs t|jSr)rrSrrrrr]:rz#UserDefinedTritonKernel.get_outputsrcCrrr|rrrrr!=rz"UserDefinedTritonKernel.get_device)rr?rr?rHrb) rKrr\r rfrFrgrFrrrar6) rrrrOrrkrWrr_rr]r!rrrrvrr>vs  Y  =r>csJeZdZdZdddZdd d Zdd d ZdddZdfdd ZZ S)InplaceBernoulliFallbackE This needs to be a custom class to handle mutation properly rerqrrc Cstdd|jDs Jdd|jD\}tjjr5||d|ddtt |j d|j dS||d|ddtt |j d|j dS)Ncsrrrr:rrrrGrjz3InplaceBernoulliFallback.codegen..css|] }tt|VqdSr)rrrr:rrrrHrrr^z, NULL)r/) rrqrlrrnrr|rrreprrEending)rrerrrrrkFs,,z InplaceBernoulliFallback.codegenrcCrrrrrrrr[Ur z(InplaceBernoulliFallback.should_allocatercC |dgSrr#rrrrr%Xrz+InplaceBernoulliFallback.get_mutation_namesrcCr^rr9rrrrr_[rz1InplaceBernoulliFallback.get_unbacked_symbol_defsrNrzrrrEr csVtjdt|d||g||dtj|tj ||_ tj |dS)Nr|rN) rurr9r!rrlrr@rrrr)rrNrrErvrrr^s  z!InplaceBernoulliFallback.__init__rr1rJrb)rNrzrrrEr rr rrrrKrkr[r%r_rrrrrvrrmA    rmcs\eZdZdZd ddZd!d d Zd"d d Zd#ddZd$fdd Ze d%d&ddZ Z S)'InplaceCopyFallbackrnrerqrrcCs |\}}}||||dSr)rcodegen_device_copy)rrerFrE non_blockingrrrrksszInplaceCopyFallback.codegenrcCrrrrrrrr[wr z#InplaceCopyFallback.should_allocatercCrqrrrrrrrr%zrz&InplaceCopyFallback.get_mutation_namesrcCr^rr9rrrrr_}rz,InplaceCopyFallback.get_unbacked_symbol_defsr[r0rqrrrErcsJtjd|||dddtj|dtj||_tj|dS)Nz aten.copy_aoti_torch_copy_)rIrJr) rurrlrr@rrrr)rr[rqrErvrrrszInplaceCopyFallback.__init__FrFrrErxcs6fdd||fD}|f}tt|d||}|S)Ncrrrr:rrrrrz.InplaceCopyFallback.create..r|)rvr9r!)rrFrErxrqrErrrrrs zInplaceCopyFallback.createrr1rJrb)r[r0rqrrrErrrr?)rFrrErrxrrrv) rrrrKrkr[r%r_rrrrrrrvrrvns    rvc@sBeZdZdZdddZdd d Zdd d ZdddZdddZdS)MutatingFirstArgExternKernelrnrerqrrcCsXt|jsJgdd|jDtt|j}||dd|d|jdS)Ncs|]}|VqdSrr9r:rrrrr(z7MutatingFirstArgExternKernel.codegen..rr^r/) r rqrrorErr|rrp)rreargrefsrrrrks z$MutatingFirstArgExternKernel.codegenrcCrrrrrrrr[r z,MutatingFirstArgExternKernel.should_allocatercCrqrrrrrrrr%rz/MutatingFirstArgExternKernel.get_mutation_namesrcCr^rr9rrrrr_rz5MutatingFirstArgExternKernel.get_unbacked_symbol_defscCrrkrrrrrhas_side_effectsr z-MutatingFirstArgExternKernel.has_side_effectsNrr1rJrb) rrrrKrkr[r%r_r}rrrrrzs   rzcr) ResizeStorageBytesvariablerr6rrrcst|ts Jdtjdt|d||g|fdtj | tj ||_ tj |d|_d|_t|tttfsFJt|tjj|j dS)NzTODO: dynamic shapesr|)rEz"inductor_ops.resize_storage_bytes_z&torch::inductor::resize_storage_bytes_)rrrurr9r!rrlrr@rrrrrIrJrrrrnever_reuse_buffersrrZ)rrr6rvrrrs   zResizeStorageBytes.__init__)rrr6rrrrrrrvrr~r r~cs(eZdZd fdd Zd d d ZZS) SetSourceTensorKernel self_tensorrstorage_tensorrrcs|tj|||gdtjjjjdt |t t t fs$Jt |tjj|jtjj|tjj||}tt|d||tt|d||g|_dS)Nz!torch.ops.aten.set_.source_Tensor)rIrNr|)r rurrr#rjrlset_ source_TensorrrrrrrlrrrrZrr!rr9rS)rrrrrvrrrs"  zSetSourceTensorKernel.__init__rcCs|d|dgSrrrrrrrr'rz2SetSourceTensorKernel.get_inputs_that_alias_output)rrrrrrrJ)rrrrr'rrrrvrrsrcsReZdZdZd!ddZd"d d Zd#d d Zd$ddZdddd%fdd ZZ S)&ScatterFallbackz This needs to be a custom class to handle mutation properly. This class handles both aten.scatter_ and aten.scatter_reduce_. It also handle the case `src` being a scalar properly. rerqrrc Cs|jd}tjjrddd}||vr||}t|jsJ|jr,dd|jD\}}}ndd|jD\}}|jd}||||jd ||g|j |j |j|| dS) NrIrr)rmultiplycsr{rr9r:rrrrr(z*ScatterFallback.codegen..csr{rr9r:rrrrr(r?r) rrlrrnr rq src_is_tensorrEgenerate_scatter_fallbackrJrIr)rrerIget_operator_enumrrrErrrrks&   zScatterFallback.codegenrcCrrrrrrrr[r zScatterFallback.should_allocatercCs"|jd}t|ts J|gSrr!)rrwrrrr% s  z"ScatterFallback.get_mutation_namesrcCr^rr9rrrrr_rz(ScatterFallback.get_unbacked_symbol_defsNTrI include_selfrNrzrrrNrrrErIr-rc st|t_jrfdd|||fD}|f} nfdd||fD}||f} tjdt|d|| ||dt|ddg|dt j | t j _t j dS) Ncrrrr:rrrr rz,ScatterFallback.__init__..crrrr:rrrr#rr|rrIr)rIrLrN)rrrrurr9r!rrrlrr@rrrr) rrNrrNrrErIrtensorsrErvrrrs&   zScatterFallback.__init__rr1rrb)rNrzrrrNrrrrErrIr-rrrrrtrrrvrrs     rcsJeZdZdZdddZdd d Zdd d Zd ddZd!fdd ZZ S)"IndexPutFallbackzQ This needs to be a custom class to handle mutation and indices properly rerqrrc Cst|jsJdd|jD^}}}g}t|}t|jD]\}}|j|dur0|t|q|tjj j q|j | |||g| RdS)Ncsr{rr9r:rrrr<r(z+IndexPutFallback.codegen..)r rqrrr"rrrlrrrdgenerate_index_put_fallbackr|r) rrerr valid_indicesr"iter_valid_indicesrrrrrrk:s  zIndexPutFallback.codegenrcCrrrrrrrr[Ir z IndexPutFallback.should_allocatercCrqrrrrrrrr%Lrz#IndexPutFallback.get_mutation_namesrcCr^rr9rrrrr_Orz)IndexPutFallback.get_unbacked_symbol_defsrNtorch._ops.OpOverloadrrr"r'rr accumulater c s|_dd|D}fdd||g|D}d}tjdt|d||fd||dtj dtj _ tj dS) NcSrrrrrrrr[rz-IndexPutFallback.__init__..crrrrHrrrr\raoti_torch_index_put_outr|zaten.index_put_)rIrJrNr) r"rurr9r!rrlrr@r#rrr) rrNrr"rrrrrJrvrrrRs   zIndexPutFallback.__init__rr1rJrb) rNrrrr"r'rrrr rrrtrrrvrr5rurc@s$eZdZeddd Zdd dZdS) DeviceCopyrrrrrxrrc Cs|stdd|Drtjjs||Stj || }|dus(Jtj |t d|f}t |}d}|rD|}t|joO|jdkoO|}|jdko[t|jo[|}|rgt|rgd|_tt|||||d||g|S)Ncss|] }|tjjvVqdSr)rlrrrrrrrqrz$DeviceCopy.create..zDeviceCopy in input programr3Tr%)r"rrr@ aot_inductoruse_runtime_constant_foldingr$rlradd_device_infor!r[rrlrr@rcrrrr_rr]r r') rrrrxx_devicerEris_destination_pinnedis_source_pinnedrrrrmsD        zDeviceCopy.creatererqrcCs\|}t|dks J|jr||d|j|ddS||d||ddS)Nrrr?)rrrHrwr)rrerrrrrkszDeviceCopy.codegenN)rrrrrxrrrr)rrrrrrkrrrrrls )rcs`eZdZdZdddZdddZd fdd Zd!ddZed d"d#ddZ d$ddZ Z S)%DynamicSelectStorageOffseta The result of computing a dynamic selection index is determined as follows: when the index in the select operation is unbacked, the actual index calculation is ambiguous for negative indices (index + size) versus non-negative indices (just index). To resolve this, we allocate an unbacked SymInt to represent the storage offset and decompose the select operation into a call to as_strided, computing the storage offset at runtime with this node. rrcCr^rr9rrrrrrz$DynamicSelectStorageOffset.get_readsrcCrrrrrrrr[r z*DynamicSelectStorageOffset.should_allocateunbacked_offset_symbol sympy.Symbolr base_offsetUnion[sympy.Symbol, int]base_dim_striderrcs>tdttddg||_||_||_||_||_ dSNr3r|) rurr9r#rrrrrr)rrrrrrrvrrrs  z#DynamicSelectStorageOffset.__init__rcC t|jgSr)r:rrrrrr_rz3DynamicSelectStorageOffset.get_unbacked_symbol_defsFrucCrfr)r(rrrrrrrgz/DynamicSelectStorageOffset.get_free_symbol_usesrerqcCrr)codegen_dynamic_select_indexrjrrrrkrz"DynamicSelectStorageOffset.codegenrEr1) rrrrrrrrrrrrrbr?rHr) rrrrKrr[rr_rWrrkrrrrvrrs   rcsJeZdZdZdddZdddZdfdd ZdddZdddZZ S)rz; The result of a call to aten._local_scalar_dense. rrcCr^rr9rrrrrrzDynamicScalar.get_readsrcCrrrrrrrr[r zDynamicScalar.should_allocatesymrkeypathpytree.KeyPathrZrrcs<|tdttdd||g||_||_dSr) rrurr9r#rrrr)rrrrZrvrrrs  zDynamicScalar.__init__rcCrr)r:rrrrrr_rz&DynamicScalar.get_unbacked_symbol_defsrerqcCrr)codegen_dynamic_scalarrjrrrrkrzDynamicScalar.codegenrEr1)rrrrrZrrrrbr) rrrrKrr[rr_rkrrrrvrrs   rcs`eZdZdZdddZdddZdfdd ZdddZed ddddZ d ddZ Z S)!rz5 The result of a call to aten._assert_scalar rrcCr^rr9rrrrrrzAssertScalar.get_readsrcCrrrrrrrr[r zAssertScalar.should_allocatescalarrnrrrcs,tdttddg||_||_dSr)rurr9r#rrr)rrrrvrrrs zAssertScalar.__init__cCrrkrrrrrr}r zAssertScalar.has_side_effectsFrurcCrfr)r(rrrrrrrgz!AssertScalar.get_free_symbol_usesrerqc CstjsdStt|jdd}tjjrdStjjr;d|d}tjj j |j dd}| d|d|j d|d dStjj j|j dd}| d |d | d t|j d| |d dS)NFrtzstd::to_string(r/)rizif (!(z()) { throw std::runtime_error("Expected z but received " + z); }zif not (z):z raise RuntimeError(z = None)r@scalar_assertsrrrrlrryrnrcodegen_cpp_sizevarrrrcodegen_python_sizevarror)rresymbol symbol_strsizevarrrrrk s& zAssertScalar.codegenrEr1)rrnrrrrr?rHr) rrrrKrr[rr}rWrrkrrrrvrrs   rc@seZdZUded<ded<dS)ExternKernelNoderrzexport_schema.NoderNrrrrrr*s  rcseZdZdZ d>ddd?fddZd@fdd ZdAddZdBddZdCd!d"Ze dDd&d'Z dEd)d*Z dFd,d-Z dFd.d/Z d0d1ZedAd2d3Ze dGd7d8ZedHd:d;ZdIfdrzrrrnontensor_argsrrrdrrTrR,Optional[dict[sympy.Symbol, pytree.KeyPath]]rrc s`tj|t|t||dd_|pi_t|tjjtjj fs.Jd|dt |d|_ |_ |dur:in|_ jdusDJtjjg_g_tj tjj r[dSdj vrddSj j}tjjj r|j|ddS|jrt|std| jj\} }dfdd } tjj || |D] \} } | | | qdS)NrsF#Fails to create FallbackKernel for r. not supported_c10d_functionalrz'NYI: Can't generate FallbackKernel for infotorch._C.Argumentr3r rrcstjtjrt|ttfsJt|tjr#t|ttfr#J|dur)dSjdur0dSdfdd }t jrO|durK|D]}||qDdSdStjsWJ||dS)Nr&rrrcsLj|jdusJjjr$jtt|d|dSdSre) alias_namesrr alias_infois_writerSrr9r!r&)rrrr add_aliasszPFallbackKernel.__init__..handle_aliasing_and_mutation..add_alias)r&rrr) rrr#ListTyperr library_utilsis_tensor_like_typeris_tensorlist_like_type)rr3roptional_tensor_argr)rrhandle_aliasing_and_mutations"     z=FallbackKernel.__init__..handle_aliasing_and_mutation)rrr3r rr)!rurruse_runtime_dispatchrRrr#r_r`rurrNrrrIrlr warn_fallbackrr rra_libraryr+mutates_and_returns_first_argrr is_mutabler%rrqrE zip_schema) rr[r>rrrrrRschemarrrr3rvrrr7sJ   zFallbackKernel.__init__rcsHt}|jtjjjur"|jD]}t|t r!| t | }q|Sr)rurrNr#_prims rng_primsgraphsafe_run_with_rng_staterErr with_readrArzr)rr!r3rvrrrs    zFallbackKernel.get_read_writesrerqcCs|||jt|ddSNrR)(codegen_unbacked_symbol_defs_for_outputsrrrrjrrrcodegen_unbacked_symbol_defssz+FallbackKernel.codegen_unbacked_symbol_defsContainer[sympy.Symbol]cCs:t|dd}rttjjj|}|dusJ|StSr)rr5rlrrrr5r:rrRresolvedrrrr_s  z'FallbackKernel.get_unbacked_symbol_defsrcstjGdddt|jsJfdd|jD}|||j\}}tjjrDt |j t j j rD|||}ddt|j jj|D}ndd|D}|j||S)Nc@ eZdZUded<dddZdS) z)FallbackKernel.codegen_args..Shimr refrrcSrr)rrrrrrrz2FallbackKernel.codegen_args..Shim.__repr__Nr;)rrrrrrrrrShim rcsg|]}|qSrr9rHrrrrrz/FallbackKernel.codegen_args..cSs"g|] \}}tjj||jqSr)rlrrrr[)rparamrrrrrscSrorrrHrrrrrq)rL dataclassr rqrrErlrrnrrNr#r_r`rrrarbrr)rrrrrrrrs  zFallbackKernel.codegen_args Optional[Sequence[torch.Tensor]]rr cCs|r dd|Dnd}|r|sJdd|D}|dSt|tjr%|jSt|ttfr`tdd|D}dd|D}t|dkrF|dS|D]}t|tjsRJt|j r[|SqH|dSdS) NcSsg|] }t|ts|qSr)rr)r:rrrrrqz.FallbackKernel.find_device..cSsg|] }|r|qSrrr2rrrrrQrcss|] }td|VqdSr)r find_devicerHrrrrr&z-FallbackKernel.find_device..cSsg|]}|r|qSrr)rrrrrrrr?) rr#rrrrr:rrcr)rrnon_torch_bind_tensor_argsdevices device_setrrrrrs0   zFallbackKernel.find_devicercCs"t|jtjjr dSt|jSr)rrNr#r_rur.rrrrrr}szFallbackKernel.has_side_effectsrcCsrt|jtjjtjjfsJd|jdt|jdt|jtjjs6d|jvr6|jjj r6t |jr6gS|j S)Nrr.rr) rrNr#r_r`rurrrarr%rrrrrr's   z+FallbackKernel.get_inputs_that_alias_outputcCst|jdks J|jSr )rr rrrrr% sz!FallbackKernel.get_mutation_namesc sftdjttsJtjj \} |}fddj D}j}t j js=g||Stdg}|||}dd d t|tjjjrb||d |dj}n|jj}t|dkrjrrjnj}|d j}||g} n fddt|jD} jdusJttjj || idd} t j!"| g||S)a ProxyExecutor Design Note We export the ExternFallbackNodes (for custom ops) into a serialized file and run it with a host side proxy executor to address the ABI problem This is currently only implemented for fbcode. Eventually, we will also make this work for OSS. Detailed design doc can be found at https://docs.google.com/document/d/1wC4DOZFaYym2t1Esz0X5yxlLI3RDnSiyRbUus3bkJ64/edit?usp=sharing z4Extern kernel node added for node %s with target %s.csg|] }j|fiqSrrRrjrrrrr"rz.N return_type6Union[torch.TensorType, torch.ListType, torch.JitType]rhUnion[IRNode, Sequence[IRNode]]rexport_schema.ArgumentcSsnt|tjtjfrE|}t|ttfrt|dksJ|d}t|tjr8t|ts+Jtj j tj | ddS|dus>Jtj j ddSt|tj rjt|tjrjt|ts^Jt|tj j dd|Dd St|tjrt|tjr|durtj j tjj ddd St|tsJtj j tjj tj | ddd St|tjrtj j |d Std t|) Nr?rr) as_tensorT)as_nonecSsg|] }tj|dqS)r) export_schemaTensorArgumentr)rrrrrrHszZFallbackKernel.export_extern_kernel_node..handle_single_output..) as_tensors)as_optional_tensor)as_intzUnsupported return type )rr# TensorTypeNoneTyperrrrrrorrrrgetElementTyperr OptionalTypeOptionalTensorArgumentIntType RuntimeError)rrhrrrrhandle_single_output0sR     zFFallbackKernel.export_extern_kernel_node..handle_single_outputrr?csg|] \}}|j|qSr)r[)r return_schemarh)rrrrrs)r@rqrmetadata)rr)rrrhrrr)#r1rrrNrrrrrqrErrLrlraot_moder$serialize_inputsr#r torchbind CallTorchBindrreturnsrarrrSr[rrrr8rextern_kernel_nodesr) rrordered_kwargsr@ serializernamed_argumentsrrroutput_argumentsrr)rrrrexport_extern_kernel_nodesN      5     z(FallbackKernel.export_extern_kernel_nodecsj}|dus J|jdkr5t|tjjsJt|tjj r4ddl m }t ||vr4t d|d_n|jdkrHt|tjjsGJt|n tjj rS|tjjv_tjj rt|tjjrjsdfd d jj\}t|fddjD}tfddt||jjD_|jrƈ}jdusJjdusJ| !jfddj|j"rj"nj#n|$tj%t&r'|(|)|*|dS)rPNrlr)inductor_fallback_opszG%s is missing a c-shim implementation, using proxy executor as fallbackT _quantizedr& torch.JitTyperrcs$t|tjr |St|tjSr)rr#rr NumberTyper is_numberrrrs   z)FallbackKernel.codegen..is_numberc3s"|] }j|fiVqdSrrRrSrrrrr|z)FallbackKernel.codegen..c3s(|]\}}t|to|jVqdSr)rcomplexr[)rr}rrrrrs  csgSr)rrrrrrrrz(FallbackKernel.codegen..)r&rrr)+rNrorr#r_r`rrlrrntorchgen.aoti.fallback_opsrrr1rrr@rcustom_ops_to_c_shimsrrqrErJr]rLrrrarbrirrI,generate_fallback_kernel_with_runtime_lookuprrrSgenerate_fallback_kernelr[rrrrr)rrer>rr args_iter exported_argsr)rrrrrksl                zFallbackKernel.codegenrhrr]cCsJd}z|}Wn tyYnwt|j|jt|t||dS)NFr%)r_rr]rrrYrr)rhr_rrrtensor_to_layouts    zFallbackKernel.tensor_to_layoutrc sPtjf}||vrttdtjj}nt}|j|g|Ri|\}}}} } Wdn1s3wYt dd|D ||} | sVt |t j jjrVt d} |durht| d|||| | dn| snJdt| d|||| | ddfd d |g} t | ttfr| _| St | trt| _| S| g_| S)z9Create an instance of FallbackKernel from an _OpOverloadsNcsrr)rr2rrrrr(z(FallbackKernel.create..r3r|rz"Not sure where to find device inforhr r"list[tuple[Any, int]]rcstttfrtfddttDSttr,fddDSttj rNt  }t j sDsDtsLtjj|j|SttrUSttjr_jjSdusmJdtddS)Nc3s,|]}|t|fgVqdSrrrgenerate_outputr"rhrrr$s  zAFallbackKernel.create..generate_output..cs*i|]\}}||t|fgqSrr)rrrXrrrr)rzBFallbackKernel.create..generate_output..zFallbackKernel output type z is not supported)rrrrrrrrr#r MultiOutputrr@ assume_unaligned_fallback_outputrirlrrrrrSymIntrr)rhr"r$rrhas_unaligned_inputpacked)r"rhrr"s<      z.FallbackKernel.create..generate_output)rhr r"rrr )rl*_fused_moving_avg_obs_fq_helper_functionalrrrlrrr rrrrr#rrrrr9rrrrr) rr>rrfake_incorrect_kernelscontextrrrrrRrrrrrrs^     " zFallbackKernel.createcs tSr)rurdrrvrrrdMrzFallbackKernel.apply_constraintrr[r0r>rzrrrrrrrdrrTrRrrrrDr)rrr)rrrrrr r1rJ)rhrrr])r>rzrr rr rrr))rrrrKrrrr_rrNrr}r'r%rrrkrrrrdrrrrvrr0s.  j      y U  ]rcs<eZdZdZdddZdddZd d dfddZZS) ComplexViewz9View a complex number as two dtyped numbers or vice versarrcCrrrrrrrr[Ur zComplexView.should_allocatercCrqrrrrrrrr'Xrz(ComplexView.get_inputs_that_alias_outputNrr[r0r>rzrrrrrrrdrRrrcstj||||||ddS)Nr)rur)rr[r>rrrrRrvrrr\s  zComplexView.__init__r1rJ)r[r0r>rzrrrrrrrdrRrrr)rrrrKr[r'rrrrrvrrQs    rc@seZdZdZd ddZdS) MemoryCheckKernelz Custom kernel for memory checking that generates direct function calls TODO - the custom op was erroring with str inputs. should be able to custom op directly. rerqrrcCsl||j\}}}t|}t|}|r&|dd|d|d|d}n d|d|d}||dS)z.Override codegen to write direct function callzV# note: dont currently distinguish between buffers returned and dealloc'd in last stepzcheck_memory_step(allocated=z, freed=z, is_final_step=r/N)rrEror)rre alive_list dead_list is_final_step alive_repr dead_reprcallrrrrkws zMemoryCheckKernel.codegenNr)rrrrKrkrrrrr psr c@r) rrrrrcCrrr|rrrrr!rzMultiOutputLayout.get_deviceNr6)rrrrr!rrrrrrrcsVeZdZdddZ ddfdd Zed ddddZdddZd ddZZ S)!rrerqrrcCs,|||js||||dSdSr)codegen_multi_output!skip_size_stride_alignment_checksrrrjrrrrks  zMultiOutput.codegenFr[r0r"rr"list[tuple[Any, ...]]r(rcs>td||gdtj||_tj|||_||_dSr) rurrlrrrrr"r()rr[r"r"r(rvrrrs   zMultiOutput.__init__rurcCs&|jd}t|tsJ|||Sr)rqrrr)rrurrrrrs  z MultiOutput.get_free_symbol_usescCst|jdkot|jdtS)Nr?r)rrqrrrrrrr[szMultiOutput.should_allocatercCsdd|jDS)NcSs.g|]}t|trt|dkr|qSr)rrrr'rrrwrrrrsz.)rqrrrrr'sz(MultiOutput.get_inputs_that_alias_outputrr?) r[r0r"rr"r)r(rrrrHr1rJ) rrrrkrrWrr[r'rrrrvrrs    rc@seZdZUdZded<dvddZdwd d Zdxd d ZdyddZdzddZ d{ddZ d|d}ddZ d~dd Z dd!d"Z dd#d$Zdvd%d&Zdd'd(Z )ddd-d.Zdd/d0Zdd2d3Z )ddd5d6Zdd8d9Zdd;d<Zdd=d>Zdd@dAZddCdDZddFdGZdvdHdIZdvdJdKZddNdOZddQdRZd{dSdTZddUdVZ ddWdXZ!e"d )ddd[d\Z#dd^d_Z$ddadbZ%d|ddedfZ&e'ddhdiZ(ddkdlZ)ddmdnZ*ddodpZ+e'ddrdsZ,d{dtduZ-e-Z.dS)rBzC TensorBox / StorageBox allow in-place mutation of Tensors rrZrrcCr(rr.rrrrrrz!MutableBox.has_exceeded_max_readsrcCr(rr+rrrrr!rzMutableBox.get_devicercCr(rrrrrrrrzMutableBox.make_loaderrcCr(r)rZrrrrrrrzMutableBox.make_indexerrHcCr(r)rZr@rrrrr@rzMutableBox.get_stridercCr(rr,rrrrrrzMutableBox.get_nameNrrcCrr)rZrrrrrrrzMutableBox.has_large_inner_fnrrrcCrrr-rrrrrrzMutableBox.mark_reusecCr(rr0rrrrrrzMutableBox.realize_hintcCr(r)rZrrrrrrrzMutableBox.unwrap_viewcCr(r)rZrrrrrrrzMutableBox.is_input_buffercCr(r)rZr rrrrr rzMutableBox.freeze_layoutFrrr cC|j||Sr)rZr r rrrr r z*MutableBox.freeze_layout_with_stride_ordercCrr)rZrrrrrrrz(MutableBox.freeze_layout_with_fill_orderrcCrr)rZrrrrrrrz(MutableBox.freeze_layout_with_same_orderrcCr+r)rZrrrrrrr z+MutableBox.freeze_layout_with_exact_stridesrcCr(r)rZrrrrrrrzMutableBox.get_read_writesrcCr(rrprrrrrrzMutableBox.get_readscCr(rrorrrrrrzMutableBox.num_readsrxcCr(rr1rrrrr rzMutableBox.get_storage_numelr-cCr(rrrrrrr  rzMutableBox.get_reduction_typercCr(rrrrrrr! rzMutableBox.get_reduction_sizecCr(rr2rrrrr" rzMutableBox.is_externcCr(r)rZr#rrrrr# rzMutableBox.is_no_oprrcCrrrrrrrr$ rzMutableBox.constant_to_devicercCr(r)rZr%rrrrr% rzMutableBox.get_mutation_namescCr(r)rZr&rrrrr& rzMutableBox.get_operation_namecCr(r)rZr'rrrrr' rz'MutableBox.get_inputs_that_alias_outputcCr(rr/rrrrr rzMutableBox.realizerurcCrrrrrrrr rgzMutableBox.get_free_symbol_usesrcCr(rr4rrrrr& rzMutableBox.get_read_namesrcCr(r)rZrrrrrr) rzMutableBox.get_defining_oprrcCrr)rZrrrrrr, rzMutableBox.codegen_referencer0cCr(rrZrrrrrr[/ s zMutableBox.layoutrcCr(rr*rrrrr4 rzMutableBox.get_layoutcCr(rr,rrrrr7 rzMutableBox.get_output_speccCr(rrrrrrr: rzMutableBox.get_sizercCrr)rZrrrrrr= rzMutableBox.dtypecCsnt|jtrt|jdt|jjd}d}|jj}n t|jd}|j}d}|tt||g}d|S)Nrz))r/r)rrZrBrrrrr)rline0endlr$rrrrr{A s    zMutableBox.__str__r1r6r8r9r:r;rr<r=r)r>r?r@rArBrCrDrErFrGr4r2rIrJrHr*r,r5r0r/r.)/rrrrKrrr!rrr@rrrrrrr r rrrrrrrr r!r"r#r$r%r&r'rrWrrrrrQr[rrrrr{rrrrrrBsb                                     rBc@seZdZedddZdS) rrZrrr~cCst|tr|Stt|Sr)rrrr)rZrrrrV s  zTensorBox.createN)rZrrr~)rrrrNrrrrrrU sc@sjeZdZdZdddZdddZdd d Zdd d ZdddZdddZ d ddZ d!ddZ d"ddZ dS)#rz7 StorageBox allow in-place mutation of Tensors rrcCs&t|jttfr|jtjjvSdSr)rrZrr`rrlrrrrrrrb szStorageBox.is_input_buffercCst|jto|jtjjvSr)rrZrrrlrrrrrrr3g s zStorageBox.is_module_bufferr-cCst|jr |jSt|jttttfsJt |j|j }|j }|j }|dus1Jt dt||j|jdd|jd|_tj|j|j_tj|j|j|j_||j_||j_|jjS)NF)rrrr_r)rrrZrrrrrrrrrr!rrr rrlrrrrr|rr~)rrr~rrrrrm s2        zStorageBox.realizercCs4t|jttfr|jjdkr|dSdSdS)zL Called on buffers we expect to be forced to realize later. r?N)rrZrrrnontrivial_read_countrrrrrr s  zStorageBox.realize_hintrrcCstdd|D|kS)Ncsrr)rlrget_dep_size_hintrrrrr rzBStorageBox.has_accumulated_enough_reads_by_size..)rrrrrr$has_accumulated_enough_reads_by_size sz/StorageBox.has_accumulated_enough_reads_by_sizecCs8t|jto|tjkp|ptjduo|tjSr) rrZrrr@realize_acc_reads_thresholdr realize_acc_reads_size_thresholdr1rrrrr s  z!StorageBox.has_exceeded_max_readsrcsh|dkr2t|jttfr2t|jr'|jddg}tfdd|Dr'dS|tj kp1| SdS)zj A heuristic to decide if we should realize a tensor that is used multiple times. r?expsigmoidc3s|]}|jvVqdSr)used_opsrHopcountrrr rjz5StorageBox.should_realize_on_reuse..TF) rrZrrr<rrrr@realize_reads_thresholdr)rr heavy_opsrr7rshould_realize_on_reuse s  z"StorageBox.should_realize_on_reusecCs||r |dSdSr)r;rrrrrr s  zStorageBox.mark_reusecCr(rrorrrrr rzStorageBox.num_readsNr1r4r))rrrr)rrrrr=rF) rrrrKrr3rrr1rr;rrrrrrr] s      rc@s*eZdZUded<ded<dZded<dS)Subgraphrrrg graph_moduleNzOptional[GraphLowering]r)rrrrrrrrrr< s r<bufferscCs,dd|D}ttdd|Dt|kS)NcS"g|] }t|tr |n|qSrrr`rrrrrrr z(_has_aliased_buffers..csrr)rrArrrr r(z'_has_aliased_buffers..)rr:)r>rrr_has_aliased_buffers srCcs`eZdZUdZdZded<dZded<dZded<dfdd Ze dddZ dddZ Z S)InvokeSubgraphz. Ir node for the invoke_subgraph HOP. NOptional[Subgraph]r/Optional[Sequence[IRNode]]operandsrr<rrr[rrrcs6tjd||d||_tj||_tj|dSrU)rurr/rlrrrr)rr/rGr[rvrrr szInvokeSubgraph.__init__rElist[Union[ShapeAsConstantBuffer, NoneAsConstantBuffer, MultiOutput]]csddlm}tjj}d}|jd}r|ddd}n|jdd}dd|D}fd d|D}g}t|D]\} } t | t t frK| | q:| || || q:|}|jdurtjj |j||jd |_t|j|jj|Wdn1swY|jj} d} |D] } t | t s| } nq| dusJt||t| d d dfdd fddt| D} | _| S)zFor each operand, get a realized input, force it to have the same strides as the subgraph inputs, then use an InvokeSubgraphr?)constrain_to_fake_tensorNeager_input_valsrrcSg|]}|jdqSrZrmrHrrrr rz)InvokeSubgraph.create..crrrrHrrrr rrfr&r(r|)r/rGr[rhrindrr?Union[ShapeAsConstantBuffer, NoneAsConstantBuffer, MultiOutput]c sdt|ttfr |S|}|dusJtt||||| j | j dt |fgddS)Nr7T)r() rrrcr!rr]r rr@rr^r_r)rhrNr)invoke_subgraphrr create_output$!s" z,InvokeSubgraph.create..create_outputcsg|] \}}||qSrr)rrrh)rQrrr;!rq)rhrrNrrrO)rrIrlrrrmrrrrrrrr.r=rr3r4 graph_outputsr!rDrr)rr/rGrIr fake_operandsrJ fx_operands new_operandsroperandrroutsr)rrQrPrr sT      zInvokeSubgraph.creatererqcCrr)codegen_invoke_subgraphrjrrrrk?!rzInvokeSubgraph.codegen)r/r<rGrrr[rrr)r/r<rGrrrHr) rrrrKr/rrGrrrrrkrrrrvrrD s     YrDcseZdZUdZded<dZded<dZded<dZded<dZd ed <d(fdd Z e d)ddZ e d*dd Z d+d#d$Zd,d&d'ZZS)- ConditionalNr predicaterFrGrE true_subgraphfalse_subgraphOptional[Sequence[MultiOutput]]rrrrr<r[rrRrrrc sj||_||_||_||_t|g|\}}tjd|||d|dur&||_tj ||_ tj |dSN)rr[rqrE) rZrGr[r\_split_by_sym_typerurrRrlrrrr) rrZrGr[r\r[rRsym_argsrrvrrrK!s zConditional.__init__rUnion[int, torch.SymInt]Union[int, sympy.Expr]cCst|tr|S|jjSr)rrrr)rrrr _maybe_exprg!s zConditional._maybe_exprrtrue_fnfalse_fn-list[Union[TensorBox, ShapeAsConstantBuffer]]c sX|}fdd|D}tjjjd}t|ts Jt|tdd|Ds+Jdd|D}||fD]/}|jduretjj |j ||j d|_t |j|jj |Wdn1s`wYq6|jdusmJ|jdustJ|jj}|jj} d |fd | ffD]\} } t|rtd | d | qt|t| ksJ|| ftt|| D]5\} \} }| |ksJ| | |f| |ksJ| | |f| j|jksJ| | |fqtd d|g|Dttjjjtjjjdd}dusJdt||||td|dfddtt|tjjjdD} | _ | S)zNCreate a Sequence of IRNodes from a conditional statement (see .lowering.cond)crrrrHrrrrw!rz&Conditional.create..rBcsrr)rr8r\rrrr{!rjz%Conditional.create..cSsg|] }tt|jdqSrZ)rr8rmrHrrrr|!rQNrMrdrezVOutput aliasing is currently not supported in compiled torch.cond. The outputs of the z% subgraph of torch.cond are aliased: css"|] }t|ts|VqdSr)rrr!)rorrrr!s rRzcannot determine devicer|)rZrGr[r\r[rRc sbg|]-\}\}}tt|dd|Ddd|D|j|jdt|fgqS)cSrNrrYrcrrrrrr!rz1Conditional.create...cSrNrrhrirrrr!s r7) rr]r rrrr^r_r)rrrh merged_output) conditionalrrrr!s"  rX)!r'rlrrrrrrrr.r=rr3r4rRrCrYrrrr!r rr^rr5rrrmrrYrr)rrZrdrerGrTrSr/ true_outputs false_outputsrrrt_of_orRr)rrkrrrm!sv   $ zConditional.creatererqcCs*|||||jt|didSr)codegen_conditionalrrrrrjrrrrk!s zConditional.codegenrcC>t|dd}rttjjj|}|dusJt|StSrrr5rlrrrr:r5rrrrr_!   z$Conditional.get_unbacked_symbol_defs)rZrrGrrr[r<r\r<r[rrRrrr)rrarrb) rZrrdr<rer<rGrfrrrrrb)rrrrZrrGr[r\rrrNrcrrrkr_rrrrvrrYC!s        ZrYrr'-tuple[list[ShapeAsConstantBuffer], list[Any]]cCs<g}g}|D]}t|tr||jq||q||fSr)rrrr)r non_sym_argsr`r3rrrr_!s  r_cseZdZUdZdZded<dZded<dZded<dZded<dZ d ed <d%fdd Z e d&ddZ e d'ddZd(d d!Zd)d#d$ZZS)* WhileLoopzSThe IR node for while_loop and while_loop_stack_output. It supports input mutation.NrFcarried_inputsadditional_inputsrE cond_subgraph body_subgraphr]rrrr<r[rrRr stack_outputrrrc sr||_||_||_||_tg||\}} tjd|| |d|dur'||_||_t j ||_ t j |dSr^)rwrxryrzr_rurrRr{rlrrrr) rrwrxryrzr[rRr{r`rrvrrr!s$  zWhileLoop.__init__cCst|s|Sddlm}dd|D}t}g}tt||D] \}\}}t||vr3|||q|t|||q|S)Nr?)clonecSr?rr@rArrrr"rBz3WhileLoop._clone_aliased_inputs..) rCrr|r:rrrrr)rwr|unwrapped_buffers seen_buffersrroriginal_inputunwrapped_bufferrrr_clone_aliased_inputs"s    zWhileLoop._clone_aliased_inputscond_fnbody_fnrc& sddlm}d*d d }tjjjd }tjjjd } || } d d| D} dd|D} dd| D} fdd|D}t|}||| }fdd|D}||| }||||fD]S}|jdurt| t smJt | tjj |j | |j d|_t|j'|jj| ||urt|jjt| ksJ||jj| |j_Wdn1swYq[|jr|jsJ|jj}|jj}t|rtd|t|dksJ||d}t|ts|tjksJ|t|dksJ|tdksJdd}|dus Jt|t|ksJ||ftt||D]?\}\}}d+dd}||||||||ksOJ||||f||ks_J|||fq!|dushJttjjjtjjj !dd}t||||t"|d||d }|jdurt|jj#tj$j%sJ||jj#| d!}t&|}fd"d|D}t'|}g}g|_(g|_)|rt|dksJd#ttjjj d$D]/\} }!t*t+|!j,|!j-d%d|!.Dd&d|!/Dd'|t0| fg}"|j(1|"|1|"qnVt|D]Q\} }!| |vr/| t|ksJd(t2|}#|j)1t3|#j4|#||1|#qt*t+|!|!|!|!|!5j6d)|t0| fg}"|j(1|"|1|"qt||D]\}$}%|$7tjj8vrrtjj9:|%7q\|S),zcreate the while_loop IR node. stack_output controls whether it stack each iterations' output, which is necessary for training. r)check_input_alias_and_mutation tensor_boxesrr fake_tensors,list[Union[int, torch.SymInt, torch.Tensor]]rrcSsbt|t|ks Jg}t||D]\}}t|tjr)|tj||ddq||q|S)NFr) rrrr#rrrrr)rrretrfkrrr_require_exact_strides>"s   z0WhileLoop.create.._require_exact_stridesrBcSrKrZrLrHrrrrR"rz$WhileLoop.create..cSrKrZrLrHrrrrS"rcSrKrZrLrHrrrrT"rcrrrrHrrrrV"rcrrrrHrrrrY"rNrMzOutput aliasing is currently not supported in compiled torch.while_loop. The outputs of the body_fn subgraph of torch.while_loop are aliased: r?z9torch.while_loop is assumed to have at least one operand. lhs_exprs Sequence[Union[int, sympy.Expr]] rhs_exprsrcSs<t|t|ks Jt||D] \}}tjj||qdSr)rrrlrrrs)rrlhsrhsrrr_guard_list_equals"sz,WhileLoop.create.._guard_list_equalsrRr|)rwrxryrzr[rRr{rcrrr)rr) all_inputsrrr"rz-NYI: while_loop_stack_output input mutations.rXcSrNrrhrirrrr"rcSrNrrh)rrrrrr"r)rrrrzonly carries can be mutated.)rrrrr^)rrrrrrr)rrrrrr);torch._higher_order_ops.utilsrrlrrrrvrrrrr.r=rr3r4rrRrCrYrr r#rrr!rrr@r5rrrmrrmodulefx GraphModuler:rrrSrr]rrrrrrrrr[rr^rrrr)&rrrrwrxr{rrfx_carried_inputsfx_additional_inputs fx_all_inputsfake_all_inputsfake_carried_inputsfake_additional_inputscarried_inputs_additional_inputs_r/ cond_outputs body_outputsrrrriborrR while_loop mutated_idxsmutated_idx_setrmutated_inputs_iter all_outputsrrh multi_out mutated_inputrwrr)rrrr0"s           "$      zWhileLoop.creatererqcCs.|||j|||jt|didSr)codegen_while_loopr{rrrrrjrrrrk"szWhileLoop.codegenrcCrqrrrrrrrr_#rsz"WhileLoop.get_unbacked_symbol_defs)rwrrrxrrryr<rzr<r[rrRrr{rrr)rwrrrrr) rr<rr<rwrrrxrrr{rrrrrb)rrrrKrwrrxryrzrrrNrrrrkr_rrrrvrrv!s      "   Nrvcs@eZdZ ddddfddZdfdd ZdddZZS)rNrr[r0r>rzrrrrrrrdrrTrRrrrc s~tj|||||d|dddlm}dd|D} ||g|| R|} | dus+J| |_tjj| d|_ |tjj| <dS)NrrRr)get_effect_keycSs g|] }t|tr |jn|qSr)rr)r'rrrrr'#sz,EffectfulKernel.__init__..) rurtorch._higher_order_ops.effectsr effect_typerlr effectful_opsrprev_effect_buffer) rr[r>rrrrrRruncovered_argsrrvrrr#s$   zEffectfulKernel.__init__rcs0t}|jdur|jt|j|Sr)rurrrrrArzr)rr!rvrrr0#s  zEffectfulKernel.get_read_writesrcCrrkrrrrrr}:#r z EffectfulKernel.has_side_effectsrrrDr1)rrrrrr}rrrrvrr#s   rc@s"eZdZed d d ddZdS) rFrurrrcCr^rr9rrrrr?#rZz!NonTensorObj.get_free_symbol_usesNr?rH)rrrrWrrrrrr>#src@sReZdZUded<ded<dddZddd d Zdd dZdddZdddZdS)r)rr+Union[FakeScriptObject, torch.ScriptObject]r'rcCrrrrrrrrK#rzTorchBindObject.get_nameNrrcCrrrrrrrrN#rz!TorchBindObject.codegen_referencecCrrr*rrrrrQ#rzTorchBindObject.get_valuetorch.ScriptObjectcCst|jtjr |jS|jjSr)rr'r# ScriptObjectreal_objrrrr get_real_objT#szTorchBindObject.get_real_objrcCsN|}t|ds Jt|}t|d}dd|D}ttj |dS)N__obj_flatten__rcSs(g|]}t|tjr||qSr)rr#rr!numelrHrrrr`#s  z1TorchBindObject.get_buf_bytes..) rrrrrrr-rIoperatorr)rreal_script_obj flat_dict flat_elems flat_sizesrrr get_buf_bytesZ#s zTorchBindObject.get_buf_bytesr;rr5)rr)rrrF) rrrrrrrrrrrrrr)F#s     r)c@s4eZdZUded<ded<d ddZddd d ZdS)rrrrrrcCrrrrrrrrm#rzGeneratorState.get_nameNrrcCrrrrrrrrp#rz GeneratorState.codegen_referencer;rr5)rrrrrrrrrrrh#s  rc@sHeZdZdddZdddZddd d ZedddZedddZdS)_CollectiveKernelrrcCrrrrrrrr[u#r z!_CollectiveKernel.should_allocatecCrrkrrrrrr}x#r z"_CollectiveKernel.has_side_effectsNrJr-rcCsRt|jtjjus Jd|j}|dur||_n|jj|_dd|jjD|_ dS)Nz,Setting cpp kernel needs a valid op_overloadcSr\rr]rHrrrr#r^z9_CollectiveKernel.set_cpp_kernel_name..) rrNr#r_r`rJrarrbrL)rrJr>rrrrV}#s  z%_CollectiveKernel.set_cpp_kernel_namer>rzrqUnion[IRNode, list[IRNode]]rr rc stjj|j||g|Ri|\}}}}} Wdn1s!wY| r1J|d| |D]} | q3|d|td||||t|} j fdd| Dj dd| Dd|vrj t td|dj |ddSdS)NrTrr|crrrrrrrrr#r~z4_CollectiveKernel.create_inplace..cSr8rrWr*rrrr#rr)rlrrrrr!r9r tree_leavesrSrrrrr) rr>rqrr_example_outputrrrrR tensor_arginpsrrrcreate_inplace#s>    z _CollectiveKernel.create_inplace!Union[TensorBox, list[TensorBox]]+Union[list[MultiOutput], _CollectiveKernel]cs@tjjj||g|Ri|\}}}}} Wdn1s!wY| r1J|d| |D]} | q3t|tr||} | dusKJt| d||||fddt |D_ t j |D]\} } t j stt| s|tjj| jqij S|||||t j st|stjjjg_ S)Nr^r|cs(g|]\}}t|t|fgqSr)rrr)rrrRrrrrr#sz9_CollectiveKernel.create_out_of_place..)rlrrrrrrrrrrrr@rrirrrr)rr>rqrrrrrrrRrrr$rRrrrcreate_out_of_place#sZ      z%_CollectiveKernel.create_out_of_placer1rr) r>rzrqrrr rr rr) r>rzrqrrr rr rr) rrrr[r}rVrrrrrrrrt#s    @rc2eZdZ ddddfddZdddZZS)_AllReduce_KernelNrr[r0r>rzrrrrrrrdrrTrRrrrc (tj|||||d|d|ddS)Nr+aoti_torch_cpu__c10d_functional_all_reduce_rurrVrr[r>rrrrrRrvrrr$  z_AllReduce_Kernel.__init__rerqcC2|d||t|jtr||dSdSNz+torch/csrc/inductor/aoti_torch/c/shim_cpu.hinclude_extra_headerr rr[rrrjrrrrk%$   z_AllReduce_Kernel.codegenrrrr=rrrvrr$  rcr)_AllReduceKernelNrr[r0r>rzrrrrrrrdrrTrRrrrc r)Nr*aoti_torch_cpu__c10d_functional_all_reducerrrvrrr.$rz_AllReduceKernel.__init__rerqcCrrrrjrrrrkD$rz_AllReduceKernel.codegenrrrr=rrrvrr-$rrcsXeZdZ d"ddd#fddZd$ddZd%ddZed&ddZd'fd d! ZZ S)( _WaitKernelNrr[r0r>rzrrrrrrrdrrTrRrrrc r)Nr+aoti_torch_cpu__c10d_functional_wait_tensorrrrvrrrM$rz_WaitKernel.__init__rerqcCrrrrjrrrrkc$rz_WaitKernel.codegencCs|jd}t|ts Jt|tr$|jd}t|ts!Jt||gSt|trB|jd}t|tr@|jd\}}|j|gSgSgSr)rqrrrrrr")rrwrcollrrrrrget_volatile_readsj$s       z_WaitKernel.get_volatile_readsrwrc Cstjj|||\}}}}}Wdn1swY|r*J|d||t|d||||}|jtt|d||dS)NrTr|) rlrrrr9r!rSrr) rr>rwrrrrrRrrrr create_wait$s(   z_WaitKernel.create_waitrcs6t}|}|D] }|jt|q |Sr)rurrrrrArzr)rr!volatile_readsvrrvrrr$s z_WaitKernel.get_read_writesrrrr)r>rzrwrrrrD) rrrrrkrrrrrrrrvrrL$s    rrrcCdt|ttfr t|St|ttfr%ttj}|D]}|t |O}q|St|t j r/t|StSr) rr7r r2rrr:rr"rr#rrrr&rrrr$  rcCrr) rr7r r1rrr:rr"rr#rrrrrr$rr)rrrr)rrrr)rrrr)rrrr)rrrrrrr)rrrrrr)rrrrr-)rrr rrr)rrr rrr)rrr rrr)r'r(rr))rr,rr-)rr2rr)rr=r>rrr)rGrHrIrHrJrHrr)rRrrSrTrr)rfrgrr)rqrrrrs)rrrrrrk)rrrrrrrr)rrrrr^r rrr)TFNFN)rrrrr rrrr rrrrr)rrrrrrr()rrHrJrHrr)rrrr)rrrr)r>rrrr)rr'rrt)rrrr(L __future__rrOrLr-rJloggingrostextwrapr~collections.abcrrrrrrr enumr r typingr r rrrrrrrrrrtyping_extensionsrrrrrrr unittest.mockrrr r!r"torch._export.serde.schema_exportserderrtorch._library.utilsrr+rtorch._loggingr#torch.fxtorch.utils._pytree_pytreertorch._dynamo.utilsr#torch._export.serde.serializer$*torch._higher_order_ops.auto_functionalizer%torch._inductorr&rZr(torch._prims_commonr)r*r+r,r-torch._subclasses.fake_tensorr.%torch.fx.experimental.symbolic_shapesr/r0r1r2r3r4r5r6r7 torch.fx.noder8torch.utils._ordered_setr:torch.utils._sympy.functionsr;r<r=torch.utils._sympy.symbolr>rr@rAcodegen.commonrBrCrDrErFrGrHrIrJrK loop_bodyrL ops_handlerrMrNrOrPruntime.benchmarkingrQ runtime.hintsrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhri virtualizedrjrkrl"torch._library.fake_class_registryrmrnrocodegen.cuda.cuda_templaterpcodegen.wrapperrqrrrrsrrr6 __version__rr ImportErrorrtrurvrwrrxrryr_r`rurz getLoggerrr1rrlgetenvrrrrrrrrrrrrrrrrrrr r+r1r;r<rFrQrerprxrrRrcrrrrrrrVrrrrrrrrr r\rrrrrArMrZrr`rrrrrrrr0rr]rr&r0r4r9r>rr\rr`rrcrrrrrrPrimitiveInfoTyperrrrrrr rr/r0rrrrCrrrrr%r>rmrvrzr~rrrrrrrrrrr rrrBrrr<rCrDrYr_rvrrr)rrrrrrrrrrrsx  8$       ,    `        *    "           '  :K ## ?s :&_J   < aQ0=$7V)b  V$(Y  tH*P=ZUE}%+!@(4L-2N76-?%,b s)/! X