7hH%SSKJr SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSK r SSK r SSK J r J r Jr SSKJrJr SSKJr SSKJr SSK JrJrJrJrJrJrJrJrJr SSKJrJrJ r SS K!J"r" SSK#r#SS K#J$r$J%r%J&r& SSK'J(s J)s J*r+ SSK,J-s J.r/ SSK0r1SSK2r1SSK3J.s J4r5 SS K6J7r7 SS K8J9r9 SS K:J;r; SSKJ?r?J@r@JArAJBrBJCrC SSKDJErE SSKFJGrGJHrHJIrIJJrJJKrKJLrLJMrMJNrNJOrOJPrP SSKQJRrR SSKSJTrTJUrUJVrV SSKWJXrX SSKYJZrZJ[r[ SSK\J]r]J^r^J_r_J`r` SSK[JaraJbrbJcrcJdrdJere SSKfJgrg SSKhJiriJjrjJkrkJlrl SSKmJnrn SSKoJprpJqrq SSK.JrrrJsrsJtrtJuruJvrvJwrwJxrxJyryJzrzJ{r{J|r|J}r}J~r~JrJrJrJrJrJrJrJr SSKJrJrJr \(aSSKJr SS KJr SS!KJr SS"KJr SS#K.Jr O\rS$\S%'SSKr\GR.rS&r\"S(5r\"S)5r\"S*5r\\\$4rS$\S+'\\\\$4rS$\S,'\GRD"\5r\R&"\GRJS-S.9r\1GRGRLr\\S/\\S/4S0S1\\\\\\S/4S/S0S144rS$\S2'GSS3jr\GRV"S&S49"S5S655rGSS7jrGSS8jrGSS9jrGSS:jrGSS;jrGSS<jr/S=Qr/S>QrGSGSS?jjrGSS@jrGSGSSAjjr\GSGSSBjj5r\GSGSSCjj5rGSGS SDjjrGS!SEjrGS"SFjrGS#SGjrGS#SHjrGS$SIjrGS%SJjrGS&SKjrGS'SLjrGS(SMjr"SNS15r\~"S'S49"SOSP55r\~"SQSR\55rGS)SSjr\~"STSU\55r\~"SVSW\55r\"SX5\"SY5\"SZ5\"S[5\"S\5\"S]5S^.rS_\S`'GSGS*Sajjr\~"SbSc\55r\\\$\\$/\4r"SdSe\5r"SfSg\5r"ShSi\5r\~"SjSk\55r\~"SlSm\55r\~"SnSo\55rGS+SpjrGS+SqjrGS,GS-SrjjrGS.SsjrGS/Stjr\~"SuSv\55r\~"SwSx\55r\~"SySz\55r\~"S{S|\55r\~"S}S~\55r\~"SS\55r\~"SS\55r\~"SS\55r"SS\5r\~"SS\55r\~"SS\55r\~"SS\55rGS0SjrGS1Sjr"SS5r\~"SS\55r"SS\5r"SS\5r"SS\5r"SS\5r"SS\5r\~"SS\55r"SS\5r\~"S'S49"SS\\^55r\~"S'S49"SS\\55r"SS\5r"SS\5r"SS\5r\~"SS\55r\~"SS\55r\~"S'S49"SS\55r"SS\5r"SS\5r\\\\\\\\\\\44r"SS5r"SS\5r"SS\5r"SS\5r"SS\5r\~"S'S49"SS\55Gr"SSG\5Gr"SSG\5Gr\~"S'S49"SSG\55Gr\~"S'S49"SSG\55Gr"SSG\5Gr"SSG\5Gr"SS\5Gr"SSG\5Gr"SSG\5Gr "SSG\5Gr "SSG\5Gr "SSG\5Gr "SSG\5Gr "SSG\5Gr"SSG\5Gr"SSG\5Gr"SSG\5Gr"SSG\5Gr"SSG\5Gr"SSG\5Gr"SSG\5Gr"SSG\5Gr\~"S'S49"SS55Gr"SSG\5Gr\~"S'S49"SSG\55Gr\~"SS\55Gr"SSG\5Gr\GRV"SS\55Gr"SS/G\5Gr"SSG\5Gr\~"S'S49"SS\55GrGS2SjGr \~"S'S49"SGSG\55Gr!\~"S'S49"GSGSG\55Gr"GS3GSjGr#\~"S'S49"GSGSG\55Gr$"GSGSG\5Gr%"GSGS \5Gr&\~"GS GS G\&55Gr'\~"GS GS G\&55Gr("GSGSG\5Gr)"GSGSG\)5Gr*GS4GSjGr+GS4GSjGr,g!\a SrS'rGNf=f(5) annotationsN) GeneratorIterableSequence)AbstractContextManager nullcontext)Enum)partial) AnyCallableClassVarLiteralOptionaloverload TYPE_CHECKINGTypeVarUnion) assert_neverNever TypeAlias)patch)ExprIntegerSymbol)identity)GraphModuleSerializer)can_auto_functionalize)metrics)compute_required_storage_lengthis_boolean_dtypeis_float_dtypemake_channels_last_strides_for StrideType)get_schema_info) &_remove_effect_token_unbacked_bindingscompute_unbacked_bindings free_symbolsfree_unbacked_symbols IterateExprsrebind_unbackedresolve_unbacked_bindingsShapeEnvstatically_known_trueSymTypes OrderedSet)CleanDivFloorDivModularIndexing)SymT)config dependencies)BackendFeature CodegenSymbolget_scheduling_for_deviceindex_prevent_reordering)Depextract_free_symbols#extract_input_node_reduction_rangesextract_read_writes var_builder)LoopBody) OpCounterCSE OpCountResult ReductionType StoreMode) benchmarker)DeviceProperties ReductionHint)argsort argsort_sym cache_on_selfceildivconvert_shape_to_inductorconvert_shape_to_symintdeveloper_warningdo_bench_using_profilingdtype_from_sizeget_dtype_sizeget_kernel_metadataGPU_ALIGN_BYTES ir_dataclass is_dynamicis_gpu sympy_dotsympy_index_symbolsympy_index_symbol_with_prefix sympy_product sympy_substensor_is_aligned)opsOpsValueV)FakeScriptObject)Node) CUDATemplate) GraphLowering)IndentedBufferrrcTF_T_U_V_IntLike_NumLikez prefix TensorBoxrIRNode _NodeOrNodesc.[U[[45$N) isinstanceintrxs L/var/www/fran/franai/venv/lib/python3.13/site-packages/torch/_inductor/ir.py _is_staticrws a#w (()frozencR\rSrSr%S\S'S\S'S\S'S\S 'S \S 'S \S 'Srg)GraphPartitionSignatureOrderedSet[sympy.Symbol] symbol_inputsz5dict[str, Union[IRNode, sympy.Expr, TorchBindObject]] input_nodes list[IRNode] output_nodeszdict[str, bool]input_deallocationboolskip_cudagraphz list[str]constant_namesN__name__ __module__ __qualname____firstlineno____annotations____static_attributes__rrxrvr{r{s/,+GF('rxr{c&^SU4SjjmT"U5 g)Nc >Ucg[U[[45(aUH nT"U5 M g[U[5(a UR 5H nT"U5 M g[U[ [ [[[RRR[[[[ 4 5(dS[#U5S35eg)NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])rrlisttupledictvalues ExpandView DynamicScalar AssertScalarrmsympylogicboolalgBooleanrrsEffectfulKernelShapeAsConstantBuffertype)nodesnode_check_tensorboxs rvr%validate_ir.._check_tensorboxs =  e} - - & t $ $  &'! KK''//#)    e %jk  rx)rOptional[_NodeOrNodes]returnNoner) node_or_nodesrs @rv validate_irrs<]#rxcF^[T[5(deSU4SjjnU$)Nc0>[[T5"U0UD6$rq)getattrr^)argskwargsnames rvfnops_wrapper..fnssD!42622rx)robjectrrrr_)rrstr)rrs` rv ops_wrapperrs" dC  3 Irxc f^[[U[[U5555mSU4SjjnU$)Nc>[U5[T5:Xde[[U55Vs/sH oTUPM sn$s snfrqlenrange)indexi inv_orders rvreindex inverse_reorder..reindex sC5zS^+++-23u:->?->il#->???Ar Sequence[_T]rr)rziprr)orderrrs @rvinverse_reorderrs*Sc%j 123I@ Nrxc^SU4SjjnU$)Nc>[U5[T5:Xde[[U55Vs/sH oTUPM sn$s snfrqr)rrrs rvrsame_reorder..reindexsB5zSZ''').s5z):;):AeAh):;;;rrr)rrs` rv same_reorderrs< Nrxc^^SUU4SjjnU$)Nc >T"T"U55$rqr)rreindex1reindex2s rvr fuse_reindexing..reindexs((rx)rrrz Sequence[_V]r)rrrs`` rvfuse_reindexingrs)) Nrxc<U(a [U5$[U5$rq)r(r')ru unbacked_onlys rvget_free_symbolsr#s$Q''Arx)rr5)rrrr5c<Uc [U5nU$[X5nU$)z) Convert strides to fill order (argsort) )rIrJ)seq shape_env sorted_idxs rvget_fill_orderr.s- $+CL  !0 rxc[U5VVs0sHupX!_M nnn[[U55Vs/sHoCUPM nnU$s snnfs snf)zx Convert stride order to fill order For channel last format, stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0] ) enumeraterr)ridxposlookupr fill_orders rvstride_order2fill_orderr<sR(1'7 8'783ch'7F 8%*3u:%67%6)%6J7 97s AAc[X5n[[U55Vs/sHnSPM nn[U5H upVXTU'M U$s snf)z! Convert strides to stride order r)rrrr)rrr_outrelems rvget_stride_orderrHsL !/s >JCHo &o1oC &Z(D ) J 's A cgrqrru guard_shapes rvir_node_to_tensorrUsKNrxcgrqrrs rvrrYsLOrxcUcgU(d%[RRRnO[nUR 5Vs/sH o2"U5PM nn[ U5(a0UR5RVs/sH o2"U5PM nnO[RU5nUR5nUR5n[U5n[U5n[RRRR5 [ R""XEXgS9R%5nSSS5 U$s snfs snf!,(df  W$=f)N)sizestridedtypedevice)r`graphsizevars size_hintrget_sizeis_storage_and_layout get_layoutrFlexibleLayoutcontiguous_strides get_dtype get_devicerNrsuppress_guardstorch empty_stridedzero_) rurshape_fnsrrrrts rvrr]s y 77##--!" .AHQKD .Q'(||~'<'<=' 6 5 Hs EE $E E cD[U[5(a U(dS/$U$rq)rrrvalues rvmay_convert_to_optionalrys!%uv Lrxc@[U[5(dUcU$[U[R5(a UR$[U[ [ 45(a[UR55$[SUS[ U5RS35 g)Nzget_device_type(: )) rrrrrrrn OutputSpecget_device_typerrrrts rvrrsz!SQY Au|| $ $vv A + , ,q||~..#A3ba)9)9(:!<=rxc[U5nUS;a[[US35S:XaggUb[U5=ncgSSKJn [ U[5(de[X#5$)N)cpucuda_backendtritonTFr5)TritonScheduling) rrr6r:codegen.tritonr rrr issubclass)rurdevice_schedulingr s rv is_tritonrsn Q F 6fXX. /8 ;!:6!B B  K0 ' . .. . ' ::rxc[U5S:H$)Nr)rrts rvis_cpurs 1  &&rxc ^^[T[5(aTR5cg[UU4Sj[ [ TR 55S- 555n[RRRTR 5S5S:H=(d= [RRRTR5S5S:*nU=(a U$)NFc3># UHEn[RRRTR 5U5T-S:Hv MG g7f)rN)r`rrsize_hint_or_throw get_stride).0r alignmentrus rv -is_aligned_realized_tensor..sC/A    , ,Q\\^A-> ?) KPQQ/sA Ar5) rrrnmaybe_get_strideallrrrr`rrrr)ruraligned_stridesaligned_last_dims`` rvis_aligned_realized_tensorr s a A$6$6$8$@s1<<>*Q./O ++ALLN2,>?1D F 77   . .qzz|B/? @A E  //rxc[U5[U5:Xa[U5[U5:Xde[X U5Hup4n[RRR US5(aM7[RRR XE5(aMg[RRRU5[RRRU5:XaM g g)zH Returns true if the strides are equal, ignoring dimensions of size 1 . r5FT)rrr`rrstatically_known_leqstatically_known_equals symbolic_hint)strides1strides2shapedims1s2s rvsignificant_strides_equalr+s u:X &3x=CM+II I5H5  77   0 0a 8 8 ww77   ''""0048H8H8V8V 9  6 rxc^[U5(dU$[S[XR5555(aU$[ XR5UR 55(dU$[ U5up#/URQn[UR 55H<upV[RRRUS5(dM6XXE'M> [URURUR UUR"5n[%['X'S95$)a Tries to match the strides of the tensor to those in the meta_strides. Strides of insignificant dimensions - size 0 or 1 - will be updated. If there are real stride differences (NHWC vs NCHW), or the tensor is not realized, then the input will be returned c3x# UH0up[RRRX5v M2 g7frqr`rrr#rr)r*s rvr2try_match_insignificant_strides..s1 7FB 008878:r5datalayout)rrrrr+ras_storage_and_layoutrrr`rrr" FixedLayoutrrroffsetrmReinterpretView)tensorstridesstorage old_layout new_striderr new_layouts rvtry_match_insignificant_stridesr?s ! ( (   '#4#4#67  $W.?.?.A6??CT U U /7G%:$$%J&//+, 77   0 0A 6 6#JJM- J _'E FFrxcURRSS9Sn[UR5VVs/sHup#UPM snnURS'SSKJn U"U5 gs snnf)Noutput)opruser_visible_output_idxs)record_original_output_strides)r find_nodesrrmetatorch._inductor.compile_fxrD)gm output_noderrrDs rvgm_original_output_stridesrJsd((%%%215K#K$4$454554K/0J"2& 4sA#c[5nUH9nU[UR5SS9-nU[UR5SS9-nM; [ U5$)NFr)r0rrrr)inputssym_varsinps rvget_symbolic_inputsrPsP!+H$S\\^5II$S^^%5UKK >rxcx\rSrSr%\"5rS\S'\R"SS9r S\S'\R"SS9r S\S '\R"SS9r S \S '\ \ RSFS j55rSGS jrSHSjrSISjrSJSjrSKSjrSLSjrSMSNSjjrSOSPSjjrSQSjrSRSjrSSSjrSTSjrSUSjrSVSjrSWSjrSXSjrSYSjr \!SZSj5r"S[Sjr#SWS jr$S\S!jr%S]S^S#jjr&S_S$jr'S`S%jr(SWS&jr)SaS'jr*SbS(jr+ScS)jr,SYS*jr-SdS+jr.S\S,jr/SWS-jr0S]SeS.jjr1SfS/jr2SHS0jr3SgS1jr4SHS2jr5ShSiS3jjr6SjS4jr7SkS5jr8ShSlS6jjr9SmS7jr:SnS8jr;SoS9jrS\S<jr?SrS=jr@SWS>jrASWS?jrBSsS@jrCStSAjrDSdSBjrEStSCjrF\G(a \!SQSDj5rHSErIg"SErIg")urnizClassVar[OrderedSet[Any]]_current_originsF)initOrderedSet[Any]originsOptional[list[str]] tracebackOptional[torch.fx.Node] origin_nodec## [RnX-[lSv U[lg!U[lf=f7frq)rnrR)rUolds rvcurrent_originsIRNode.current_originss4%%"%- * &)F #cF #sA1 A >Ac0[RXU5 grq)r __setattr__)selfattrrs rv_post_init_setattrIRNode._post_init_setattrs 4u-rxcURS[UR55 URS[R(a[ R "5OS5 URSS5 g)NrUrWrY)rbr0rRr6debug_ir_tracebackrW format_stackr`s rv __post_init__IRNode.__post_init__sV  :d6K6K+LM  V5N5N//1TX   t4rxcB[SUR555$)Nc38# UHoRv M g7frqrrdeps rvr(IRNode.get_read_names..'?.>s((.>r0 get_readsrgs rvget_read_namesIRNode.get_read_names&?dnn.>???rxcUR$rq)rWrgs rv get_tracebackIRNode.get_traceback)s ~~rxcUR$rqrYrgs rvget_origin_nodeIRNode.get_origin_node,rxcgrqrrgs rvget_defining_opIRNode.get_defining_op/rxcdS[USS53nU(a[U5S:aUSSS3nU/$)Nzorigins=rU@=z...)rr)r`shortenrUs rv common_reprIRNode.common_repr2s@WT9b9:; s7|b( "c*Gyrxc.[U5[URU55-n[[[U55nU(a5[ SR U55n[ U5RSUS3$[ U5RSUS3$)Nz, z( z )(r)rrmaprindentjoinrr)r`linesr multiline new_liness rv str_helperIRNode.str_helper9sU d4#3#3G#<==Se_% uzz%01I4j))*#i[< <4j))*!E7!4 4rxcUR$rqrrgs rvrIRNode.get_dtypeD zzrxcDUR5$![a gf=frq)rNotImplementedErrorrgs rvmaybe_get_dtypeIRNode.maybe_get_dtypeGs& >># #"    c2[S[U5S35e)Nz#get_layout() is not implemented by !rrrgs rvrIRNode.get_layoutMs!$GT |ST"UVVrxcDUR5$![a gf=frq)rrrgs rvmaybe_get_layoutIRNode.maybe_get_layoutP& ??$ $"  rc"UR5$rq)rrgs rvget_output_specIRNode.get_output_specVs  rxcDUR5$![a gf=frq)rrrgs rvmaybe_get_output_specIRNode.maybe_get_output_specYs( '') )"  rc>[UR5[5$)z4True for single tensor output (excludes MultiOutput))rrrLayoutrgs rvhas_tensor_outputIRNode.has_tensor_output_s$446??rxc2[S[U5S35e)Nz!get_size() is not implemented by rrrgs rvrIRNode.get_sizecs!$Ed4j\QR"STTrxcDUR5$![a gf=frq)rrrgs rvmaybe_get_sizeIRNode.maybe_get_sizef% ==? ""  rc"UR5$rqrrgs rvr' IRNode.shapels}}rxc4[UR55$rq)r[rrgs rv get_numelIRNode.get_numelpsT]]_--rxc[RRR[R "UR 5S55$Nrr`rrr-rEqrrgs rvis_zero_elementsIRNode.is_zero_elementss0ww55ehht~~?OQR6STTrxc0[S[U535e)a If the IRNode refers to data which has not been materialized (e.g., it is a Pointwise/Reduction that could potentially have more compute fused into it), realize the IRNode into physical memory, ending the possibility of fusing into it, but allowing, e.g., multiple users to access the data without having to recompute. Check StorageBox.realize for a particularly notable implementation. TODO(ezyang): I think, in principle, every IRNode should have an implementation of this, and most of the time no-op is OK, but you really do have to audit each IRNode for this, so for now, raise an error if it's not implemented. Note that some code in graph.py will catch this thrown error and suppress it with a warning. zrealize NYI on rrgs rvrealizeIRNode.realizevs "ODJ<"@AArxNc0[S[U535e)Nzcodegen_reference NYI on rr`writers rvcodegen_referenceIRNode.codegen_references!$=d4j\"JKKrxcgrqrrgs rvrIRNode.get_devicerrxc0UR5nUceU$rq)rr`rs rvget_device_or_errorIRNode.get_device_or_errors"!!! rxcgNFrrgs rvhas_exceeded_max_readsIRNode.has_exceeded_max_readsrxc>[[U5R5erqrrrrgs rv make_loaderIRNode.make_loader!$t*"5"566rxc>[[U5R5erqrrgs rv make_indexerIRNode.make_indexerrrxc>[[U5R5erqrrgs rvrIRNode.get_striderrxcDUR5$![a gf=frq)rrrgs rvrIRNode.maybe_get_striderrc>[[U5R5erqrrgs rvget_nameIRNode.get_namerrxcDUR5$![a gf=frq)rrrgs rvmaybe_get_nameIRNode.maybe_get_namerrczUR5[RR;$![a gf=fr)rr`r graph_inputsrrgs rvis_input_bufferIRNode.is_input_buffers4 ==?agg&:&:: :"  s *- ::cgrrr` thresholds rvhas_large_inner_fnIRNode.has_large_inner_fnrrxcgrqrr`userss rv mark_reuseIRNode.mark_reuse rxcgrqrrgs rv realize_hintIRNode.realize_hintrrxc>[[U5R5erqrrgs rv unwrap_viewIRNode.unwrap_viewrrxc>[[U5R5erqrrgs rv freeze_layoutIRNode.freeze_layoutrrxc>[[U5R5erqrr`r allow_paddings rvfreeze_layout_with_stride_order&IRNode.freeze_layout_with_stride_order"$t*"5"566rxc>[[U5R5erqrr`rs rvfreeze_layout_with_fill_order$IRNode.freeze_layout_with_fill_orderrrxc>[[U5R5erqrr`rs rvfreeze_layout_with_same_order$IRNode.freeze_layout_with_same_orderrrxc>[[U5R5erqrr` exact_stridesrs rv freeze_layout_with_exact_strides'IRNode.freeze_layout_with_exact_stridesrrxc>[[U5R5erqrrgs rvget_read_writesIRNode.get_read_writesrrxc6UR5R$rqrreadsrgs rvrsIRNode.get_reads##%+++rxc4[UR55$rq)rrsrgs rv num_readsIRNode.num_readss4>>#$$rxc>[[U5R5erqrrgs rvget_storage_numelIRNode.get_storage_numelrrxc>[[U5R5erqrr`rs rvget_free_symbol_usesIRNode.get_free_symbol_usesrrxc>[[U5R5erqrrgs rvget_reduction_typeIRNode.get_reduction_typerrxc>[[U5R5erqrrgs rvget_reduction_sizeIRNode.get_reduction_sizerrxcgrrrgs rv is_externIRNode.is_externrrxcgrrrgs rvis_no_opIRNode.is_no_oprrxc>[[U5R5erqrrs rvconstant_to_deviceIRNode.constant_to_devicerrxc>[[U5R5erqrrgs rvget_mutation_namesIRNode.get_mutation_namesrrxc>[[U5R5erqrrgs rvget_operation_nameIRNode.get_operation_namerrxc>[[U5R5erqrrgs rvget_inputs_that_alias_output#IRNode.get_inputs_that_alias_outputrrxcgrqrrgs rvr IRNode.dtypes(+rxr)rUzOrderedSet[Node]rzGenerator[None, None, None])rarrr rrrrrOrderedSet[str])rrVrrXrzOptional[Operation]T)rrr Sequence[str])TT)rzSequence[object]rrrrrrr torch.dtype)rzOptional[torch.dtype]rr)rzOptional[Layout]rr)rzOptional[OutputSpec]rrrSequence[Expr])rOptional[Sequence[_IntLike]])rz.Union[_IntLike, sympy.Rel, Sequence[_IntLike]]rrr Optional[str]rqrzOptional[IndentedBuffer]rrrOptional[torch.device]r torch.devicer$Callable[[Sequence[Expr]], OpsValue]r Callable[[Sequence[Expr]], Expr]rSequence[_IntLike]rrr Optional[int]rrrrsrrrrnFr list[int]rrrrrrbrrrlist[_IntLike]rrrrerrrrrzdependencies.ReadWritesrzOrderedSet[Dep]rrsrrirrrr}rSequence[sympy.Expr]rrTrrnrrD)Jrrrrr0rRr dataclassesfieldrUrWrY staticmethod contextlibcontextmanagerr\rbrhrtrxr|rrrrrrrrrrrrpropertyr'rrrrrrrrrrrrrrrrrrrrrr rrrsrrr"r%r(r+r.r1r4r7r:rrrrrxrvrnrns'2<,/> +00e.#rprqrrrgs rvrtOperation.get_read_names"rvrxc6UR5R$rqrrgs rvrsOperation.get_reads%rrxc[erqr}rgs rv get_outputsOperation.get_outputs(rrxc[5$rqr/rgs rvget_unbacked_symbol_defs"Operation.get_unbacked_symbol_defs+ |rxc[5$)a When unbacked_only=True: Returns the unbacked symbols which are required to be in scope in order to successfully perform codegen for this buffer. For example, a buffer that corresponds to an extern kernel call that takes i0 as an argument would return {i0} here. This is used to generate necessary dependencies that ensure we actually bind i0 in codegen before you try to use it. Note that this is NOT transitive; in particular, if this buffer takes in as input another buffer with dynamic shape (e.g., (i0,)), we will not report it here, because you will already have a dependency on that buffer, which will eventually have a dependency on i0 if necessary. When unbacked_only=False: Similar to `unbacked_only=True` but including all free symbols instead of only free unbacked symbols. r/r!s rvr"Operation.get_free_symbol_uses.s ,|rxcg)z Gets extra global memory size needed by this buffer. Some algorithms (e.g. group gemm) may require extra global memory in the generated code. rrrgs rvget_workspace_sizeOperation.get_workspace_sizeFs rxryNr>rQrA)rrTr[rIrg)rrrrr?rhr list[Buffer]rr}r`rkri)rrrrrhrr|rr7r+r.rrrtrsrrr"rrrrxrvrwrwsc2" #"-@,"%*! !0rxrwcx^\rSrSr%S\S'S\S'S\S'S\S 'S S!S jjrS"S jrS#U4S jjrS$S jr\r S%Sjr S&Sjr S'Sjr S'Sjr \S(Sj5r\\R$4S)Sjj5r\S*Sj5rS+Sjr\S$Sj5rS,S-SjjrS S.SjjrS/SjrS0SjrS1SjrS2SjrS3SjrS4SjrSr U=r!$)5LoopsiNrTrrFrCallable[..., Any]inner_fnrZrangesc^[5R"/U4SjUR5QURT5P76$)Nc3<># UHn[UT5v M g7frqrrers rvr-Loops.get_free_symbol_uses..Ys F+Qq-00+)r0unionrinner_fn_free_symbolsr!s `rvr"Loops.get_free_symbol_usesUs>|!! F$++ F  & &} 5  rxc  URSURRS3[UR5UR 5/UVs/sHo"S[ X53PM sn-SUR<3/-5$s snf)N'= origin_node=)rrrrr inner_fn_strrrY)r`namesrs rv_to_str Loops._to_str]sDKK$$%Q'DJJ!!#  [TU]5 grq)superrhr` __class__s rvrhLoops.__post_init__hs rxc$URS5$)Nrrrgs rv__str__ Loops.__str__ks||K((rxcUR$rqrrgs rvrLoops.get_devicep {{rxcUR$rqr{rgs rvr|Loops.get_origin_nodesr~rxcUR$rqrrgs rvrLoops.get_sizevrrxcUR$rqrrgs rvget_pointwise_sizeLoops.get_pointwise_sizeyrrxcURSS5nURSS5nU"U0UD6nURSU5 URSU=(d UR5 [R U5$)NrYrW)poprbrWrmcreate)clsrrrYtbrs rvr Loops.create|smjj5 ZZ T *    ]K8 ["*; <""rxc[U5VVs/sH0up#US:Xa[RRO [ X5PM2 snn$s snnfNr5)rrSZerorZ)rrlnrs rv_index Loops._indexsI"&) )FEGGLL(Fv(Q Q)   s7A c|[[R"55n[R"U5 [R "[ SS5 UR"UR56 UR5sSSS5 sSSS5 $!,(df  O=fSSS5 g!,(df  g=fNallow_indexingT) rBr` MockHandlerset_ops_handlerrrrr inner_fn_argsgetvalue)r` opcounters rvinner_fn_opcountLoops.inner_fn_opcountsy 1  i ( LL)94 @ MM4--/ 0%%' A @ ) ( @ @ @ ) ( (s#B--B? B- B B-- B;c:URUR54$rq)rrrgs rvrLoops.inner_fn_argss DKK(**rxct[RR"UR/UR 5Q76$rq)r`KernelFormatterHandler ir_to_stringrrrgs rvrLoops.inner_fn_strs3''44 MM  ..0  rxczUcSn[U[R5nUR5RU:$r)maxr6realize_opcount_thresholdrnum_opsrs rvrLoops.has_large_inner_fns9  I 6#C#CD $$&..::rxc`URUR5n[URX!S9$NrL)rrr=r)r`rrs rvrLoops.inner_fn_free_symbolss% DKK(#DMM5VVrxc[R"[SS5 UR5(aJ[ UR 5UR 5UR55RsSSS5 $[ UR 5UR 55RsSSS5 $!,(df  g=fr) rrrr%r?rrr(rrgs rvrsLoops.get_readss \\.*:D A&&((*$$&MMO++-% B A+$$&MMO%B A AsAB8=1B88 CcH[UR5R5$rq)r0r read_buffersrgs rvrtLoops.get_read_namess$//1>>??rxcH[UR5R5$rq)rrrrgs rvrLoops.num_readss4((*7788rxc2[S[U5S35e)Nz+get_reduction_size() is not implemented by rrrgs rvr(Loops.get_reduction_size!9$t*Q G  rxc2[S[U5S35e)Nz+get_reduction_type() is not implemented by rrrgs rvr%Loops.get_reduction_typerrxc2[S[U5S35e)Nz+constant_to_device() is not implemented by rrrs rvr1Loops.constant_to_devicerrxrr`rk)rrDrrr>r[rQrArJ)rr rr rrm)rrZrlr4rrK)rrCrzSequence[Sequence[_IntLike]]rqr\rrrOrderedSet[Symbol]rhr?rirlrNrn)"rrrrrr"rrhr__repr__rr|rr classmethodrrrr4INDEXrrKrrrrrrsrtrr(r%r1r __classcell__rs@rvrrNs    %* ! !    )H  # #:>**  ((+  ; W @9    rxrcUR(a [R"[S5U5$[R"SU5$)Nnanr)is_floating_pointr^constantfloat)rrs rv nop_loader_fnrs1 ||E%L%00||Au%%rxcV\rSrSrS SjrS SjrS SjrS SjrS SjrSr g) PointwiseictUR5(a[[URS9$UR$Nr)rr rrrrgs rvrPointwise.make_loaders,  " "= ; ;}}rxc/$rqrrgs rvr(Pointwise.get_reduction_sizes rxcgrqrrgs rvr%Pointwise.get_reduction_typerrxc|UR5n[R"U=(d SU"U5U"U55$Nunnamed)rr^storer` output_nameindexervarsloaders rv store_outputPointwise.store_outputs2 !!#yy1 74=&,OOrxcUR5n[R"[SU5"U5n[ XR X R S9$FMove this to a given device. Requires that all reads are to constants.override_devicerrrr)rrrConstantBufferrrrr`rr&s rvr1Pointwise.constant_to_devicesB!!#n.?HPf[[  rxrNrUrlrNr#rOr$!Callable[[Sequence[Expr]], Never]r%rKrrrn) rrrrrr(r%r'r1rrrxrvrrsFP"P3P P  P rxrcR\rSrSr%S\S'SrS\S'S SjrS SjrS rg) ScatterirXoutput_indexerNrE scatter_modec UR5n[R"[SU5"U5n[ UUR UUR URURS9$)r+r,)rrrrr5r6) rrrr.r4rrr5r6r/s rvr1Scatter.constant_to_devices[!!#n.?HP**;;..**   rxcUR5nUcSn[R"UU"URU55U"U5URS9$)Nr )mode)rr^r!r5r6r"s rvr'Scatter.store_output sT !!#  #Kyy  D''- . 4L""   rxrrnr1) rrrrrr6r1r'rrrxrvr4r4sB44"L)"   " 3     rxr4 logical_ormaximumminimummuladd bitwise_xor)anyrminprodsumxor_sumz"dict[str, Callable[..., OpsValue]]REDUCTION_COMBINE_FNc^^^T[;a [T$TS;aSUUU4SjjnU$TS:Xa SSjnU$[ST35e)Nargmaxargminc>Uup#UupET S:Xa[R"X$5nO[R"X$5n[R"X$5n[ T 5(a[R "X"5n[R "XD5n [R "U[R"X55n[R "U[R"X55nT (a[R"X55O[R"X55n [R "U[R"Xz55n[R"XbU5[R"XcU54$)NrK) r^ltgteqr!ner< logical_andwhere)aba_valuea_indexb_valueb_indexmaskequala_isnanb_isnantiearg_break_ties_leftrreduction_types rvargmax_combine_fn3get_reduction_combine_fn..argmax_combine_fn-s ! G G)vvg/vvg/FF7,Ee$$&&2&&2~~dCFF7,DEucoog.OP'w(VVG-  >>$(CDD $1 $1 rxwelford_combinec\Uup#nUupVnXR- nXG-n Xy- n X(U --X6-X-U-U --U 4$rqr) rSrTa_meana_m2a_weightb_meanb_m2b_weightdelta new_weight w2_over_ws rvwelford_combine_fn4get_reduction_combine_fn..welford_combine_fnNs]&' "F(%& "F(OE!,J -I** emh6BB rxzunknown reduction_type=)rStuple[object, object]rTrortuple[OpsValue, OpsValue])rS#tuple[OpsValue, OpsValue, OpsValue]rTrqrrq)rGr)r_rr^r`rms``` rvget_reduction_combine_fnrr%s--#N33 / / $ )>  &  :! , , 2 2 1  "!"$;N;K"LMMrxc4^\rSrSr%S\S'S\S'S\S'S\S 'S!S jr\rS"S#U4S jjjrS$S jrS%S jr S&Sjr S'Sjr S(Sjr S"S#Sjjr S)Sjr\S*S+Sjj5r\S,Sj5r\\R(S4S-Sjj5r\S.Sj5r\S.Sj5r\S/Sj5r\S0Sj5r\S*S1Sjj5r\S2Sj5r\S3Sj5r\S*S4Sjj5r\S5Sj5rS rU=r $)6 ReductionidrZreduction_rangesrDr_rF src_dtyperHreduction_hintc$URS5$)N)rrur_rrgs rvrReduction.__str__ls||LMMrxc|>^[TU]T5[5R"U4SjUR56-$)Nc3<># UHn[UT5v M g7frqrrs rvr1Reduction.get_free_symbol_uses..ss P:OQq-00:Or)rr"r0rrur`rrs `rvr"Reduction.get_free_symbol_usesqs7w+M:Z\=O=O P$:O:O P>   rxcUR$rq)rurgs rvr(Reduction.get_reduction_sizevs$$$rxcUR$rq)r_rgs rvr%Reduction.get_reduction_typey"""rxc [R"URURURUR X455n[R "U=(d SU"U5U5$r)r^ reductionrrvr_rstore_reduction)r`r#r$r%reduction_varsrs rvrReduction.store_reduction|sU JJ NN    MM$ /   "";#;)WT]ERRrxcX[UR5[UR5-$rq)rrrurgs rv index_lengthReduction.index_lengths!4;;#d&;&;"<<[RRRU5n [RRR[ U55n US:H=(dV [RR U[ R5(+=(a US;=(a [Rn [U 5(a[U 5(d[RS4$[R"U5n U Rn SnU (a][ R""[R$R&USS9n[ R""[R$R&USS9nO SSjnUnU S:XaU"X5nUS:Xa[R(U4$Ub[+U[,5(a[.R0"[2S S5 [5U5unnSSS5 WbjWbg[RRR[ UU-55nU U:Xa,[6R9S UUUUU5 [R(S 4$[R(U4$X::d XS -S-:a[RS4$[;UUUUUUS:waUOS U[RS9nSSjnU"U5unnU(a U"U5unn[=U5S:Xa[RS4$[>R@"URC5URE55uunnnSnSnUHn[RRRGUU5n[RRRIUU[KURM555n [OSU 55n!U!(aUS- nMUS- nM UU:a[R(U"X54$[RPU"X54$!,(df  GN&=f)NscanrIr5 T)inner_reductionFcgrr)reduction_numel_hint numel_hints rvinner_reduction_splits4Reduction.num_splits..inner_reduction_splitssrxrzUse previous IRNode's range and reduction_ranges instead of split. current ranges: %s, current reduction ranges: %s, current split: %d, new ranges: %s, new reduction ranges: %srrrErc R^[S[UR5UR5UR 5S9US9nUR 5nUR ceUR Vs/sH=n[U[5(dM[U[R5(aM;UPM? nn/nSn[URSS9Hm[U4SjU55(dMURTR5 TR ["R$R&;dMd["R$R&TR n[)UR*SS5nUR-5 [)UR*SS5U:wdMSnM XE4$s snf) Nrrrrr4r3FcUR$rqrlrts rv@Reduction.num_splits..get_read_indices..saffrxkeyc3T># UHoTRR;v M g7frq)rr')rrmds rvrAReduction.num_splits..get_read_indices.. sF:aBHH111:%(rT)ComputedBufferrrrrr range_varsrrrrNumbersortedrrappendrrr`rname_to_bufferrr4 decide_layout) rcb read_writesrindiceschangedbuforiginal_striders @rvget_read_indices.Reduction.num_splits..get_read_indices s]%<<>++- B,,.K))5 55%///Aa&/9!U\\/J/  GG[..4DEF:FFFNN288,ww!''"8"88gg44RWW=*1#**h*M))+"3::x>/Q&*GF# #!s-F$ F$+F$rc3*# UH oS:v M g7fr5Nrrrs rvr'Reduction.num_splits..<s/w!Aw)rrsrrsrrs)rrtrztuple[Sequence[Expr], bool]))r`rrr$r[ has_featurer8REDUCE_TO_SINGLE_ELEMENTr6split_reductionsrwrHrrGrmulti_processor_count functoolsr choicesreduction_split_factorINNERrrrmrrrr>logdebugrtrr7index_vars_squeezerr(simplify_with_ranges stride_hintsrkeysrOUTER)"r dst_dtypervrrrur_reduction_numel input_noderr should_splitpropsnum_smmin_elements_per_threadrouter_reduction_splitssplit new_rangesnew_reduction_rangesextracted_numel_hintrrrrrrranges1 num_outer num_innerrjr:outers" rv num_splitsReduction.num_splitss !ww//==oNWW%%33M&4IJ %/ ##FN,S,ST T ( ( '' /00Z 5K5K ((!+ + ''/,,"$ @I@Q@Q 00&$A "AJ@Q@Q 00&%A "  &)    &< " ?*+?LEz$**E11%*Z*K*K\\.2BDI5(00  $B,A. )!,JGQ w<1  ((!+ +'3'F'F JJL!..0( $NW  A  55aAAgg&&33>4 #7G/w//EQ Q  y  &&(>$) !&&(>$) MJIs P  PcR^^^^^^TVs/sH,n[RRRU5PM. snm[ X#5mSUUU4SjjmUS;aD[ SST[ RT55R5mSUU4SjjmU4Sj$TmT$s snf)z1Convert inner_fn from a reduction to an pointwisec >^[R"TUU4Sj[R"TVs/sHn[ U5PM sn655$s snf)Nc36># UHnT"TU5v M g7frqr)rrrvalue_fns rvr=Reduction._unroll_reduction_fn..fn..[s&#UF++#s)rreduce itertoolsproductr)rru combine_fnrurs` rvr*Reduction._unroll_reduction_fn..fnXsN##"+"3"3,<=,sArKrJNc>UVs/sHn[R"U5PM nnT"X5[R"T"U5[R 54$s snfrq)rexpandr^ index_exprrint64)rrr flatten_indexrs rvr0Reduction._unroll_reduction_fn..value_fnlsO4::6a%,,q/6:U+NN=#8%++F;s Ac>T"U5S$rr)rrs rvr0Reduction._unroll_reduction_fn..us E1rx)rrZrr )rrZrrZrrp) r`rrevaluate_static_shaperrr6rrr) rrur_rvrurrrrs `` @@@@rv_unroll_reduction_fnReduction._unroll_reduction_fnJs@P ?O!AGG   2 21 5?O .nH    1 1' 112BC  ln  ) 3E *  . -HIM s3B$c ^^^^^^[RRR[ T55mTS:XanSU4Sjjn U "S5U "S5U "S5U "S5S.mTTR 5;d TS35eSUUU4Sjjn [ RUUU [U5S9$TS:Xa-TS;a SU4S jjn O SUU4S jjn [ RUTXS9$[T[5(a[RRRT5[R:aQ[ U5S:wd[UR5(a([ RUTUR!TTTU5US9$UR#UTUTUTTTU 5 upSU4S jjnU"U5nU[$R&:XaU nUS :Xa7U ce[)U 5unnUceUceUR+UTUTUTUUTU5 $US:aUR-UTUTUTTUUU 5 $[.R[1UTTUTTUUS 95$)Nrc>T[R:Xa [U5$TR(a,[U[R 5(de[ U5$[U[R5(de[U5$rq) rrrrrtyping SupportsFloatr SupportsIntrs)valrs rvpy_cnst!Reduction.create..py_cnstsg *9$00%c6+?+?@@@@ :%%c6+=+=>>>>s8Orxr5)rErFrDrBz* not supported for zero-dimension tensors!c8>[R"TTT5$rqr^r)rrr_rtypes_to_initss rvconst_fn"Reduction.create..const_fns||ON$CYOOrxr-rc2>[R"ST5$rr)rrs rvrReduction.create..fns<<955rxcr>TVs/sHn[RRPM nnT"X5$s snfrqrrr)rrreduction_indexrrus rvrrs2=M&N=Muww||=MO&N#E;;'O$4cl>[T5(aU$US:a[U[R5$U$r)rwrr6 min_num_split)rrs rv_maybe_increase_split/Reduction.create.._maybe_increase_splits2/** qy5&"6"677 rxrr)rrrzUnion[bool, float, int])rrsrr_)rrsrrs)r`rrsimplifyr[rrrrrrrrr6unroll_reductions_thresholdrWrrrrHrr>!create_multilayer_existing_rangescreate_multilayerrmrt)rrrrvrrrur_rwrrrrhintrr rrrrs ` ` `` @@rvrReduction.createzs''**33MBR4ST a  $qz"1: qz O"_%9%9%;; !""LM ; P P##!F| $  a !5566 <<##Y$   0 0  33OD001v&!+vfkk/B/B##11.  $ nn            &e, ]22 2!N B;) ))/R0 ,J,) ))'3 3388 $  QY((    !!1-#-   rxc US;aL[U5(a [S5$[U5(ag[R"U5R $US;aL[U5(a [S5$[U5(ag[R"U5R $[U5(aSOSn[U5(aSOSnUUUUX"U4X"U4[S5U4S .U$) N)rrJz-infF)rCrKinfTrr5)rErDrFrBwelford_reducerbonline_softmax_reduce)r!rr riinforCr)r_rzeroones rvdefault_accumulatorReduction.default_accumulators . .e$$V}$!%(({{5)--- . .e$$U|#!%(({{5)---(//uQ&u--d1#40 $D1&+FmT%:   rxc:US:Xag[RX5$)Nrr)rtrr_rs rv default_valueReduction.default_value:s! - -,,^CCrxcUS:XaU$US::a*US::a$U[R:Xa[R$US::a*US::a$U[R:Xa[R$U$)Nri)rHr OUTER_TINY)rrrws rv_multilayer_second_step_hint&Reduction._multilayer_second_step_hintBsg B;! ! C^^UunUGtmnTU-U-mSUU UU 4SjjnT (ac[T 5n[R"[R"TU5[R"T U55n[R"XST 5$U"5$)Nc$>T"TT"T/55$rqr)rr& new_indexrsrvbodyCReduction._multilayer_wrap_loader..wrapper_fn..bodysi');<.wrapper_fns"1 _*/ 'Y ?2_DG = =-o> vvNN7K8NN?K@zz$g66v rx)rSequence[Symbol]rr8rr_) r(Viewdynamic_reshape_indexerr`rrr-rr) rr&rurrr3r4r dense_indexr6r5rs ` ` `` @@rv_multilayer_wrap_loader!Reduction._multilayer_wrap_loaderss>> Z .. / ((>> HH_u,a 0   # 6F    (rxc^^^[ST55(d ST<35e[RU[U5[U5-5mSUUU4SjjnU$)Nc3*# UH oS:Hv M g7frrrrs rvrDReduction._multilayer_wrap_loader_existing_ranges..s3?a6?rz8Only enabled for numel_hint == 1, found original_ranges=c >US[T5nU[T5SnT"UT"[U5[U5-55$rq)rr) merged_indexnew_reduction_index original_idxr-r&original_rangesrs rvr6EReduction._multilayer_wrap_loader_existing_ranges..wrapper_fnsQ((>#o*>?L$S%9%;T"/UQUQ5$rqr)rrintermediate_loaders rvintermediate_fn;Reduction.create_multilayer_helper..intermediate_fns''A'A'AB BrxNr)rrZrrZrr_)rfloat16bfloat16rrtrrrr`rrrr[r%rrm)rrrrvr6rFrHrrr_rrwintermediate_dtype intermediaterOrrNs @rvcreate_multilayer_helper"Reduction.create_multilayer_helpers$0 ??   !''          *668 C% C8J C  C WW%%// o0NO 99 ~ -Cs?/C"DDDD (&!+C,@,B!C-#-   rxc [U5n [XS- -U5n URXr5n URUUU UU U U 5nUR UUUUUU/UQUPU /UUU 5 $)rLr5)r[r2rr<rU)rrrrvrrrur_rrwrrr3r4r6s rvrReduction.create_multilayers&((89o;UC ##N>00        ++       f e  L     rxc jURUUUUU5n URUUUU UU/UQUQUU SU 5 $)rLr)rIrU) rrrrvrrFrHrrr_rwr6s rvr+Reduction.create_multilayer_existing_ranges)sc$@@   %   ++      % +o + +     rxrr[r`rrlrN r#rOr$r2r%rKrr8rrrirzSequence[Sequence[Expr]]rnrq)rrTrrFrvrFrCallable[..., OpsValue]rrZrurZr_z%Union[ReductionType, Literal['scan']]rrrOptional[IRNode]rtuple[ReductionHint, _IntLike]) rz^^%^ (^^@-N-,-- - 2 --^)6(=(='+^ ^ ^  ^ % ^  ^ )^ &^ &^ %^  ^ ^ @$/ ,<DD$/D ,DD%(:G  &4D >(,('(-(" (  (  (5(%( ((TP($2  &  0  J8= = =  = ' = ( = $2= = ,= &= = &=  = = ~(,+ + +  + % +  + )+ &+ + &+ %+  + + Z$ $ $  $ % $ ( $ $2$ "$ ,$ &$ &$  $ $ rxrtcz^\rSrSr%S\S'SU4SjjrSSjrSrU=r$) MultiOutputReductioniTrs output_indexc >^[T5(aT4m[T5S:XaTSn OSU4Sjjn [T U] UUU UUUUUS9 Xlg)Nr5rc4>^^[UU4SjT55$)Nc34># UH o"TT5v M g7frqr)rrr reduction_idxs rvr@MultiOutputReduction.__init__..loader..nsHiR]33i)r)rrm inner_fnss``rvr&-MultiOutputReduction.__init__..loaderksHiHHHrxr)rrKrmrKrztuple[OpsValue, ...])callablerr__init__ri) r`rrrprrur_rvrwrir&rs ` rvrsMultiOutputReduction.__init__Ws I  " I y>Q q\F I# I4B I% I -))  )rxc P[R"URURURUR X455n[ U[[45(d[U55eXPRn[R"U=(d SU"U5U5$r) r^rrrvr_rrrrrrrir)r`r#r$r%rrrs rvr$MultiOutputReduction.store_reduction|s JJ NN    MM$ /   &5$-00CT&\NC0(()"";#;)WT]ERRrx)ri)rrTrrFrpz)Union[INNER_FN_TY, Sequence[INNER_FN_TY]]rrdrurdr_rDrvrFrwrHrirsr[) rrrrrrsrrr rs@rvrhrhTs#)#)#)= #) " #) , #)&#)#)&#)#)JS"S3S S ) S  SSrxrhcj\rSrSr\\R S4SSjj5rSrg)OnlineSoftmaxReductioniNc ^^^^^^^[UUUUUUU4Sj[U555n U Hn U R5 M U $)z. Create the reduction disregarding splitting. c3p># UH+n[R[TTTTTSTTU5 5v M- g7f)rN)rmrrh) r output_idxrrrrrwrurvs rvr0OnlineSoftmaxReduction.create..sO 0    $$+"   0s36)rrr) rrrrvrrru num_outputrwrresultsrs `````` ` rvrOnlineSoftmaxReduction.createsB   $J/   A IIKrxr)rrTrrFrvrFrrrrKrurKr}rsrwrHrr^rSequence[TensorBox]) rrrrr rHrrrrrxrvrxrxs)6(=(='+!!! ! % !  !)!!&!%! !!rxrxc\rSrSr\\R 4SSjj5r\SSj5r \S Sj5r Sr g) WelfordReductionic^^^^^US;de[RRR[ T55nS UUU4Sjjn US:XaU "S5n U "S5n U "S5n XU 4$US:XaDS UUUU4SjjmUS:XaT"US5U "S5U "S54$[ U4SjU55$[ RTTTUSTTUUS9upU[R:XaU nUS:aURTTUTTUUU5$[S 5Vs/sH)n[R[TTUTTUTUU5 5PM+ nnUHnUR5 M U$s snf) N)rrbcV>^SUU4Sjjn[RTTU[T5S9$)Nc2>[R"TT5$rqr)rrrs rvr8WelfordReduction.create..const..inner_fns||rxr-rrKrr_rrr)rrrrrs` rvconst&WelfordReduction.create..consts7   ##!F| $ rxrr5cV>^SUU4Sjjn[RTTU[T5S9$)Ncr>TVs/sHn[RRPM nnT"X5$s snfrqr)rrrr&rus rvr7WelfordReduction.create..copy..inner_fns2=M&N=Muww||=MO&N!#77'Or r-rr)r&rrrrrus` rvcopy%WelfordReduction.create..copys788!''!%< (rxrc34># UH nT"U5v M g7frqr)rrrs rvr*WelfordReduction.create..s: "T"XX ro)r_rr)rrsrrm)r&z4Callable[[Sequence[Expr], Sequence[Expr]], OpsValue]rrm)r`rrrr[rrtrrHrrrrmrrr)rrrrprrur_rwrrmeanm2weightrrr{r~rrs `` `` @rvrWelfordReduction.creates!FFFF''**33MBR4ST   a 8DqB1XFV# # a  L   !11IaL)58U1X==: :::& **    aL  )++   ]22 2!N 19((   2$Ah '     $""  '  A IIK% s0E#cg)N)rrrrrs rvrWelfordReduction.default_value.srxc ^^^^^^^[T5m[RRR [ R "TT-S55(+n U (aBUS:wa<S U4Sjjn TRUTUS[U SS9[U SS94UTSTUS9$[TTS- -T5m[RUT[UUUUU4SjU55/UQTPT/UU5n U Hn U R5 M S Sjm[RRR[U55n TRTX5n[RUT[U4S jU 55UT/SU5$) rLrrbc2>[R"UT5$rqr)rrmrrs rvr4WelfordReduction.create_multilayer..constantMs||E511rxrr5)rrrprrur_rrwc 3L># UHnTRUTTTTSS9v M g7f)r)r4N)r<)rr&r3rrrurs rvr5WelfordReduction.create_multilayer..es> (F++$# ,(s!$cU"/UQUQ5$rqr)rrr&s rvintermediate_loader_fnBWelfordReduction.create_multilayer..intermediate_loader_fnxs 4E4O45 5rxc3T># UHn[TUR5S9v M g7f))r&N)r r)rrrs rvrrs&&A.q}}G&r)rrKrmrKrrsrr_)rrKrrKr&rVrr_)r[r`rrr-rrrr r2rrrrrr%)rrrrprrur_rrwr5r intermediatesrrr3rrs` ` ` ` @@@rvr"WelfordReduction.create_multilayer4s ((89((>> HH_u,a 0   +<< 2# 24B 2KN 2 2 ((aLHA.HA. !10-)  o;UC (//     (   f e  L  # &A IIK 6! 6+ 69 6  6WW%%// f0EF 99 :  &&   &   G    rxrN)rrTrrFrpSequence[Callable[..., Any]]rrfrurfr_rDrwrHrrr`)rrTrrFrprrrfrurfr_rDrrirwrHrr) rrrrr rHrrrrrrrrrxrvrrs)6(=(=vvv0 v  v ( v&v&v vvp$/ , Z Z Z 0 Z  Z ( Z &Z Z &Z  Z Z rxrc^\rSrSr%S\S'S\S'S\S'S\S'S \S 'S \S 'S \S'S\S'S S!U4SjjjrS"U4SjjrS#SjrS$SjrS%Sjr S&Sjr S&Sjr S'Sjr S(Sjr S S!Sjjr\\R"4SS.S)Sjjj5r\S*Sj5rSrU=r$)+Scanirf scan_rangesr=Callable[[tuple[Any, ...], tuple[Any, ...]], tuple[Any, ...]]rzFCallable[[Sequence[_IntLike], Sequence[_IntLike]], Sequence[_IntLike]]rrHrwrsrituple[torch.dtype, ...]dtypestuple[Callable[..., Any], ...]rpc>^[TU]T5[5R"U4SjUR56-[5R"U4SjUR 56-$)Nc3<># UHn[UT5v M g7frqrrs rvr,Scan.get_free_symbol_uses..O>N"1m44>Nrc3<># UHn[UT5v M g7frqrrs rvrrHi"1m44ir)rr"r0rrrr}s `rvr"Scan.get_free_symbol_usess] G ( 7l  Od>N>NO l  HdiiH   rxc>[UR5[UR5-[UR5:Xde[TU]5 grq)rrrrrrhrs rvrhScan.__post_init__=4;;#d&6&6"773tyy>III rxc"^URX45m[U4SjUR55n[R"UR UR U5n[R"U=(d SU"T5X`R5$)Nc32># UH o"T5v M g7frqrrrrs rvr'Scan.store_reduction..D^x}}^r ) rrrpr^rrrr!ri)r`r#r$r% scan_varsrresultrs @rvrScan.store_reductionskll4+DT^^DD$++t?yy  $9gclF;L;L4M  rxcg)Ncustomrrgs rvr%Scan.get_reduction_typesrxcUR$rq)rrgs rvr(Scan.get_reduction_sizer~rxcUR$rqrrgs rvr Scan.get_size yyrxcUR$rqrrgs rvrScan.get_pointwise_sizerrxcX[UR5[UR5-$rq)rrrrgs rvrScan.index_length!4;;#d&6&6"777rxcURUR5nURUR[R5nUR X5nU4$rq)rrrr4rrr`rrrs rvrScan.inner_fn_argsC DKK(T--t}}=ll5)v rxcURUR5nURUR[R5nUR X#5n[ URXAS9$r)rrrr4rrr=rr`rrrrs rvrScan.inner_fn_free_symbolsM DKK(T--t}}=ll5)#DMM3TTrxT)can_fallback_to_atenc ^^^/USTQUTS-SQmUT/m[RRU[R5(dS/[ U5-$[ U5S:aB[RRU[R 5(dS/[ U5-$[RRn U R[T55n [ U5[ U5:XdeU R[R"U S55(a=[[ U55V s/sHn [RUX,X<US9PM sn $UR!UUSUSTTTUU S9up}["nU S:at[$R&R(SL=(d [*=(a [,S:=(a [ U5S:HnU(dU(aS/[ U5-$Sn O[.nSUUU4Sjjn[[ U55V s/sH/n [0RU"S UX,UX<UUTTUUUU S. U D65PM1 nn UHnUR35 M U$s sn fs sn f) Nr5r-r)rrraxispointwise_rangesrr scan_numelz3.3.0c>[U5[T5:Xde[U5[T5:Xde/USTQUQUTSQ$rqr)r scan_indexrrrs rvrScan.create..reindex Sz?c+&66 66u:%5!66 66>U5D\>J>tu> >rx) rrrrrprrrrrrwri)rrKrrKrrer)r`rrr8SCANrTUPLE_REDUCTIONrrr[r-rLerrrrrrversionhip has_tritontriton_version SplitScanrmr)rrrrprrrrwrrrrrir scan_typesupports_splitrr~rrrs ` @@rvr Scan.creates=T%4[<4q +;<Dzl ww""6>+>+>??6CK' ' v;?177#6#6 N22$ $ 6CK' '77##&&}['AB 6{c)n,,,  ) )%((:q*A B B%*#f+$6%7L   ! .&4 ! %7 &)^^)q\-#!!&4 & " > !!T)Wj.V^w=V%v;!# "' 6CK//!"J%  ? ?.!&c&k 2% $!3 #   ! .!&4'+ +)##1!- "!3%  *F NN N s ;$I3 6I8c L^^SUU4Sjjn [RUUUU UUSUS9$)Nc,>T"/USTQUQUTSQ5$rqr)rrmrrs rvr6#Scan.num_splits..wrapper_fnK s*Fc%4jF=F3tu:FG Grxr)rrrvrrrur_r)rrKrmrKrr_)rtr) rrrrrrrrrr6s `` rvrScan.num_splits> sA H H###(!&$  rxrr`rr>) r#rOr$z%Callable[[Sequence[_IntLike]], Never]r%rKrr8rrrNrlrJrir)rrTrrrpz+tuple[Callable[[Sequence[Expr]], Any], ...]rrfrrsrrrwrHrrrr rSequence[Optional[TensorBox]])rrTrrFrrVrrsrrfrrfrrrrrr_)rrrrrr"rhrr%r(rrrrrr rHrrrrr rs@rvrrs MM SS!! ##--      "  7    $     8 U )6(=(=_&*__(_? _  _  _R_&_#__ '__B   7    (  # R   (  rxrc\rSrSrSrg)ri[ rNrrrrrrrxrvrr[ srxrcX^\rSrSr%S\S'S\S'S\S'S\S'S \S 'S \S 'S \S'S\S'S\S'SSU4SjjjrS U4SjjrS!SjrS"SjrS#Sjr S#Sjr S#Sjr S$Sjr S%Sjr SSSjjr\\R"4S&Sjj5rSrU=r$)'Sorti` rf sort_rangesrz:Callable[[Sequence[Expr], Sequence[Expr]], Sequence[Expr]]rrHrwrsrirrrrprstable descendingc>^[TU]T5[5R"U4SjUR56-[5R"U4SjUR 56-$)Nc3<># UHn[UT5v M g7frqrrs rvr,Sort.get_free_symbol_uses..u rrc3<># UHn[UT5v M g7frqrrs rvrrx rr)rr"r0rrrr}s `rvr"Sort.get_free_symbol_usesq s[ G ( 7l  Od>N>NO l  HdiiH   rxc>[UR5[UR5-[UR5:Xde[TU]5 grq)rrrrrrhrs rvrhSort.__post_init__| rrxc6^URX45m[U4SjUR55n[R"UR XPR UR5n[R"U=(d SU"T5X`R5$)Nc32># UH o"T5v M g7frqrrs rvr'Sort.store_reduction.. rrr ) rrrpr^sortrrrr!ri)r`r#r$r%rrrrs @rvrSort.store_reduction soll40DT^^DD$++v{{DOOLyy  $9gclF;L;L4M  rxcg)Nrrrgs rvr%Sort.get_reduction_type srxcUR$rq)rrgs rvr(Sort.get_reduction_size r~rxcUR$rqrrgs rvr Sort.get_size rrxcUR$rqrrgs rvrSort.get_pointwise_size rrxcX[UR5[UR5-$rq)rrrrgs rvrSort.index_length rrxcURUR5nURUR[R5nUR X5nU4$rq)rrrr4rrrs rvrSort.inner_fn_args rrxcURUR5nURUR[R5nUR X#5n[ URXAS9$r)rrrr4rrr=rrs rvrSort.inner_fn_free_symbols rrxc  ^^^/USTQUTS-SQmUT/m[RRU[R5(dS/[ U5-$[RR n U R[T55n Sn [RR=(a% U R[R"X55n U (dS/[ U5-$[ U5[ U5:XdeU R[R"U S55(a=[[ U55Vs/sHn[ R#UX.X>US9PM sn$SUUU4Sjjn[[ U55Vs/sH3n[$R#['SUX.UX>UUTTUUUUUS. U D65PM5 nnUHnUR)5 M U$s snfs snf)Nr5r"r-c>[U5[T5:Xde[U5[T5:Xde/USTQUQUTSQ$rqr)r sort_indexrrrs rvrSort.create..reindex rrx) rrrrrprrrrrwrirr)rrKrrKrrer)r`rrr8SORTrrrr[r6r persistent_reductionsr-rrrrrrmrr)rrrrprrrrrwrr sort_numel max_rblockis_persistent_kernelrirr~rrrs ` @@rvr Sort.create s=T%4[<4q +;<Dzl ww""6>+>+>??6CK' '77##&&}['AB  MM / / Q..uxx /OP $6CK' '6{c)n,,,  ) )%((:q*A B B%*#f+$6%7L   ! .&4 ! %7  ? ?0!&c&k 2' &!3 %   ! .!&4'+ +##1!-!) $!3'  ,F NN Q s $G&:G+rr`rr>) r#rOr$rXr%rKrrKrrrNrJrir\)rrTrrrpz'tuple[Callable[[list[Expr]], Any], ...]rrfrrsrrrrrwrHrr rr)rrrrrr"rhrr%r(rrrrrr rHrrrr rs@rvrr` s/  GG!! ##-- L      "  2    '     8 U )6(=(=LL(L; L  L  LLL&LL 'LLrxrc:[USS9 g![a gf=f)NFfreezeT)r5rrts rvrr s&a. s c[USS9upUR5(aUR5 UR5$![a gf=fNFr)r5should_pad_strides pad_strides is_contiguousr)ru_bufferr4s rv is_contiguous_storage_and_layoutr" sS/%@  $ $ & &    ##%% sAA AAc [U[5(a[URUUUUUS9$[U[5(a5[URUUUUUS9upgXRR 54$[U[ 5(aU(asU(a6UR5 UR 5R5(deO6UbURX4S9 O#UbURXTS9 OUR5 [ U5UR 54$[U[5(a#[URUS9upXR4$[e)z Try to simplify x into a StorageBox and a Layout. allow_padding only affect how we apply stride_order. When allow_padding is True, we have the freedom to add padding when applying the stride_order. rwant_contiguous stride_orderrrrr)rrrmr5r3 StorageBoxrBufferrr rrrr8r4r) rurr%r&rrrr4buffers rvr5r5 sJ!Y$ FF+%''   !Z  ) FF+%''  &&##%%%!V !||~335555)11 2*22!3!!}alln,,!_%%* FF xx rxc^[USS9up#URU5$![a gf=fr)r5is_stride_orderedr)rur&r!r4s rv"is_stride_order_storage_and_layoutr-I s8/%@'' 55 s  ,,c[U[[45(a[UR5$[U[ 5(adUR n[UR[UR5-[-S:H5(+n[UR5=(d U$[U[5(a+UR5[RR ;$g)NrF)rrrmr( is_unalignedr3r8r4r-r7rRrrTr)rr`runaligned_buffers)rr4has_unaligned_layouts rvr/r/S s$J/00DII&&$((#8 MMN6<<8 8? Ja O$  DII&>*>>$}}!''";";;; rxc\rSrSr%S\S'SSSjjrSSjrSSjrSSjr\ S Sj5r S!S jr S"S jr S#S jr S$S jrS%S jrS&SjrS'SjrS(SjrSrSrS'SjrS'SjrS)SjrS*SjrSrS+SjrSrg),BaseViewie rnr3c8URRU5$rqr3r"r!s rvr"BaseView.get_free_symbol_usesi syy--m<T"T"U55$rqrrinnerrs rvr$&BaseView.make_indexer..indexers & &rx)rrKrr)r3rr8)r`r$r=rs @@rvrBaseView.make_indexero s4 &&(%%' ' 'rxcp^^URR5mUR5mSUU4SjjnU$)Nc >T"T"U55$rqrr<s rvr&$BaseView.make_loader..loader| r?rxr)r3rr8)r`r&r=rs @@rvrBaseView.make_loaderx s4 %%'%%' ' ' rxc6URR5$rq)r3rrgs rvrBaseView.dtype syy""$$rxc6URR5$rqr3rrgs rvrBaseView.get_layout yy##%%rxc6URR5$rqr3rrgs rvrBaseView.get_device rJrxcgrqrrgs rvr|BaseView.get_origin_node rrxc6URR5$rqr3rrgs rvrBaseView.get_name yy!!##rxc"UR5$rqrrgs rvrBaseView.get_pointwise_size }}rxc8URRU5$rqr3rrs rvrBaseView.mark_reuse yy##E**rxc6URR5$rqr3rrgs rvrBaseView.has_exceeded_max_reads yy//11rxc6URR5$rqr3rrgs rvrBaseView.realize yy  ""rxc6URR5$rqr3rrgs rvrBaseView.realize_hint yy%%''rxc6URR5$rqr3rrgs rvrBaseView.get_storage_numel yy**,,rxc6URR5$rqr3r+rgs rvr+BaseView.is_extern yy""$$rxc6URR5$rq)r3is_module_bufferrgs rvrpBaseView.is_module_buffer yy))++rxc6URR5$rqr3rtrgs rvrtBaseView.get_read_names yy''))rxc[R"[SS5 [UR 5UR 55R sSSS5 $!,(df  g=fr)rrrr?rrrrgs rvrsBaseView.get_reads sD \\.*:D A&  " e B A As 2A A'czUn[U[5(a#URn[U[5(aM#U$rq)rrr3r3)r`rus rvrBaseView.unwrap_view s1H%%AH%%rxcUR5n[R"[SU5"U5n[ UUR 5UUR 5S9$r*)rrrr.rrrr/s rvr1BaseView.constant_to_device sN!!#n.?HP.."==?   rxrNr`r)rz*Callable[[Sequence[Expr]], Sequence[Expr]]rWrUrErGrQrAr[rJr^rIrNr?rhrn)rrrrrr"r8rrrurrrr|rrrrrrrr+rprtrsrr1rrrxrvr3r3e s L=C%%&&$+2#(-%,*  rxr3cP\rSrSr%S\S'\S5r\S5rS Sjr Sr Sr g ) ri rerct[RRn[[ [ R U55nUR5nS/[U5[U5- -[U5-n[U5[U5:Xde[[U55HnXS:XaX4ceX4X'MX4bN[RRRR[ R"X4S5SS9(aMoURXX4- SS9S:XaMS5e U$) zReplace `-1` with correct sizesNrr5Tsize_obliviousrfallbackzKBroadcast failed in ExpandView({x.get_size()}, {new_size}) on dimension {i})r`rrrrrrrrrr evaluate_exprrr)runew_sizerold_sizers rv_normalize_sizeExpandView._normalize_size s77##ELL(34::<6S]S]:;d8nL8}H ---s8}%A{b {...&k $(8(8(B(B(P(Pa()Q) ))(+ *Ca)PTUUaU&"rxc URX5n[U5(Ga3[U5up4[U5[UR5- nUS:de[ R R/U-n[URUR5H|upxUR[RRRR[ R "US5SS9(dUO[ R R5 M~ [#UR$UR&[)U5UUR*5n [-X9S9$[/XS9$)Nrr5Trr2)r3r)rrr5rrrrrrrrr`rrrrrr6rrrr7r8r) rrurr;r<skipr=rrr>s rvrExpandView.create s&&q3  # #"7": Gx=3z#77D19 9'',,$.J #J$5$5z G !!77++55CCq)$D !H%!!  X!! J#C Cq00rxcUR$rqrrgs rvrExpandView.get_size rrxc^^UR5nURR5m[U5[T5- mUU4SjnU$)Nc>[UTS5n[U5[T5:Xde[[T55H*nTUS:XdM[RR X'M, U$r)rrrrrr)rractualrs rvr*ExpandView.make_reindexer..reindex s]tu&Eu:V, ,,3v;'!9>$ww||EH(Lrx)rr3r)r`targetrrrs @@rvr8ExpandView.make_reindexer s>##%6{S[( rxrNrJ) rrrrrrrrr rrr8rrrxrvrr s8 4116rxrcP\rSrSr%S\S'\S5r\S5rS SjrSr Sr g ) PermuteViewi redimsc URU5n[U5[[[U555:Xde[ U5(a}[ U5up4[ URURUVs/sHoTRUPM snUVs/sHoTRUPM snUR5n[X6S9$[XS9$s snfs snf)Nr2)r3r) _map_neg_dimsr0rrrr5r6rrrrr7r8r)rrurr;r<rr>s rvrPermuteView.create s  &$:eCI.>#????  # #"7": G$!!  -12T#T2/34t!""1%t4!! J#C C-- 34s :C CcXUVs/sHo"S:aUO [U5U-PM sn$s snfrr)rrr(s rvrPermuteView._map_neg_dims* s+@DEaxSY_4EEEs'c[URUR55[[[ UR555:XdeUR R 5nURVs/sHo!UPM sn$s snfrq)r0rrrrr3r)r`rrs rvrPermuteView.get_size. sq$,,TYY78J #dii. !=    yy!!#!%+AQ+++s8B cN^[UR5VVs0sHupX!_M snnm[[UR55Vs/sHnTUPM snm[ T5[ [[UR555:XdeU4SjnU$s snnfs snf)Nc8>TVs/sHoUPM sn$s snfrqr)rrinvs rvr+PermuteView.make_reindexer..reindex: s&)*c!Hc* **s)rrrrr0)r`rrrrs @rvr8PermuteView.make_reindexer5 s )$)) 45 4qt 45$S^454!s1v45#*U3tyy>-B"CCCC +65s B B"rNrJ) rrrrrr rrrr8rrrxrvrr s: .."FF,rxrcJ\rSrSr\SS.Sj5r\SSj5rS SjrSr g) SqueezeViewi@ N)r(c [U5(Ga![U5up4/n/nUb=[U[5(dS5eSU::aU[ UR 5:de[ [UR UR55HmunupUc,US:wa$URU5 URU 5 M5M7Xr:wa$URU5 URU 5 M`US:XaMhS5e [URURUUUR5n [X:S9$Uc;[R!XR#5V s/sH oS:wdM U PM sn 5$UR#5US:Xde[R!U[ UR#55VV s/sHup{Xr:wdM U PM sn n5$s sn fs sn nf)Nzexpected integer dim argumentrr5zexpected squeezed size to be 1r2)rr5rrrsrrrrrrr6rrr7r8r9rr) rrur(r;r<rr=rrrr>rs rvrSqueezeView.createB s  # #"7": GHJ!#s++L-LL+CxC#joo*>$>>>%.s:??JDUDU/V%W!>D;qy -"))&1!x -"))&1#qyJ*JJy&X%!!  !! J#C C ;;;qjjl"El1f1l"EF F::<$) ));;q1::<1H"U1HAH11H"UV V#F#Vs G $G 6 G G c^^UVs/sH oS:wdM UPM nn[U5VVs/sHup1US:wdM UPM snnm[U5mSUU4SjjnX$4$s snfs snnf)Nr5c>[U5[T5:Xd UST35e[RR/T-n[ TU5H up#X1U'M [ U5$)N )rrrrrr)rr-rrlengthnot_ones rvr%SqueezeView.squeezer..reindexn sbu:W- C%'/C C-/Igu-!"#.# #rx)rzlist[sympy.Expr]rztuple[sympy.Expr, ...])rr)rrrrrrrs @@rvsqueezerSqueezeView.squeezerh sa#.t!AvAt.!*4;AF1;T $ $  /;s AA A A c[S5e)Nzuse SqueezeView.create())AssertionError)r`r3s rvrsSqueezeView.__init__w s788rxr)rrmr>) rrrrr rrrrrsrrrxrvrr@ s3"#W#WJ ! !9rxrcb\rSrSr%S\S'S\S'SrS SjrS Sjr\r\ S 5r SS jr S r g ) GenericViewi{ rerrrcUR$rq)rrgs rvr8GenericView.make_reindexer ||rxc[[UR55Vs/sHn[[R U5PM nn[ URU55nSSR[[U55SU3$s snf)Nzlambda , r) rrrrZr4r rrrrr)r`r index_old index_news rv reindex_strGenericView.reindex_str svCHTYYCX CXa *4::q 9CX  i01 3sI#6789+FF  s$BczURURSUR3SUR53/5$)Nsize=zreindex=)rr3rrrgs rvrGenericView.__str__ s= YY% {+x8H8H8J7K-L M  rxc$U"U[U5US9$)Nr3rr)r)rrurrs rvrGenericView.create sX@@rxcUR$rqrrgs rvrGenericView.get_size rrxrNr[rJ) rrrrrr8rrr r rrrrrxrvrr{ s>  G HAArxrc\rSrSr\S5r\S5r\S5r\S S Sjj5r \S S Sjj5r Sr g) r9i c[R"U5n[R"U5n[RRR R nU"[R"US55(aX-nU$r)rrr`rrrrLt)rrrs rvhandle_negative_indexView.handle_negative_index s[ll3||D!((22@@ #q) * **C rxc^ [U[[45(deURUR 5U5um n[ R RRT U5(aU$Sn[[T 55S:d[[U55S:aSnSU;aU 4SjnU"U[U5US9$[U5(dU(aU(a%[U5(d[RU5n[USS9upV[UR UR"U[$R'U5UR(5n[+XWS9$UR-T U5nU"U[U5US9$)NFrTc4>[S/[T5-5$r)rr)rrs rv fake_reindex!View.create..fake_reindex saS3x=011rxr)r%r2)rrrrresolve_negative_sizerr`rrstatically_known_list_equalsrr(r" ExternKernelrequire_contiguousr5r6rrrrr7r8r:) rrurunbacked_symbols_in_sizesrr;r<r>rrs @rvr View.create sT(UDM2222 66qzz|XN( 77   8 88 L LH$)! %h/ 01 4(23a7(, % = 2ADNLI I -a 0 04M(2RST2U2U !33A6"74"P G$!!  11(;!! J#C C--hAX@@rxcPUVs/sH,n[RRRU5PM. nnUVs/sH,n[RRRU5PM. nn[ U5n[ [ U55HInXS:XdM [RRX'[[U5[U55X' O [RRR[U5[U55 X4$s snfs snf)Nr) r`rrrrrrrrOner1r[ guard_equals)rrrurs rvrView.resolve_negative_size s:BC(QAGG$$--a0(C:BC(QAGG$$--a0(C>s8}%A{b #ggkk &}X'> h@WX  & %%mH&=}X?VW!!DCs 3D3D#NcURXU5nU$![[4a= [U5/nURX5nURXR5n[ Xg5nU$f=frq)_dynamic_reshape_indexerr IndexErrorr[r)rrr dense_dimrflatrrs rvr:View.dynamic_reshape_indexer sr :228yQG  + :!(+,D33HCH33DCH%h9G  :sA A#"A#c^^[RRRn[ [ U55Vs/sHn[ [RU5PM snm[[TU55n[U5nUSL=(a& U[ U5S- :g=(a [ U5S:HnU(a'UceURU5nURU5 /mU(GaKU(GaCUR5n UR5upU S:Xa=TR[RR5 URX45 GOU S:XaURU 5 GOU"U 5U"U 5:Xa<TRU 5 [RRR!X5 GOfU"U 5U"U 5:a~U"U 5U"U 5:a1UR5upX-U -n X-n U"U 5U"U 5:aM1TRU 5 [RRR!X5 OU"U 5U"U 5:a[RR"nU nTR[%XU55 X-nU"U 5U"U 5:aGUR5nTR[%XU55 X-nX-n U"U 5U"U 5:aMG[RRR!X5 O[&eU(a U(aGMCU(alUR5n [RRR!U S5 TR[RR5 U(aMlU(aEUR5up[RRR!U S5 U(aMEUbB[ U5S:Xa3TR)5 TR5nTR+UU5 OTR)5 [ T5[ U5:XdeUU4SjnU$s snf)z7 Perform a reshape entirely by modifying indexing math Nr5c>^[U5[T5:Xd[U5[T545e[[TU55m[U4SjT55$)Nc3<># UHn[UT5v M g7frq)r\)rru replacementss rvrAView._dynamic_reshape_indexer..reindex..B sHiA|44ir)rrrr)rrr% view_exprs @rvr.View._dynamic_reshape_indexer..reindex? sMu:T* CSZT,C C*D% 01LHiHH Hrx)r`rrrrrrZr4VIEWrrrrrrrrrr3rreverseinsert)rrrrr stack_new stack_oldreordering_dense_dimold_dimsize_oldvarsize_newvar2 size_new2divisormodulus dense_exprrr%rs @@rvrView._dynamic_reshape_indexer s GG$$.. CHH BV BVQ *499a 8BV T8,- N  T ! #S^a// #H "  ( ((mmI.G   W % I }}H%MMOMC1}  .  #1Q  *8$ ((;;  %  --hA8$y'::)Ih,??&/mmoOD/C/C'3H )Ih,??  %  --hA8$y'::''++"  w!GH!+)Ih,??'mmoG$$_S7%KL%/G'1H  )Ih,??   --hA$$=II@ }}H GG   ) )(A 6   UWW\\ *i %MMOMC GG   ) )(A 6i  S]a%7    "J   Y 3    9~X... I W s$Q/rrq)rrZrrZrr]r&Callable[[Sequence[_T]], Sequence[_V]])rr]) rrrrrrrr rrr:rrrrxrvr9r9 s*A*AX " " $( $%!  0  RRrxr9c^\rSrSr%SrS\S'SU4SjjrSSjr\rSSjr SSjr SS jr \ S 5r SS jrS rSS jrSSjrSSjrSrSSSjjrSS SjjrS!SjrSrU=r$)"r8iG z*Pretend our storage has a different layoutrr4c>[TU]5 [UR[5(a0[ R USURR55 gg)Nr3)rrhrrr3r3rr_rrs rvrhReinterpretView.__post_init__M sC  dii * *   tVTYY-B-B-D E +rxcPURURUR/5$rq)rr3r4rgs rvrReinterpretView.__str__R s&     rxc6URR5$rqrQrgs rvrReinterpretView.get_name\ rSrxc.URR$rq)r4rrgs rvrReinterpretView.get_device_ s{{!!!rxcgrqrrgs rvr|ReinterpretView.get_origin_nodeb rrxc.URR$rq)r4rrgs rvrReinterpretView.dtypee s{{   rxc@[URR5$rq)rr4rrgs rvrReinterpretView.get_sizei sDKK$$%%rxc@[URR5$rq)rr4rrgs rvrReinterpretView.get_stridel sDKK&&''rxc^SU4SjjnU$)NcZ>TRR5n[R"TR 5U"U55nTRR TR R :wa6[R"UTR TR R 5$U$rq)r4rr^loadrrr3to_dtype_bitcast)rr$ tmp_loaderr`s rvr&+ReinterpretView.make_loader..loaderp spkk..0G$--/75>BJ{{  DIIOO3++J DIIOOTT!!rxrrKrr_rr`r&s` rvrReinterpretView.make_loadero s " rxc6URR5$rq)r4rrgs rvrReinterpretView.make_indexerz {{''))rxcUR$rqr4rgs rvrReinterpretView.get_layout} rrxcgrqrrgs rvrReinterpretView.freeze_layout rrxc[URRU5[URRU5-[URRU5-$rq)rr4rrr7r!s rvr"$ReinterpretView.get_free_symbol_uses sQ T[[--} =t{{11=A Bt{{11=A B rxc p[RRRURUR R UR RUR RUb URO#[RRRUR RS9$r) r`r wrapper_codecodegen_reinterpret_viewr3r4rrr7 writelinerrs rvr!ReinterpretView.codegen_reference sww##<< II KK   KK   KK   & 2F  8L8L8V8V++## =  rxcgrrrgs rvrReinterpretView.num_reads rxrr>r[rQrArJrUrWrGr`rkrqrPri)rrrr__doc__rrhrr rrr|rurrrrrrrr"rrrr rs@rvr8r8G s4 NF  H$"!!&( * %* ! !   rxr8cf\rSrSr%SrS\S'\S5rS Sjr\r \ S5r S Sjr SS jr S rg ) DtypeViewi z(Pretend our storage has a different typerF target_dtypec[U5(aM[U5up4[URUURUR UR 5n[X5S9$[XS9$)Nr2)r3r$) rr5r6rrrr7r8r#)rru new_dtyper;r<r>s rvrDtypeView.create sa  # #"7": G$!!!!!! J#C Ca88rxcPURURUR/5$rq)rr3r$rgs rvrDtypeView.__str__ s  4+<+<=>>rxcUR$rq)r$rgs rvrDtypeView.dtype s   rxc6URR5$rqr3rrgs rvrDtypeView.get_size rSrxcL^^TRR5mUU4SjnU$)Nc|>[R"T"U5TRTRR5$rq)r^r r$r3r)rr=r`s rvr&%DtypeView.make_loader..loader s*''c D4E4EtyyW Wrxr3r)r`r&r=s` @rvrDtypeView.make_loader s" %%' X rxrNr[rJrU)rrrrr!rr rrr rurrrrrrxrvr#r# sE2 9 9?H !!$rxr#c8\rSrSr\S5r\SSj5rSrg) SliceViewi c\^^^^^ ^ [RRm UR5Um[ SX4T455(a![ R m [ RmOT Rm T RmUU U 4SjmUUU4SjnU"USTS5nU"XCTT5nX44$)zb Normalize start and end such that both are in the range [0, x.get_size()[dim]] and start <= end. c38# UHn[U5v M g7frq)r(rrus rvr0SliceView.normalize_start_end.. sH1GA$Q''1Grqc>TRX5(aUOT"X5nTRX25(aUnU$T"X25nU$rq)statically_known_geqr")rulowerupper clamped_lower clamped_fullmax_funcmin_funcrs rvclamp,SliceView.normalize_start_end..clamp s^221<<(1BT  00FF   m3   rxcD>UcU$TRUT5nT"XU5$rq)r)rr<r=r4rBrdim_sizes rv clamp_wrap1SliceView.normalize_start_end..clamp_wrap s,{++C:CU+ +rxr) r`rrrrBrMinMax evaluate_min evaluate_max) rrur(startendrFrBrEr@rArs ` @@@@@rvnormalize_start_endSliceView.normalize_start_end s 77##::<$ H%h1GH H HyyHyyH,,H,,H  , 5!Xq1Xx8zrxc ^^^^ [R"T5m[T[R5(dTS:deTS:XaUS:aTS:XaU$[ UR 55m U(aURUTTU5umn[UT- TS- -T5T T'[U5(av[U5upx[ UR5n U TT-U T'[URURT U URURTT--5n [!XzS9$UU UU4Sjn [#UT U S9$![a Nf=f)Nrlr5r2c>[U5[T5:Xd SUST35e[U5nUTT-T-UT'U$)Nz wrong ndim r)rr)rr(rrLsteps rvr!SliceView.create..reindex sPu:X. P+eWAhZ0P P.KEsd*U2E#JLrxr)rrrrr TypeErrorrrrNr2rr5rr6rrr7r8r5) rrur(rLrMrRrBr;r<r=r>rrs `` ` @rvrSliceView.create sQ||D!$ ++tax77 zcY.419 % 00CDJE3 uq!94@  # #"7": Gj//0J(o4JsO$!!  !!J$5$5c$:U$BB J#C C  ah@@E   sE E  E rN)r5T)rrrrr rNrrrrxrvr5r5 s+""H(A(Arxr5cR\rSrSr%S\S'S\S'S SjrS SjrSSjrSS jrS r g ) BaseConstanti rFrrTrcgNrrrgs rvrBaseConstant.get_size srxcUR$rqrrgs rvrBaseConstant.get_device rrxcgrqrrgs rvr|BaseConstant.get_origin_node# rrxc[5$rqr/rgs rvrsBaseConstant.get_reads& rrxrNrJrQrArh) rrrrrrrr|rsrrrxrvrWrW s"  rxrWcR\rSrSr%S\S'S\S'S\S'S SjrSS jrSS jrS rg )Constanti* r rrFrrTrc^SU4SjjnU$)NcZ>[R"TRTR5$rq)r^rrrrr`s rvr&$Constant.make_loader..loader1 s<< DJJ7 7rxr rr s` rvrConstant.make_loader0 s 8 rxcgrqrrgs rvrConstant.realize6 rrxc@[URURUS9$)N)rrr)rbrrrs rvr1Constant.constant_to_device9 sdjj 6JJrxrNrUrNrn) rrrrrrrr1rrrxrvrbrb* s# J    KrxrbcH\rSrSr%S\S'S\S'S\S'S SjrS S jrS rg )IndexingConstanti= r rrFrrTrc^SU4SjjnU$)NcZ>[R"TRTR5$rq)r^rrrres rvr&,IndexingConstant.make_loader..loaderD s>>$**djj9 9rxr rr s` rvrIndexingConstant.make_loaderC s : rxc@[URURUS9$)N)rrr)rmrrrs rvr1#IndexingConstant.constant_to_deviceI sdjj 6RRrxrNrUrn)rrrrrrr1rrrxrvrmrm= s J   Srxrmc b[S[U[RU5U555$)Nc3H# UHupnUS:H=(d X:Hv M g7frr)rleftrightrs rvr2is_contiguous_strides_for_shape..P s." D  "T]"" s ")rrrr)rr's rvis_contiguous_strides_for_shaperyM s5 !$ N55e[[U5R5erqrrgs rvrOutputSpec.get_device` rrxc>[[U5R5erqrrgs rv storage_sizeOutputSpec.storage_sizec rrxrNrQri)rrrrr!rrrrrxrvrr\ sC77rxrc\rSrSrS\"S54SSjjrSSjr\rSSjrSSjr SSjr \ SS j5r SS jr SS jrS r\ S 5rSrSrSrSSjrSSjrSSjrSrg)rig NrcUc[RU5nXlX l[ U5[ U5:Xd SUSU35e[ SU55(deX0lX@lXPlg)Nr , stride=c3N# UHn[U[[45v M g7frq)rrrrsrs rvr"Layout.__init__..v s#66t.sorted_indices s,J=@AS'$$W-SA AAs-rTrF) rrrrr`rrrrrrr _shape_envr) r`rrr( non_1_indicesrrstride_orderedexprs rvr,Layout.is_stride_ordered sl4;;3u:--- $DII. .ww))#):a? .  +88-Q++a.-8#01=aq=1 B u%E *s5z"A'-yN8 $#s5zA~&A!$~!e'<. s* 6 q3 . / /6s-/r5N)rLT)r}rr6pad_channels_lastrrr`get_current_noderrFgetrrchainrrrrpadding_stride_thresholdrLrnum_comprehensive_padding) in_stridesrraligncurrent_fx_noder&rr new_stridespaddedrankrprev_idxrs rv _pad_stridesLayout._pad_strides s $E* z?a  ''F,N,N - -  ,,. ?F + +0D0D0H0H u1 1   __Z6    ' 3 ,\: "'J"89"8Qq"8 9&' qM"":ab>;ID!(+H *T^;F777FNa    !F- 88tL Q  .  Trxrc"\rSrSrSrSSjrSrg)r6i<z A Tensor layout we cannot changec^U4SjnU$)z1A closure containing math to read a given elementc$>[U5[TR5:Xde[U5[TR5:XdeTRn[ UTRTR5Hup#nUS:wdMXU--nM U$r)rrrr7r)rrrrszr`s rvr$)FixedLayout.make_indexer..indexerBs{u:T[[!11 11u:TYY/ //[[F#&udkk499#ER7#Fl2F$FMrxrr`r$s` rvrFixedLayout.make_indexer?s rxrNrW)rrrrr!rrrrxrvr6r6<s * rxr6c^\rSrSrSrSr\S5r\S5r\S5r \S5r \S5r SS jr SS jr S rS rSSU4S jjjrSrU=r$)riNz(A Tensor layout we are allowed to changeFc[U5S:Xa/$[RR/n[ USS5HnUR X!S-5 M [ [ U55$)Nrr5r)rrrrrrr)sizesreversed_stridesrs rvr!FlexibleLayout.contiguous_stridesTs^ u:?I!GGKK=U12Y'D  # #DB+?$? @(H-.//rxc[[[U555[U5:XdX45e[RR nS/[U5-nUHnX#U'X U-nM U$)z Create a stride based on the order the dimensions should be filled in. In this format, channels last would be: [1, 3, 2, 0] N)r0rrrrr)rr next_strider:rs rv fill_orderedFlexibleLayout.fill_ordered]sm%E +, 50AAQE>QAggkk &3u:%A$AJ%a0Krxc[[[U555[U5:Xde[U5n[R X5$)zz Create a stride based on the sorted order of a permuted range. In this format, channels last would be: [3, 0, 2, 1] )r0rrrrr)rrrs rvrFlexibleLayout.stride_orderedns@%E +, 50AAAA,U3 **5==rxcDU[R:Xa[RU[5$U[R :Xa[RU[ 5$U[R:Xa[RU5$[RSU5 [e)a9 Create a stride based on a memory format. Memory format is translasted into a stride order, so channels_last is the same as: FlexibleLayout.stride_ordered(sizes, [3, 0, 2, 1]) This interface does not support memory_format `torch.preserve_format` which should be used to deduce a format from another source z>stride_ordered_for_memory_format, unsuppored memory_format: %s) r channels_lastrrNHWC_STRIDE_ORDERchannels_last_3dNHWDC_STRIDE_ORDERcontiguous_formatrrrr)r memory_formats rv stride_ordered_for_memory_format/FlexibleLayout.stride_ordered_for_memory_formatzs E// /!008IJ J e44 4!008JK K e55 5!44U; ; IIP & %rxc([U5[U5:XdeUVs/sH,n[RRR U5PM. nn[ [ [U55URS9n[RX5$s snf)z Create a stride that has the same stride order as given stride For example, if given stride is [1000, 1, 100, 10], the fill order should be [1, 3, 2, 0] r) rr`rrrrr __getitem__rr)rrrurs rv same_orderedFlexibleLayout.same_orderedsu5zS[(((BHI&Q!''""55a8&IE#f+.F4F4FG **5==Js3Bc,URURU5nUR5(a-U(a&URX0RUR5n[ UR URURUUR5$rq)rrrrrr6rr7)r`rrr=s rvas_stride_orderFlexibleLayout.as_stride_orderso((E:  " " $ $**:yy$**MJ KK JJ II  KK   rxcUnUR5(a-U(a&URX0RUR5n[ UR URURUUR 5$rq)rrrrr6rr7)r`rrr=s rvas_exact_stridesFlexibleLayout.as_exact_stridess^"  " " $ $**:yy$**MJ KK JJ II  KK   rxcURURU5nUR5(a&URX RUR5n[ UR URURUUR5$rq)rrrrrr6rr7)r`rr=s rv as_fill_orderFlexibleLayout.as_fill_ordersm&&tyy%8  " " $ $**:yy$**MJ KK JJ II  KK   rxcURURU5nUR5(a&URX RUR5n[ UR URURUUR5$rq)rrrrrr6rr7)r`rr=s rv as_same_orderFlexibleLayout.as_same_ordersm&&tyy&9  " " $ $**:yy$**MJ KK JJ II  KK   rxc>U(a[RX45nO[RU5n[TU]XX55 grq)rrrrrs)r`rrrr&r:rs rvrsFlexibleLayout.__init__s5 $11$EG$77=G 6rxrr`rqr>)rrrrr!rrrrrrrrrrrrrsrr rs@rvrrNs2N00  > >&&0 > >        77rxrc@^\rSrSrSrSU4SjjrSSjrSrSrU=r $) NonOwningLayoutiz,Is a view into the storage of another tensorc>UR5n[TU] URURUR UR 5 Xlgrq)rrrsrrrrview)r`rr4rs rvrsNonOwningLayout.__init__s?"  MM LL KK MM   rxc>UR5R5$rq)rrrgs rvrNonOwningLayout.make_indexers}}++--rxcURR5RnUS:XagSSKJn [ R RRX5$)NrTr5) ALIGNMENT) rrr7utilsr r`rrstatically_known_multiple_of)r`r7r s rvmaybe_guard_aligned#NonOwningLayout.maybe_guard_alignedsB%%'.. Q;$ww<[U[5(d[SUS35eUR5n[TU]UR URURURURS9 X l X0l g)NzJA `CommBufferLayout` can only be initialized with a `FlexibleLayout` (got z).r) rrrrrrrsrrrrr7rr)r`r4rrfixedrs rvrsCommBufferLayout.__init__s &.11 ++1("6  ! <<++<<<<  !1$rx)rr)r4rrrrr) rrrrr!rrsrr rs@rvrrs; %$O%%)% %%rxrc\rSrSr%S\S'\R "SS9rS\S'\R "SS9rS\S 'SS jr S r SS jr S r g) NoneLayoutirRrcS/$rrrrxrvrNoneLayout.)srxdefault_factoryrbrcS/$rrrrxrvrr*s1#rxrcgrrrgs rvrNoneLayout.storage_size,r rxcU$rqrrgs rvrNoneLayout.as_fixed/ rxcUR$rqrrgs rvrNoneLayout.get_device2rrxrNrirQ) rrrrrrprqrrrrrrrrxrvrrsC #"!'' DD)D#))+FFIFrxrc^\rSrSrS U4Sjjr\S Sj5r\RSSj5rSSjrSSjr Sr \ SSj5r S r SS jrS rU=r$)MutationLayoutSHOULDREMOVEi6c>[TU]UR5UR5UR 5S5 XlUR 5R5n[RRU5 grq) rrsrrrr get_bufferrr`rmark_buffer_mutated)r`rrrs rvrs#MutationLayoutSHOULDREMOVE.__init__7sc   & & (     OO      ))+ ##D)rxc6UR5R$rq) real_layoutrrgs rvr!MutationLayoutSHOULDREMOVE.strideBs!(((rxcgrqr)r`rs rvrr2Fs rxc>UR5R5$rq)r1rrgs rvr'MutationLayoutSHOULDREMOVE.storage_sizeJs!..00rxcp^U4SjmT"UR5n[U[5(dS5eU$)Nc>[U[5(aT"UR5$[U[5(aT"UR 55$[U[ 5(aT"UR 5$U$rq)rrr+rr3r MutableBoxr3)r unwrap_viewss rvr9;MutationLayoutSHOULDREMOVE.get_buffer..unwrap_viewsNsb&"<==#FMM22&(++#F$6$6$899&*--#FKK00Mrxz1MutationLayoutSHOULDREMOVE must refer to a buffer)rrrr))r`rr9s @rvr-%MutationLayoutSHOULDREMOVE.get_bufferMs< dkk*&&)) ? ) rxc6UR5R$rq)r-r4rgs rvr1&MutationLayoutSHOULDREMOVE.real_layout] '''rxc UR5 [RRUR 55 [ U[ 5(a URnUR5 U(d[RUR5UR5UR5[UR5UR55VVs/sH.upE[RR R#XE5PM0 snnS9RnUR5 [ URR$[&5(de[)U5URlUR$s snnf)Nr-)rr`rr.rrrrmr3rrrrrrrrrrr4rr+)rsrcdst unsafe_aliasrSrTs rv realize_into'MutationLayoutSHOULDREMOVE.realize_into`s  ##CLLN3 c9 % %((C ""~~'mmo*!$CLLNCLLN C CGG$$11!7 C #d  #((//>::::4S9xxs5FcU$rqrrgs rvr#MutationLayoutSHOULDREMOVE.as_fixedr'rxc6URR5$rq)rrrgs rvr'MutationLayoutSHOULDREMOVE.make_indexerrrx)r)rrnrrrre)rrrrr)rr)r`rW)rrrrrsrursetterrr-r1r rCrrrr rs@rvr+r+6sg *)) ]]  1 (  D**rxr+cp^\rSrSr%S\S'S\S'S$U4SjjrS%SjrS&SjrS'S jrS(S jr S)S jr \ S*S j5r S+S jr S,SjrS-SjrS.SjrS/SjrSrSrS0S$SjjrS$SjrS$SjrS0S$SjjrSrS1SjrS2S3SjjrSrS4SjrS4SjrS5SjrS0S6SjjrS7S jr S8S!jr!S9S"jr"S#r#U=r$$):r)irOrrr4cF>[TU]5 URSS5 gr)rrhrbrs rvrhBuffer.__post_init__s   t4rxc>UR5R5$rq)rrrgs rvrBuffer.make_indexers --//rxcJUR(dU5eUR$rqrlrgs rvrBuffer.get_namesyy$yyyrxc[UR[5(aURR5$[ [ UR5R 5erq)rrr4rrrrrrgs rvrBuffer.get_examples@ dkk6 * *;;**, ,!$t{{"3"<"<==rxc>UR5R5$rq)rrrgs rvrBuffer.get_devices##%0022rxcgrqrrgs rvrBuffer.get_defining_oprrxc6UR5R$rq)rrrgs rvr Buffer.dtypes &&&rxc:/UR5RQ$rq)rrrgs rvrBuffer.get_sizes("''((rxc:/UR5RQ$rq)rrrgs rvrBuffer.get_strides*"))**rxc6UR5R$rq)rr7rgs rv get_offsetBuffer.get_offsetr>rxc[UR[5(a UR$[[ UR5R 5erq)rrr4rrrrrgs rvrBuffer.get_layouts7 dkk6 * *;; !$t{{"3"<"<==rxcUR$rqrrgs rvrBuffer.get_output_specrrxc"UR5$rq)rrgs rvrBuffer.get_storage_numels~~rxc[UR[5(a@[UR[5(d URR 5Ulgggrq)rrr4rrrrgs rvrBuffer.freeze_layoutsF dkk6 * *: KK4 4 ++..0DK4 *rxc[UR[5(deURRXS9UlgNr')rrr4rrrs rvr&Buffer.freeze_layout_with_stride_orders2$++~6666kk11%1U rxc[UR[5(deURRU5Ulgrq)rrr4rrrs rvr$Buffer.freeze_layout_with_fill_orders/$++~6666kk//6 rxc[UR[5(deURRU5Ulgrq)rrr4rrr s rvr $Buffer.freeze_layout_with_same_orders/$++~6666kk//7 rxc[UR[5(deURRXS9Ulgrj)rrr4rrrs rvr'Buffer.freeze_layout_with_exact_stridess9$++~6666kk22 3  rxc[RRR[R "UR 5S55$rrrgs rvrBuffer.is_zero_elementsrrxcv^TR5(a[[TR5S9$U4SjnU$)Nrc>TR5n[R"TR=(d SU"U55$r)rr^rrrr$r`s rvr&"Buffer.make_loader..loaders/'')G88DII2GENC Crx)rr rrr s` rvrBuffer.make_loaders3  " "=0@A A D rxc"UR5$rqrrs rvrBuffer.codegen_referencerVrxcgrqrrgs rvrBuffer.decide_layoutrrxc[UR[5(a%URRR 5/$grY)rrr4rrrrgs rvr:#Buffer.get_inputs_that_alias_outputs2 dkk? 3 3KK$$--/0 0rxc[UR[5(a%URRR 5/$grY)rrr4r+rrrgs rvr4Buffer.get_mutation_namess3 dkk#= > >KK&&//12 2rxc6[UR5/5$rq)r0rrgs rvrtBuffer.get_read_namess4==?+,,rxc[5$rqr/r!s rvr"Buffer.get_free_symbol_uses |rxc[5$rqr/rgs rvrBuffer.get_unbacked_symbol_defsrrxcgrqrrgs rvrBuffer.realizerrxcgrrrgs rvshould_allocateBuffer.should_allocatesrxrr>rWr[)rz!Union[torch.Tensor, sympy.Symbol]rQrBrErJrIrMrGrHr`rUrqrPror?rkrrNrI)%rrrrrrhrrrrrrurrrr_rrrrrrr rrrrrr:r4rtr"rrrrr rs@rvr)r)s   50> 3'')+(>  1 V78 ,1  U    -%*! !  rxr)cJ\rSrSrSSjrSSjr\RrS SjrSr g) OperationBufferi cU/$rqrrgs rvrOperationBuffer.get_outputss v rxcU$rqrrgs rvrOperationBuffer.get_defining_opr'rxcX[RU5 [RU5 grq)r)rhrwrgs rvrhOperationBuffer.__post_init__sT"%rxrNrrrwr>) rrrrrrrwr7rhrrrxrvrr s#55&rxrc\rSrSrSSjrSrg) InputBuffericgrrrgs rvrInputBuffer.num_readsr rxrNri)rrrrrrrrxrvrrsrxrc\rSrSrSrSrg) DonatedBufferi#aA Represents a donated buffer which is a saved tensor that is not alias to any fwd inputs, fwd user outputs, and bwd outputs. We generally cannot inplace reuse the input tensor memory during backward since it might be used in another function. However, donated buffer can be inplace reused during backward to save memory. rN)rrrrr!rrrxrvrr#srxrc8\rSrSr%SrS\S'SSjrS SjrSrg) r.i-NrRr,c^SU4SjjnU$)Nc>TR5R5n[R"[R R TR5TR5U"U55$rq) rrr^rr`r constant_namerr,rvs rvr&*ConstantBuffer.make_loader..loader1sPoo'446G88%%dmmot7K7KL rxr rr s` rvrConstantBuffer.make_loader0s  rxc[[RRUR 5U5UR S9$N)rr4)r.r`rrrr4rs rvr1!ConstantBuffer.constant_to_device:s/&&t}}?   rxrrUrn) rrrrr,rrr1rrrxrvr.r.-s.2O+2 rxr.cV\rSrSrS SjrS S SjjrS S SjjrSSjrSSjrSr g)NoneAsConstantBufferi@c[5$rqr/rgs rvrsNoneAsConstantBuffer.get_readsBrrxc[5$rqr/r!s rvr")NoneAsConstantBuffer.get_free_symbol_usesErrxNcJ[RRR$rq)r`rrnone_strrs rvr&NoneAsConstantBuffer.codegen_referenceJsww##,,,rxc[SS9$Nr)rrgs rvr$NoneAsConstantBuffer.get_output_specMs &&rxcgrrrgs rvr&NoneAsConstantBuffer.has_tensor_outputPrrxrrhr`rkrqrPrHrI) rrrrrsr"rrrrrrxrvrr@s0%*! ! -'rxrcN\rSrSr%S\S'S S SjjrS S SjjrS SjrSrg)riTrrc.[URU5$rq)rrr!s rvr"*ShapeAsConstantBuffer.get_free_symbol_usesXs  =99rxNch[RRRUR5$rq)r`rrcodegen_sizevarrrs rvr'ShapeAsConstantBuffer.codegen_reference]s!ww##33DII>>rxcgrrrgs rvr'ShapeAsConstantBuffer.has_tensor_output`rrxrr`rkrqrPrI) rrrrrr"rrrrrxrvrrTs+ J%*:!: !: ?rxrc^\rSrSr%S\S'SSjrSSjrSSjrSSjrSSjr SSS jjr SU4S jjr SS jr S S jr S!S jr\S"Sj5rS#S$Sjjr\S%Sj5rS&SjrSSjrS'SjrS'SjrS(SjrSrU=r$))ridrr3cURb UR$[URS5(aURR$g)z} Returns self.name if it exists, otherwise returns the name of the data node if that exists. If neither exist, returns None. Nr)rrr3rgs rvget_computed_buffer_name'ComputedBuffer.get_computed_buffer_namehs: 99 99  499f % %99>> !rxc6URR5$rqr3rrgs rvrComputedBuffer.num_readssrnrxc6URR5$rqr3rsrgs rvrsComputedBuffer.get_readsvrnrxc6URR5$rqrtrgs rvrtComputedBuffer.get_read_namesyrvrxc[R"[SS5 URR 5(aT[ UR 5URR5URR55sSSS5 $[ UR 5URR55sSSS5 $!,(df  g=fr) rrrr3r%r?get_store_functionrr(rrgs rvrComputedBuffer.get_read_writes|s \\.*:D Ayy++--*++-II002II002B A+++-II&&(B A AsA*C 1C  Cc[UR5U5[UR5U5-[UR5U5-URR U5-$rq)rrrr_r3r"r!s rvr"#ComputedBuffer.get_free_symbol_usess`* T]]_m <t0-@ At0-@ Aii,,]; < rxc>UR5(dVUR[RR;a.UR 5S:XaUR R5$[TU]5$r) r%rr`rmutated_buffersrr3rrrs rvrComputedBuffer.make_loadersZ'')) !8!88 A%99((* *w"$$rxcUR5R5R5n[UR[ [ [45(a+[URRURU5$[UR[5(de[URRURU5$rq) rrrrrr3rtrrr rrrr'rs rvr!ComputedBuffer.get_store_functions//#,,.;;= dii)T4!8 9 949944diiI Idii33 3349911499gF Frxc [UR[5(Ga[R"UR R 5UR R55uupnUR5Rn[SU55(deUVVs/sHkn[U[R5(dM$[URUVs0sH%ofS:wdM U[RR _M' sn5PMm nnnU(a[UR ["[$45(aUR R'X5nOUnUVs/sH,n[(R*R,R/X5PM. n nSSKJn U "XR555$gs snfs snnfs snf)aD If our layout is still flexible, try to determine the stride order based on stride orders of reads. TODO(jansel): A better algorithm here would look at downstream consumers of this value and try to do global graph-level layout optimization. This is also something just begging to be autotuned. c3v# UH/n[U[R[R45v M1 g7frq)rrr7StarDep MemoryDepr@s rvr0ComputedBuffer.get_fill_order..s2A1|33\5K5KLMMs79rr5pick_loop_orderN)rrr4rr7rr3rr(rrrrr\rrrrrrrr`rrr schedulerrr) r` index_varsrrrrvrrstride_lengthsrs rvrComputedBuffer.get_fill_ordersx dkk> 2 2.:.M.M ,,. 0L0L0N/ + (Z!((*00E  Aa!7!78Y 177n$WnUVPV_Q _n$WX  dii$66"ii// KG(GMR"MRTAGG$$11$@U"7&~}}GG#%X"s*%#F? F? F:-F: F?(3G:F?c[UR[5(a:UR5nU(aUR U5 gUR 5 ggrq)rrr4rrrrrs rvrComputedBuffer.decide_layoutsC dkk> 2 2'')E2259""$ 3rxc[R"URR5URR 5SS9up[ R "[SUR55 [UR5UR5(aUOUSSU/UQ76nSSS5 /n/n/n/nUR5HfupXS;a-U(aeURU5 URU 5 M:XS;deURU5 URU 5 Mh Xg4WXE44$!,(df  N=f)Nqrkr,r5r)r7rr3rr(rrr.rrArr%itemsr) r`r var_rangesr.r reduce_vars index_size reduce_sizerrs rvget_default_sizes_body%ComputedBuffer.get_default_sizes_bodys6(:: II ( ( *DII,H,H,JSV \\.*;T__=N O'')0022Ra DP !#   $$&DAG|&&!!!$!!!$G|#|""1%""1%'($0III)P Os 37D;; E c^^^TR5uup4nupgU(aU"X44XVU45uup4nupg/URR5QmUb[U[5(a[ U5S:XdeUup[U[ 5(de[U [5(de[SU 55(deURn X:Xd U U45eU V s/sH oT;dM U PM n n TU - m/UR5Qm[RRT[R5(dTR!UR#55 UUU4Sjn Xg-n [%['T55(+=(d [(R*(+nU "UU UU5unnnU "X}XN5unnn[,R."UUSS9uunnn[1UU"U5U"U5/UUU5nUU4U4$s sn f)a This is a main place where we do loop transformations in a backend-agnostic way. Here we: 1) Remove any 1 dimensions 2) Fuse contiguous dimensions together 3) Reorder dimensions based on stride orders Optional argument extra_indexing_constraints can be used to append additional indexing expressions to existing ones derived from buffer's body. This can be useful to fuse scheduler nodes with compatible ranges, e.g. (s0*s1*...,) and (s0, s1, s2, ...) on CPU by preventing indexing simplifications and obtaining index/reduce ranges for the scheduler node compatible with other nodes. Optional argument recompute_sizes_body_func can be used to recompute sizes and body on the default body. This can be useful to append additional loop transformations. rc3B# UHn[U[5v M g7frq)rrr)rfs rvr6ComputedBuffer.simplify_and_reorder..5sH4Gqz!T**4Gsc >T RXUT 5up$nU"U5nU(aD[RRR UU[ T X55up&n[ XV5nOUnX(U4$rq)_apply_loop_reorderingr`rr_simplify_loopsr;r) x_vars support_varsrsimplify_loopsreindex0rr_prunerindex_formulas memory_addrsr`s rvsimplify_and_reorderAComputedBuffer.simplify_and_reorder..simplify_and_reorderFsy(,(C(Ce\) %EXf%F*+''*:*:*J*J,^VK+' *(="8+ +rxprk)rindexing_exprsrrrrrrrrrget_write_exprsr`rrr8PREFER_STORE_LOOP_ORDERextendget_read_exprsrWrr6loop_ordering_after_fusionr7index_vars_no_squeezerA)r`extra_indexing_constraintsrecompute_sizes_body_funcrrr.rrextra_indexing_rangesextra_indexing_exprexpected_var_rangesrrrshould_merge_loops iter_ranges iter_reindexr reduce_rangesreduce_reindex iter_varsrrrs` @@rvr#ComputedBuffer.simplify_and_reordersV4  ' ' )  %Z  %Z % *)4k1J  )) 94..5578 % 15u==23q8 9:T 6 !3T:: ::1488 88H4GHHH HH"&// &? #%B ? /#.a>2I. # 1 1N0--/0 ww""4)O)OPP    3 3 5 6 ,$"/ t,- - VV5V5V1V (<     ( $ \1,@ {, ( ~q 0)rzetuple[tuple[list[sympy.Expr], list[sympy.Expr]], LoopBody, tuple[list[sympy.Expr], list[sympy.Expr]]]NN)r*Optional[tuple[dict[Any, Any], list[Any]]]rOptional[Callable[..., Any]]rz:tuple[tuple[list[sympy.Expr], list[sympy.Expr]], LoopBody]rqrlrIrn)rrrrrrrrsrtrr"rrrrrKrrrrrr(r%r.rr1rr rs@rvrrds K %%* %* ! ! 6%G%N%J JJDRVBFq2$Nq2$@q2 D q2f  !B!BF..,44rxrc^\rSrSrSrS U4SjjrS SjrSrS SjrSSjr SSjr SSS jjr S r U=r $)TemplateBufferizh Represents a Triton (in the future other type) of template operator that we can fuse an epilogue onto. c>[TU]SUS9 [RU5UlX0l[ RRU5Ul [ RRU5 gr) rrs InputsKernelunwrap_storagerMmake_kernel_renderr`rregister_bufferrregister_operation)r`r4rMr)rs rvrsTemplateBuffer.__init__sW d62"11&9 "4GG++D1  ""4(rxc URSS9$)NT normalize)r?rgs rvrTemplateBuffer.get_read_writess''$'77rxc ^^^UR5mUR5R5mUU4Sjn[R"X R 5SUS9nUR HfmTRR5mUU4SjnU=R[R"UTR 5SSS9R-slMh U$)Ncb>[U5S:Xde[R"TT"U5S5$)Nrfake)rr^r!)rrr$rs rvdummy1TemplateBuffer.extract_read_writes..dummys,v;!# ##99T75>6: :rxrr.c~>[U5S:Xde[R"TR5T"U55 gr)rr^rr)rrr$rOs rvr4r5s-6{a'''8rxT) rrrr7r?rrMr4r)r`r/r4depsr$rOrs @@@rvr?"TemplateBuffer.extract_read_writess}}//#002 ;// ==?B) ;;Cjj--/G 9 JJ,::s||~rTe J rxc6[RR$rq)rrrrgs rvr(!TemplateBuffer.get_reduction_sizesww{{rxcgrqrrgs rvr%!TemplateBuffer.get_reduction_typerrxcgrrrgs rvrTemplateBuffer.should_allocaterrxc*UR5S4S4$rYr)r`rrs rvr#TemplateBuffer.simplify_and_reorders$      rx)rMr)r)r4rrMSequence[IRNode]r)rrrrgrlrNrIr!)rr"rr#)rrrrr!rsrr?r(r%rrrr rs@rvr%r%ss ) )! )/ )  )82 RVBF  $N  $@   rxr%cx^\rSrSrSS U4SjjjrS S U4SjjjrS SjrS SjrSSjrSr U=r $)TritonTemplateBufferic B>[T U]XU5 X@lU/UlUb[R R R[R R R4n[RRRnXv;d SUSU35eURSR5nU=RUV s/sHn [[!US9X5PM sn - slU(aUO [#5UlSUlSUlgs sn f)a NOTE:[TritonTemplates with multiple outputs] We want the ability for TritonTemplates to output multiple tensors. Triton kernels have no notion of outputs and this is done by creating tensors that are then mutated by the kernel. Currently our STORE_OUTPUT codegen doesn't support creating multinode outputs for triton templates. We work around this by creating an extra input buffer during the lowering and we mark them as mutated inputs. Nz$Mutated inputs are only allowed for z but got rr)rrsmutated_inputsoutputsrr^ higher_orderflex_attentionflex_attention_backwardr`r current_noderrMrMutationOutputrr0allowed_prologue_inps subgraph_inps subgraph_outs) r`r4rMr)rErL allowed_setrJrrrs rvrsTritonTemplateBuffer.__init__s " );<,&*V  % &&55 &&>>K77//66L. 6{m9\N[ .[[^..0F LL))Cz8#D) L &; !  "SW?CsDc,>[TU]U5nUR(a URO/nUR(a URO/nUHzn[ U[ R 5(aUR[XQ55 M>[ U[5(a"URURU55 MuUcMze UH?n[ U[5(a"URURU55 M:UcM?e U$rq) rr"rNrMrrrrupdaterrn)r`rresrNrMrOrrs rvr")TritonTemplateBuffer.get_free_symbol_uses sg*=9.2.@.@**b .2.@.@**b  C#uzz** +C?@C(( 333MBC{"{ !!C#v&& 333MBC{"{ !  rxcUR$rq)rFrgs rvr TritonTemplateBuffer.get_outputs7rrxcUR$rq)rLrgs rvget_allowed_prologue_inps.TritonTemplateBuffer.get_allowed_prologue_inps:s)))rxc&SURS3nU$)NzTritonTemplateBuffer(layout=rr)r`rs rvrTritonTemplateBuffer.__str__=s,T[[M; rx)rLrErFrMrNr!)rEzOptional[Iterable[IRNode]]rLzOptional[OrderedSet[str]]rrr`rkrr?r[) rrrrrsr"rrXrrr rs@rvrCrCsl 6:;? )D 3 )D 9 )D )D)DX%*! !.*rxrCc^\rSrSrSrS U4SjjrSSjrSSjrSrSSjr SSjr SS jr SS jr SS jr S rU=r$) ChoiceCalleriEa Represents a possible choice used in autotune_process.py. During autotuning, self.benchmark() is first called to get benchmark result, and if this choice is selected, self.output_node() is called to get the output_node. Children classes: TritonTemplateCaller, CUDATemplateCaller. cR>[TU]5 XlX0lX lX@lgrq)rrsrr4r description)r`rrr4r_rs rvrsChoiceCaller.__init__Ns(   &'rxc^^UR5m[R(a[UU4Sj5$[R "TTSU05$)Nc>T"T6$rqr)algorsrvr(ChoiceCaller.benchmark..`s D$Krxr) to_callabler6/profile_bandwidth_with_do_bench_using_profilingrPrF benchmark)r`rrrcs `@rvrgChoiceCaller.benchmark]s?!  A A+,?@ @$$T4%>>rxc[erqr}rgs rv call_nameChoiceCaller.call_namecrrxc[erqr}rgs rvreChoiceCaller.to_callablefrrxc"UR5$)z Hash key for the underlying kernel. By default, we assume there are no runtime params, so kernel hash key defaults to choice caller's hash key. )hash_keyrgs rvkernel_hash_keyChoiceCaller.kernel_hash_keyis }}rxc[erqr}rgs rvroChoiceCaller.hash_keyprrxc[erqr}rgs rvrIChoiceCaller.output_nodesrrxc0$)zRInformation returned here is logged to the autotune log file when that is enabled.rrgs rv info_dictChoiceCaller.info_dictvs rxcg)Nunsupported_choicerrgs rvautoheuristic_idChoiceCaller.autoheuristic_idzs#rx)r_rr4r) rrrrr4rr_rrr)rrr[)rrm)rz[TU]UUSUS9 X0lSUlX l[ SU55Ulg)N)r4rMr)rLc3# UH]n[U[5=(dA [U[RRR 5=(a UR v M_ g7frq)rrr~r _inductorselect_algorithmExternKernelCallerhas_out_variant)rchoices rvr/MultiTemplateBuffer.__init__..sU% - v7 8 65??#C#C#V#VW+**  -sA%A')rrs_choice_timings_fn_choice_timingsoriginal_inputsr_output_plannable)r`r4rMchoice_timings_fnunfiltered_choicesrLrs rvrsMultiTemplateBuffer.__init__sU #"7  #4DH%!$% - % " rxcUR$)zN Are all possible choices TritonTemplates or Extern Kernels with out variants )rrgs rvoutput_plannable$MultiTemplateBuffer.output_plannables %%%rxc^URcUR5UlUR$rq)rrrgs rvchoice_timings"MultiTemplateBuffer.choice_timingss+    '#'#:#:#[TU]XU5 X@lXPlX`lgrq)rrsworkspace_sizetemplatesupports_epilogue_fusion)r`r4rMr)rrrrs rvrsCUDATemplateBuffer.__init__s& );<, (@%rxc8URb UR$S$r)rrgs rvr%CUDATemplateBuffer.get_workspace_sizes&*&9&9&Et""L1Lrxc~UR5H)n[R"UR5SS5 M+ grq)rr^r!r)r`rAs rvemulate_store_fn#CUDATemplateBuffer.emulate_store_fns,&&(F IIfoo't 4)rx)rrr)rrsrrcrrrrr>) rrrrrsrrrr rs@rvrrsE A  A  A#' A  AM55rxrc<^\rSrSrSU4SjjrSU4SjjrSrU=r$)CppTemplateBuffericL>[TU]XU5 X@lXPlSUlgrq)rrsrrrF)r`r4rMr)rrrs rvrsCppTemplateBuffer.__init__s$ );<  /3 rxc6>[UR[5(al[UR[5(deURSn[U[ 5(deURn[U[ 5(deU$[TU]!5$r) rrr4MultiOutputLayoutrFrr)rrr)r` first_outputr4rs rvrCppTemplateBuffer.get_layouts} dkk#4 5 5dllH55 55<<?LlF33 33!((Fff-- --M7%' 'rx)rrFrr>rG)rrrrrsrrr rs@rvrrs4 ( (rxrcl\rSrSr%S\S'S SjrS Sjr\SSj5r\ S5r SSjr SS jr S r g )r'irrMc^[[R"5n[RmURHqn[ U[ 5(aURU4SjU55 M5[ U[5(aMLURT"UR555 Ms [[R"U4SjUR555n[R"UU[5S9$)Nc3P># UHnT"UR55v M g7frqrz)rrurs rvr/InputsKernel.get_read_writes..sBEqWQZZ\22E#&c3P># UHnT"UR55v M g7frqrz)rrrs rvrrs#. /AGCLLN # #/Ar)rwrites index_exprs) r0r7r<rrMrrrrRrr@rr ReadWrites)r`rinputrrs @rvrInputsKernel.get_read_writess<++,.&&[[E%&& BEBBE#899 '%.."234!L,,-. /3/?/?/A.  &&"   rxc6UR5R$rqrrgs rvrsInputsKernel.get_readsrrxc[U[5(a URn[U[5(a URn[U[5(a*[U[ 5(d[ RU5n[U[5(aURU5$[U[5(aU$[U[[ 45(dU5eU$rq) rrrmr3r(r3r8r realize_inputunwrap_storage_for_inputTorchBindObjectr)rrus rvr%InputsKernel.unwrap_storage_for_inputs a # #A a $ $A a " ":a+I+I**1-A a # # //2 2 a ) )H!fo677::7rxc/nUHdn[U[5(a&UVs/sHn[RU5PM nnO[RU5nUR U5 Mf U$s snfrq)rrrr'rr)rM inputs_newrurs rvr(InputsKernel.unwrap_storage%sk A!T""GHIq!\::1=qI 99!<   a   JsA/cgrrrgs rvr+InputsKernel.is_extern0rrxcgrrrgs rvrInputsKernel.num_reads3r rxrNrgrh)rurnrrnrIri)rrrrrrrsr rrrr(r+rrrrxrvr'r'sD  ,,$rxr'c(\rSrSrSSjrSSjrSrg) NopKerneli7cgrrrgs rvr.NopKernel.is_no_op8rrxc[5$rqr/rgs rvrsNopKernel.get_reads;rrxrNrIrh)rrrrr.rsrrrxrvrr7s rxrcV\rSrSrSr\S5r\S Sj5r\S5rS Sjr Sr g) ConcatKerneli?zb There isn't actually a real kernel for concat, we just change the storage for the upstream data. c  USR5nUSR5n[USR55nS/nXR/nSUs=::a[ U5:de e[ S[ U55HnXR5n UR XR5 [ U 5[ U5:XdeXR5U:XdeXR5U:Xde[ [ U55HEn X:Xa XZX-XZ'M[RRRXZX5XZ'MG UR XR5 M [RU5n [R(a#[R!XUSR"5n [ [ U55H|nXn [%U 5(dMU R'5n [)U [*5(dM@[R-U R.U R05(dMq[3U5n O [5SU55n[RR6R8Sn[)U[5(deUSLa"[5SU55(a [3U5n [;S[+UUUU S9/S9n[=U5n/n[ [ U55GH nUR?X[@RCUX&UXxSS95nURDR U5 [)XRF[H5(aXRFRK5nOXRFnURM5(dM[OXR5RP5(dM[SU5(aMUR URU55 GM [ U5S:aR[RRWU[XRZ5(a[RR]U5 [RR_U5Ul0URcURD5Ul"[RReU5 U$) Nrr5c38# UHn[U5v M g7frq)rr8s rvr&ConcatKernel.create..ns-WPV1.CA.F.FPVrqFc3# UHvnSUR;=(a[ URSR[RS9=(d* URSR[RS9v Mx g7f)rrN)rFr rrrrargs rvrrrsr< $ SXX  --EB)rrrrrr4rM)rB)3rrrrrrrr`rrrrrr6rrrrrrrrr6rrrr"rBrJrrr(rCr5rrMr3r3rrrWrrVr7rr8FOREACHregister_operation_listr*rr(r+)rrMr(rrr offsets_start offsets_endr input_sizer output_striderur4any_input_is_storage_and_layout fx_node_args concat_kernelkernelop_names input_bufferinput_unwrappeds rvrConcatKernel.createEs%%'q ##%q **,- }o C'#h-'''''q#f+&A++-J   /z?c(m3 339&&(E1 119'')V3 333x=)8"*+ "=HK"#''"2"2"?"? Z]#HK *   x} -''99(C  ' '"//M s6{#A A$Q''K88fmmTT$B8$LM$+.-WPV-W*W'ww++003 ,---- *e 3< $ < 9 9 ;8DM$$    M*s6{#A++   Cq!1;>!L  ' ' 5&)..(33"().."<"<">"().. //1169//16677"<00 ? ? AB'$* x=1 !4!4V^=S=S!T!T GG + +H 5WW44]C "11-2F2FG  ""=1 rxNc[U[5(aURURU5$[UR[5(a[URR [ 5(aURR(dgUcg[UR55[UR55:Xdg[S[UR5UR5555$[URR [5=(a [UR[5(+$)NFTc3x# UH0up[RRRX5v M2 g7frqr.r/s rvr=ConcatKernel.can_realize_into_without_copy..s1EFB  88@@Er1)rrrmcan_realize_into_without_copyr3rr4r6rrrrrrExternKernelAlloc)rr@rAs rvr*ConcatKernel.can_realize_into_without_copys c9 % %44SXXsC C chh 3 4 4sxx <<xx00{s~~'(C0@,AA!#.."2CNN4DE  #((//>: : HH'D @  rxc [U[5(d&[U5(a[U5up4[X4S9n[U[5(dU5e[U[5(aUR UR U5$[U[5(aiUR5 [UR S5(deURX5(a&[U5UR l UR $[RUR5UR!5UR#5[%UR'5UR'55VVs/sH.upV[(R*R,R/XV5PM0 snnS9nUR Xr5$s snnf)Nr2r4r-)rrr8rr5rmrCr3r(rrrrr4rrrrrrrr`rrr)rr@rAr;r4rSrTpws rvrCConcatKernel.realize_intosV #//$S))"7"<%7B#//44/ c9 % %##CHHc2 2 c: & & KKM388X.. ..00::"1#"6xx   >>#--/__&   ??DA  --a3?  (( s.5F:cgrrrgs rvrConcatKernel.should_allocaterrxrrqrI) rrrrr!r rrrCrrrrxrvrr?sL ^^@  <))@rxrc^\rSrSr%SrS\S'\R"\S9r S\S'Sr S \S 'Sr S \S 'Sr S \S '\R"\ S9rS\S'SrS\S'SrS\S'SrS\S'\R"\S9rS\S'\R"\ S9rS\S'S>S?U4SjjjrS@SjrSASjrSrSrS?SjrS rSBSCS!jjrSDS"jrS#r\S$5r\ SES%j5r!\ S&5r"\ S'5r#\ S(5r$\ SFSGS)jj5r%\ SHS*j5r&\ SHS+j5r'\ S,5r(\ S-5r)\ S.5r*\ S/5r+S?S0jr,S1r-SBSIS2jjr.S3r/S4r0SHS5jr1SJS6jr2S?S7jr3S?S8jr4S9r5S:r6SHSKS;jjr7SJS<jr8\8r9S=r:U=r;$)Lrirztuple[Any, ...] constant_argsr zdict[str, Any]rNzOptional[ReinterpretView] output_viewrOpython_kernel_namecpp_kernel_namez Iterable[str]ordered_kwargs_for_cpp_kernelzFOptional[Union[torch._ops.OpOverload, torch._ops.HigherOrderOperator]] op_overloadzOptional[list[dict[str, Any]]]arg_propertiesz#Optional[dict[str, dict[str, Any]]]kwarg_propertiesz"dict[sympy.Symbol, pytree.KeyPath]unbacked_bindingszlist[MutationOutput]mutation_outputsc 2>[T U]UUUS9 X@lU(aUO0UlX`lXlUR U5 URU5 XlUR5 0Ul /Ul [RRUlgNr)rrsrrrr set_cpp_kernel_nameset_python_kernel_namer collect_arg_kwarg_propertiesr r r`rrJfx_node) r`rr4rMrrrr r r r rs rvrsExternKernel.__init__s   + &fB &&   1 ##$67-J* ))+!# "ww++ rxc U/URQ$rqr rgs rvrExternKernel.get_outputss-t,,--rxc[5$rqr/rgs rvr%ExternKernel.get_unbacked_symbol_defs!rrxc[UR[RR5(afURR R Vs/sH:nUR(aMURURURS.PM< snO.[[UR55Vs/sHn0PM snUl[UR[RR5(aSURR R Vs0sH'nURURURS._M) snO0Ul[UR[RR5(aUR (dRURR R Vs/sH!oR(dMURPM# snUlURR R Vs/sHoR(dMUPM snUlg/Ulgs snfs snfs snfs snfs snf)N)rrr)rr)rrr r_ops OpOverload_schema arguments kwarg_onlyr real_typerrrrMr allarg_propertiesr schema_kwargs)r`rurs rvr )ExternKernel.collect_arg_kwarg_properties$s$**EJJ,A,ABB))11;;    >55$($4$4$<$<$F$F6$Fq,,FAFF$F62 ++33=="=a="D "$D ? 7 6"s0I/(I: I#.I III+Ic[UR[5(a!UR5 UR 5 ggrq)rrr4rapply_constraintrrgs rvrExternKernel.decide_layoutIs0 dkk> 2 2  ! ! #     3rxcP[X5up#U(aURU5 ggrq)rS make_comment)r`wrapper origin_str_detailed_origin_strs rvcodegen_commentExternKernel.codegen_commentNs%+>t+M(    , rxc[erqr}r`r# s rvcodegenExternKernel.codegenSrrxcXl[RR(a3[ UR [ RR5(dgUR nURcURS:XaVURS:XaURRS5SOURRSS5nSUS3UlgURRUlgg)Natenr4.rrz at::_ops::z::call)r r`r cpp_wrapperrrr rr r namespace _overloadnamerrreplacer r)r`r ropnames rvr ExternKernel.set_cpp_kernel_nameVs.ww""*   ejj33+ +  !!    '6)++y8OO))#.q100c: *4F86'B$'-~~':':$ (rxcXlUbgURnUcg[U[RR 5(aSUR 3UlgURRSS5SUR 3Ulg)Nztorch.ops.higher_order.._ops..ops.r. ) r r rrrr HigherOrderOperatorrrr2 )r`r rs rvr #ExternKernel.set_python_kernel_namens"4  ) !! >   > > ? ?(??P&QD #$$,,Xw?@&//ARS  #rxc:UR5=n(a URO[RRn[RR (a4[RR RURU5$UR$rq) rrr`r device_typer/ rget_c_shim_func_namer r )r`drs rvget_kernel_nameExternKernel.get_kernel_name}sn!%!22A29L9Lww"" GG 5 5d6J6JF S (( rxc [RUR5UR5UR 5UR 5UR 5UR5S9nUR5 U$)N)rrrrrYrW) rrrrrrr|rxr)rurs rv copy_inputExternKernel.copy_inputsa   <<>++-]]_::<))+oo'    rxc ^^X#S.n[R"U5unm/m/n/nUHnTR[U[5=(a [U[ 5(+5 TS(aURU5 M[[U[ R5(a2[RRRRUSS9nURU5 M UU4Sjn UV s/sHoRU 5PM nn UHn [U 5(dM[U SS9 M! /n UGHn [U [ 5(dhU R#5[RR$;a<U R[RR$U R#55 M[U [ 5(dhU R#5[RR&;a<U R[RR&U R#55 M[U [(5(a"U RU R+55 GM5[U [,R.R0R 5(arU R2R4n U R2R6S:XaU ceU R[,R8R:U R=55 GMU R[?U SS95 GM U "X5upU"U 0UD6nSn[R@R=n(a[RBRDRGS 5n[I5n[RBRJ[,RLRNRP:XaUS n[S[RB5nU [UU[RBU5 SSS5 [WUUU5n[U[X[Z45(dU/OUnUHn[U[,R\5(dM$UR^(dM7S n[RRBRDRGS S5=n(aUS U3nU[Rl0M UUUU U4$s sn f!,(df  N=f)N)rrr)rc6>/n[U5n[U5nTH@nU(aUR[U55 M&UR[U55 MB [R"UT5nUR S/5UR S054$)Nrr)iterrnextpytreetree_unflattenr) new_tensor_argsnew_non_tensor_argsr it_tensorsit_non_tensors is_tensorr args_spec is_arg_tensors rvunflatten_args3ExternKernel.process_kernel..unflatten_argssFo.J!"56N* MM$z"23MM$~"67 + %%fi8A55$aeeHb&99 9rxTrr )rrr5zEsparsity not handled. Please file issue for sparse inference weights. stack_tracez Found from : )1rG tree_flattenrrrrnGeneratorStaterrr`rrrcreate_symintnoderrr5r3r constantstorchbind_constantsr get_valuerrirrrrr default_generators clone_staterrrJrFrrr_higher_order_opseffects with_effectsr%r*r&rrTensor is_sparsedisable_cudagraphs_reason)rrrr binded_args args_flat tensor_argsnon_tensor_argsrrP ru example_args device_indexnew_args new_kwargsexample_outputr r node_meta_valctxexample_out_lirmsgrR rN rO s @@rvprocess_kernelExternKernel.process_kernels $6 %22;? 9  %'C  3'O 30O,O R ""3'c5::..''**44FFsQUFVC&&s+ :6AA[((+[ AA$Q''%a5  Aa**qzz|qww?P?P/P##AGG$5$5ajjl$CDq(++JJLAGG$?$??##AGG$?$? $MNA//##AKKM2Au11@@AA xx~~ xx}}.<3KKK##JJ11,?KKM##$5aT$JK'* .lL8Z8JN -- -9 -NN//33E:M-C~~$$(?(?(G(G(T(TT -a 0 >>J 9>=!  ntUm<<    A!U\\**q{{{]"#''"6"6";";"?"? t"TT;T E!2;-@C471          OBjs4S'9S,, S:c [U[5(de[U[5(aU$UR5n[R R UR55nUceUR5nUbSUR;a[UR[5(aURSR[RS9(d/URSR[RS9(a)UR![#UR%555 OUR'5 [(R*"UR%5SS9upVUSnUR-5"U5n[R R.R1X5n[R R.R3X5n [R R.R5X5n [7Xy5U -n X:wa[8R;SU U U5 [<e[UR>[AURC5URE5UR%5U U S9S9$) z In order to pass this to an extern kernel we need a ReinterpretView not a View. This allows us to avoid some unneeded copies. rrrrkrz@convert_to_reinterpret_view failed: stride=%s offset=%s index=%srr2)#rrr3r8rr`rr-rr|rFr4rr rrrr r"rrr7rrrr stride_vars offset_varrXrrrr3r6rr) rru x_unwrap_viewrx_unwrap_view_fx_node index_argsrrrr:r7expecteds rvconvert_to_reinterpret_view(ExternKernel.convert_to_reinterpret_view s!X&&&& a ) )H  gg  !7!7!9: # 3 3 5 " -.333=//@@%**51??"'"5"5@)--e4BB"'"8"8C  7 7.}/E/E/GH   ' ' )!-!@!@ JJL"   ]  ,  55eH''""..uA!!,,U?Z1F:   IIR   & %,,.kkmZZ\   rxc(Uc [5$[U[R[RR R [45(a [US9$[U[5(aY[RR[R"URUR!5UR#5S95$[U[$5(aU$[U[&5(aUR)UR*5$[U[,5(a1[-UR)UR*5UR/5S9$[U[05(a@UR35 [5UR755(aUR9U5$[U[<5(aUR35 U$[U[>[45(aU$URAU5$![:a Naf=f)N)rrr2)!rrrrrrrrrsrrbr`radd_tensor_constantrr9rrrr.rmrr3r8rr3rrrrx rr( NonTensorObjrA rs rvrExternKernel.realize_inputOs 9') ) a%**ekk&9&9&A&A3G H H(a0 0 a " "77.. QWWAKKM!,,.Q  a ( (H a # #$$QVV, , a ) )"&&qvv.q||~  a " " IIK$Q]]_55::1== a $ $ IIKH a,(=> ? ?H~~a  +sH HHc[U5(a@[UR55S:XaU$UR5H nUS:XdM Us $ URU5$r)rrrrA )rrurs rvrequire_stride1ExternKernel.require_stride1psR  # #1<<>"a',,.Q;H)~~a  rxc  UcUceUR5S;a U(dU$[U5(Ga][UR5[5(aU(ai[ USS[ X5(aJ[[RRRUR5R55OUUS9 U$[ USSSUUS9 U$[UR5[[45(ayU(a$UR5RU5(d>U(aG[!X1R5RUR#55(aUb [%X5$U$[UR5[&5(a[UR5R)5[5(a [+S5e[UR5R)5[5(aU(a2UR5R)5RU5(dMU(aH[!UUR5R)5RUR#55(aU$[U[,5(akU(a$UR5RU5(d>U(a9[!X1R5RUR#55(aU$[U[.5(a[UR0[25(a[UR0[45(d[UR755(a}[UR75R0[85(dPUR;UR05UlU(aUR=XUS9$U(aUR?XUS9$SnUR#5nUb[RRn[C[EUR#555Vs/sHJnURGX8S5(dMURIUR#5US 5(dMHUPML nnUH.n [JRLRNRQXSS 5nM0 URSU5n[ USSUUUS9 U(a[ X5(deU$U(a<UbUce[JRLRNRUX5n[%X5$U$![@a GNgf=fs snf) N)rr5TF)rr%r&rr$zHthe MutationLayoutSHOULDREMOVE's real layout shouldn't be FlexibleLayoutr'rrr5)+rrrrrrr5r-rr`rr size_hintsrr6rr,r+rr?r+r1rrrmr3r3r8rrrx require_stride_orderrequire_exact_stridesrrrr#r;rrloweringslice_rA r) rrurrr expanded_dims orig_sizerrr(s rvrequire_stridesExternKernel.require_strideszsm M$=== ;;=F "=H ! # #!,,..99*#(- B!KK- ! 0 0 ; ;ALLN;;EBB!1%||~'<'>-!<<>#8#8!**, H q) $ $1668,,qvv77%ammo66q}}335FGG 88@33 4#44 5#.2 JJL  $ww''Hs1::<011A33M4DaH11!**,q/1E1 %OO,,33AAqA% NN1  !''   5a?? ?? (]-F FF((//=A21D DW'  s*36U*UU-6&U- U- U*)U*c"URXUS9$)N)rrr )rrurrs rvr "ExternKernel.require_exact_stridess"" -#  rxc"URXUS9$)N)rrr )rrurrs rvr !ExternKernel.require_stride_orders""1"OOrxc.URU[5$rq)r rrs rvrequire_channels_last"ExternKernel.require_channels_last!s''+<==rxc.URU[5$rq)r rrs rvrequire_channels_last_3d%ExternKernel.require_channels_last_3d%s''+=>>rxcSnU"U5(aU$URU[RUR555$)NcSnU"U5[RR;=(a- [RRU"U5R$)NcPUR5$![[4a gf=frq)rAttributeErrorrrts rv safe_get_namePExternKernel.require_contiguous..is_mkldnn_tensor..safe_get_name,s+ ::<'&(;<  s %%)r`rrV is_mkldnn)rur s rvis_mkldnn_tensor9ExternKernel.require_contiguous..is_mkldnn_tensor+sH a AGG$5$55BGG%%mA&67AA rxr rrr)rrur s rvrExternKernel.require_contiguous)sB  A  H,,>44QZZ\B rxchURU[RUR555$rqr rs rvrequire_contiguous_strides'ExternKernel.require_contiguous_strides?s-(( ~00>  rxcgrqrrgs rvr ExternKernel.apply_constraintGrrxc[U[[45(de[U[5(a [U5nUR(dS5e[ U5n[ UR5nX4:aq[ R SURXC- 5 [X45H?nURUSnURXb;aX&OURUS5 MA U$)Nz/ExternKernel.arg_properties should not be emptyzv%s has %d unprovided positional arguments. Will check if they are in the keyword arguments or will use default values.rr) rrrrr rrrr rr)r`rrn_args n_pos_argsrarg_names rvfill_non_provided_args#ExternKernel.fill_non_provided_argsJs$u .... dE " ":D""U$UU"T,,-    II^  #   6...q1&9 )$,,Q/@/ rxcr[RR(Ga[/nSnU(aeUR(aT[ UR 5[ U5:XdS5eURVs0sHoDR S5U_M nn[UR 5HupVUb.UR X5nU(aUR S5OSnOb[ UR5U-n UR(a7U [ UR5:aURU R S5OSnUR[RRRXh55 M U$[[RRRUR 5$s snf)NzDnames passed to codegen_const_args does not match self.constant_argsrr) r`rr/ r rrrrrMrrval_to_arg_strr) r`rrname_to_arg_propertiesrrruproptype_rs rvcodegen_const_argsExternKernel.codegen_const_argslsi 77   F &* ",,4--.#e*<Z<594G4G*4GSGGFOS(4G'*"$"4"45)5155eh?D04DHHV,$Edkk*Q.C ..3T=P=P9Q3Q++C044V<!  agg22AA!KL6Mqww++::D>E 1))(3a,,MM!$ 11hBXBX6X..228<@@H! MM!''"6"6"E"Ea"OP?( !KK--//DA#Qqww++::1=>?/  s:FcURbSURRn[USS5nURSS5nUR SS5SnUSU3nU$SnU$) Nrunknown_namespacer6 r7 r. r5r unknown_op)r rrr2 rsplit)r`r op_namespaceop_names rv get_op_nameExternKernel.get_op_namesv << #\\((F"6<9LML'//'BL'..sA6q9L%ax0G#Grxc [R(a[RR(d[ UR 55S:Xag[RRRUR 55n[RRRUR55nUR5nURSUR5SUSUSU<S3 5 ggg)Nrzassert_size_stride(rr) r6 size_assertsr`rr/ r[rrcodegen_shape_tuplerr rr)r`r# rrr s rvcodegen_size_asserts!ExternKernel.codegen_size_assertss   qww':':T]]_-277'';;DMMOLDWW))==doo>OPF&&(G   %dmmo%6bb7+UVW (; rxc j[R(a[RR(d~UR 5nU[RR ;nUR5nU(a!URSUS[SU<S35 gURSUSUS35 ggg)Nzassert_alignment(rrz # buffer z (op: z) is assumed to be not aligned) r6alignment_assertsr`rr/ rr0r rrT)r`r# ralignedr s rvcodegen_alignment_asserts&ExternKernel.codegen_alignment_assertss  # #AGG,?,?==?D!''";";;G&&(G!!'vR/@7+QO!!vVG94RS-@ #rxcNUR5nUR5nU//U4$)z4 get output sizes and strides, for template_codegen )rr)r`_size_strides rvget_group_strideExternKernel.get_group_strides* //#r{G##rxc [RRnUR5nUR 5nUVs/sHoAR U5PM nn[ [U55Vs/sHn[SU35PM nn[[ [U55URSS9n[U5VV s0sHupX_M n nn [ [U 55Vs/sHoZUPM n nU Vs/sHoVUPM nnUR5n U "U5n [RRRXbU /5upn[S5unn[[!Xo"UVs/sH nU"U5PM sn555n[#[$R&"U 5U5n U [)U54$s snfs snfs sn nfs snfs snfs snf)z3 Manually get canonicalization of the output index r= T)rrc)r`rrrrrrrrYrrrrrr@rrr\rrr)r`rrr:rurr index_orderrrrrr$r new_sizesrrradd_var replacements rv canonicalizeExternKernel.canonicalizes 77## //#29:'Q%%a(':;@U;LM;La(1QC1;L MU3w<0g6I6ISWX +4[+AB+Axs#(+AB$)#f+$67$6q$67-23UmU 3##% #%&WW%5%5%E%E w& " F !% 73z7 3R 1GAJ 3R+STU 5<<. <eI&&&+;MC734Ss#F=1GG'G ;G/GcU(a[O[n[[R"5nUR H nX2"U5-nM UR R5H nX2"U5-nM U$rq)maybe_free_unbacked_symbolsmaybe_free_symbolsr0rrrrr)r`rmaybe_get_symbolsrrs rvr"!ExternKernel.get_free_symbol_usessq ,9 '>P  u|| $ &%%C "3' 'A&;;%%'C "3' 'A(rxc ,[USS5nSU<3/nU[R"U5Vs/sH'nURS[XR53PM) sn- nUR SUR <35 UR U5$s snf)Nr zpython_kernel_name=rr)rrpfieldsrrrYr)r` kernel_namerrqs rvrExternKernel.__str__,sd$8$? !+ 1  $++D1 1zzl!GD**56 71    |D$4$4#789u%%  s.B) r r rr r rr r r rr r r rNNNNrNr>rrrqr rOrr)r rOrr)rzituple[Any, list[Any], list[Any], Callable[[Any, Any], Any], Optional[dict[sympy.Symbol, pytree.KeyPath]]])NNF)rzOptional[Sequence[int]]rrLr`)rrVr[rk) rrrA r ro rx rr r r r r r rr r r r r r r r r r r r r"rr rr rs@rvrrs%'M?'(..tDFND-1K*1(, ,%)O])4?3D3D4!=  6:N29<@9@->t-T*T&(, ,,<.#$J! - ";0    u  u u nB B H!!@!!*.6: Z'Z4 ZZx  PP>>??*    DP@4 J:   $'@%* !  !  &HrxrcT^\rSrSrSSjrSSU4SjjjrSSjrSrU=r$) ExternKernelOuti;c&URU5 grq)generate_extern_kernel_outr) s rvr* ExternKernelOut.codegen=s**40rxc >[T U]SUURU5UU=(d 0SUUUU 5 [RR U5Ul[RRU5 grq)rrsr(r`rr*rr+) r`r4rMrrrr r r r rs rvrsExternKernelOut.__init__@si       '  Lb    )  GG++D1  ""4(rxcgrrrgs rvrExternKernelOut.should_allocate[rrxrlr>r rI) rrrrr* rsrrr rs@rvr r ;s=1&() ))6rxr c,^\rSrSrSU4SjjrSrU=r$) RandomSeedsi_c >[R"[R5n[TU][ U[RU/S9/UR URU//SS[RRS9 g)Nrzaten.randint.low_outzat::_ops::randint_low_out::call)r4rMrr r r ) rrrrrsr6rCrr- randintlow_out)r`countrlimitsrs rvrsRandomSeeds.__init__`slU[[) kkW !::vzzE7;5> ,,  rxr)r rsrrTrrrrrrrsrr rs@rvr r _s   rxr cX^\rSrSrSSjrSSU4SjjjrS SjrSrSrU=r $) risc&URU5 grq)generate_extern_kernel_allocr) s rvr* ExternKernelAlloc.codegents,,T2rxc >[T U]SUURU5UU=(d 0SUUUU5 /Ul[R R U5Ul[R RU5 grq) rrsr(rFr`rr*rr+) r`r4rMrrr r r r rs rvrsExternKernelAlloc.__init__wsp       '  Lb    )  ') GG++D1  ""4(rxcgrrrgs rvr!ExternKernelAlloc.should_allocaterrxc[erqr}rgs rvr "ExternKernelAlloc.apply_constraintrrx)rrFr>)rNNNrNrI) rrrrr* rsrr rr rs@rvrrss?3&() ))<""rxrcN^\rSrSrSrSU4SjjrS SjrS SjrS SjrSr U=r $) rKizH An output buffer that represents the mutation of a pre-existing buffer c>[TU]SUS9 UR5n[RR U5 U/UlX0l[RRU5Ul gr) rrsrr`rr.mutation_names mutating_noder*r)r`r4 mutated_noder mutated_node_namers rvrsMutationOutput.__init__s^ d62(113 ##$5601(5GG++D1 rxcUR$rq)r rgs rvrMutationOutput.get_defining_ops!!!rxcUR$rq)r rgs rvr4!MutationOutput.get_mutation_namesrrxcgrrrgs rvrMutationOutput.should_allocaterrx)r r r)r rwrrrrorI) rrrrr!rsrr4rrr rs@rvrKrKs!2"#rxrKc^\rSrSr%Sr0rS\S'\S Sj5r\S Sj5r S U4Sjjr S Sjr SS jr S r U=r$) TMADescriptoriaL An IR node representing a generic host-side TMA descriptor in the Triton API Mostly useful for user-defined Triton kernels relying on host-side TMA; but can, in principle, be used for Inductor's Triton templates, too. See TMADescriptorExperimental and TMADescriptorStable for the two implementations (the old API and the new API) zdict[Any, TMADescriptor]_CACHEc[U5S:XdeUSS:Xa[U/USQ76$USS:Xde[U/USQ76$)Nrr experimentalr5r)rTMADescriptorExperimentalTMADescriptorStable)rr9tma_metas rv _create_implTMADescriptor._create_impls\8}!!! A;. (,VBhqkB BA;(* **&v< < [TU]S[[UUR 5S95U[ U5S5 Xl[RRU5Ul [RRU5 g)Nr2) rrsrr8rrr9r`rr*rr+)r`r9rMrrs rvrsTMADescriptor.__init__ss   !,,.   -   GG++D1  ""4(rxc&URU5 grq)generate_tma_descriptorr) s rvr* TMADescriptor.codegen''-rxcUR$rq)r9rgs rv get_tensorTMADescriptor.get_tensorrrx)rr9)r9rnr ztuple[str, tuple[Any, ...]]rr )r9rnr>r_)rrrrr!r rr r rrsr* r& rr rs@rvr r s(*F $)=='B= =='B )*.rxr cH^\rSrSrSrSSU4SjjjrSrU=r$)r iz the new host-side TMA Descriptor API: (the ones obtained via create_{1d,2d}_tma_descriptor calls). See also TMADescriptorStable for the new API. c^>[U5S;de[U5[U5:XdeUcUR5RnX lX0lX@l[UR5UlU/n/URQURQUR Pn[TU]!UUUS9 g)N)r5rr9rMr) rrr|r block_dims element_sizerrrs)r`r9rr+ r, rMrrs rvrs"TMADescriptorExperimental.__init__s4yF"""4yC O+++  !++-66L $( N  YY __     '  rx)r+ rr, rrq) r9rnrlist[Union[int, torch.SymInt]]r+ r. r, r]rrrrrrr!rsrr rs@rvr r sG'+   - 3  $     rxr c0^\rSrSrSrSU4SjjrSrU=r$)r iz the new host-side TMA descriptor API (the ones obtained via TensorDescriptor.from_tensor). See also TMADescriptorExperimental for the old API. c2>X l[TU] UU/US9 g)Nr* ) block_shaperrs)r`r9r2 rs rvrsTMADescriptorStable.__init__!s&& 8%  rx)r2 )r9rnr2 r. r/ rs@rvr r s  rxr cJ^\rSrSrSU4SjjrSSjrSrU=r$)SubgraphBufferi+cn>[T U]SX5 X0lX@l[R R U5Ul[R RU5 [R RURXE5Ul [UR5nUHTnXpRRUR'URRRUR5 MV UVs/sHoRPM snUlSSKJs Jn [R("UR5 U R+SSSS9 URR,"UR6 SSS5 SSS5 gs snf!,(df  N=f!,(df  g=f)NrFATEN) max_autotunemax_autotune_gemmmax_autotune_gemm_backends)rrsrHexample_inputsr`rr*rr+ make_subgraphsubgraphrPrMrgraph_input_namesr sym_inputstorch._inductor.configrr6set_graph_handlerrrun) r`r4rrHr; subgraph_namer? sym_inpsym_varinductor_configrs rvrsSubgraphBuffer.__init__,s= v3,GG++D1  ""4(--dgg~U (5 !G7>MM & &w|| 4 MM + + 2 27<< @"8BBzG<.CodegenGraphiPc2XlURUlgrqrr)r`rs rvrs5SubgraphBuffer.codegen..CodegenGraph.__init__Qs" !JJ rxrL N)rrd)rrrrrsrrrxrv CodegenGraphrJ Ps 'rxrN )rMr'codegen_subgraph_with_flattened_outputsr= r? r)r`r# rN r outer_inputss rvr* SubgraphBuffer.codegenOsg ' ' 8<{{C{!++-{ C77  ' -doo - - YYK DsA1)r; rHrr= r? ) r4rrrrHtorch.fx.GraphModuler; list[Any]rC rr>)rrrrrsr* rr rs@rvr5 r5 +sC!8!8"!8 ! !8 " !8  !8F   rxr5 cz^\rSrSrSrS SjrS S U4SjjjrS SjrS U4SjjrSSjr SSjr S r U=r $)UserDefinedTritonKerneli]cz^SSKJn SSKJn UR UR 5m/n/n/n[ TU5(a[TS5(a&URU4SjTR55 O.[TS5(deURTR5 [TS5(a<TRH+nURTRRU5 M- O.[TS5(deURTR5 TR nTRmTX4U4$) Nr) Autotuner)kernel_side_table restore_idxc3V># UHnTRRUv M g7frq)r arg_names)rrrs rvrBUserDefinedTritonKernel.get_kernel_and_metadata..ks$*4FqFII''*4Fs&) restore_value reset_idx reset_to_zero)triton.runtime.autotunerrW *torch._higher_order_ops.triton_kernel_wraprX get_kernel kernel_idxrrrrrY r] r^ rrr[ r_ configs)r`rW rX rd restore_value_argsreset_to_zero_argsrrs @rvget_kernel_and_metadata/UserDefinedTritonKernel.get_kernel_and_metadata^s6P"--doo>(*(* fi ( (v}--"))*4:4F4F*v7777"))&*>*>?v{++))A&--fii.A.A!.DE*v7777"))&*>*>?nnGYYFw4FFFrxc SSKJn UR5unnnnURUUURUUUR 5unnn UR V s0sHoURU 5_M n n [URV s/sHoRU PM sn 5n /n/n/n/n[R"U R5[[R"S5U 55GH|unnUR!U5 UR!U5 [#U[$5(a@UR!UR'55 UR!UR)55 M~[#U[*[,[.[0R245(a-UR!U5 UR![5U55 MUU ;a)UR!S5 UR![*5 GM UcYU"5(a)UR!S5 UR![*5 GMBUR75 UR75 GMe[9S[5U5SU35e UR;U5 UR=UUUUUUSUR?5UR@RBS9 gs sn fs sn f) Nr)triton_version_uses_attrs_dictrrzUnsupported arg type: rT) arg_typesraw_argsraw_keys triton_metar roriginal_fxnode_name)"torch._inductor.utilsrj rg !define_user_defined_triton_kernelrgridr r r0 constexprsr[ rrrrrepeatrrrrnrrrsrrrrrrrr& generate_kernel_callrr r)r`r# rj rrd re rf new_namern extra_launch_argsr named_argsrconstexpr_namesrrk raw_keys_filteredraw_args_filteredrrs rvr* UserDefinedTritonKernel.codegen}syH  ( ( *       5 5   KK   II     261S1S 1SAt$$Q' '1S  %6CTCT%UCTa&6&6q&9CT%UV! ')')"    I$4$4R$8:K L ID#  $ $T *  $ $S )#v&& C1134  1C#udEJJ!?@@ C   c+( B  %233KKO$$S)%))+%))+),B49+RPSu*UVVC F W%$$  &&#??$!%!2!2 % [ &Vs J>KcP>[TU]U5[URU5-$rq)rr"rrr r}s rvr",UserDefinedTritonKernel.get_free_symbol_usess- w+M:=M II}>   rxc[5$rqr/rgs rvr0UserDefinedTritonKernel.get_unbacked_symbol_defsrrxc >/n0n/nUR5Hup[U [5(aX[R UR U 55n X;a[ RXU5n URU 5 XU'MrURU 5 XU'M [U5S:wdeUSR5Ul [TU]5S[URS9U[U5U5 XlX lUR%5up n U R&Vs/sH oU;dM UPM snUlSSKJn [U 5S:aU SR.O0nU"U 0UEUEU5Vs/sHnUUPM snUlUR0Vs/sH!n[3[URS9UU5PM# snUl[6R8R;U5 gs snfs snfs snf)Nrr)identify_mutated_tensors)rrrrmr'rrr rrrrrrrsrrrc rr rg r[ r ra r r mutable_argsrKr r`rr+)r`rc rr tma_descriptor_metadata kernel_argsrMrrr rrrrd rrr autotuned_kwargsrrrs rvrs UserDefinedTritonKernel.__init__s %%'DA!Y'' 99$:L:LQ:OP/%,,Q0JKA a q $$Q'q (6{aQi**,    dkk *  -   % $ < < >A"++. +Ck/AC+. * X03G q0@71:,,b0;;;*:;=T    ((! ( :T[[93 E(!  ""4(%.  ! s' G-4G-8G2(G7c,[UR5$rq)rr rgs rvr#UserDefinedTritonKernel.get_outputssD))**rxcUR$rqrrgs rvr"UserDefinedTritonKernel.get_devicerrx)rrr rc r r r r>r`rkrrrQ) rrrrrg r* r"rrsrrrr rs@rvrU rU ]sQG>N b%* ! !  3) 3)j+rxrU cX^\rSrSrSrS SjrS SjrS SjrS SjrS U4Sjjr Sr U=r $) InplaceBernoulliFallbacki= This needs to be a custom class to handle mutation properly c SUR5un[RR(a\UR UR 5SUSSR [[UR55SUR35 gUR UR 5SUSSR [[UR55SUR35 g)Nc3@# UHoR5v M g7frqrrrs rvr3InplaceBernoulliFallback.codegen..s;{!##%%{rrz, NULL)r) rMr`rr/ rr> rrreprrending)r`r# rus rvr* InplaceBernoulliFallback.codegens;t{{; 77     '')*!A3b3tTEWEW;X1Y0ZZabibpbpaqr    '')*!A3b3tTEWEW;X1Y0ZZ[\c\j\j[kl rxcgrrrgs rvr(InplaceBernoulliFallback.should_allocate(rrxc>URSR5/$rrMrrgs rvr4+InplaceBernoulliFallback.get_mutation_names+ A'')**rxc[5$rqr/rgs rvr1InplaceBernoulliFallback.get_unbacked_symbol_defs.rrxcR>[TU]S[UR5S9UR U/5UUS9 [ R RUR55 [ R RU5Ul [ R RU5 g)Nrr ) rrsrrr(r`rr.rr*rr+)r`r rurrs rvrs!InplaceBernoulliFallback.__init__1s~   alln -    $ #   ##AJJL1GG++D1  ""4(rxrlr>rIror rrrrr!r* rr4rrsrr rs@rvr r s& + ) )rxr ct^\rSrSrSrS SjrS SjrS SjrS SjrS U4Sjjr \ SSSjj5r S r U=r $)InplaceCopyFallbacki?r cNUR5up#nURX2U5 grq)r codegen_device_copy)r`r# rAr@ non_blockings rvr* InplaceCopyFallback.codegenDs%#'#4#4#6 <##Cl;rxcgrrrgs rvr#InplaceCopyFallback.should_allocateHrrxc>URSR5/$rr rgs rvr4&InplaceCopyFallback.get_mutation_namesKr rxc[5$rqr/rgs rvr,InplaceCopyFallback.get_unbacked_symbol_defsNrrxc >[TU]SUUUSSS9 [RR USR 55 [RR U5Ul[RRU5 g)Nz aten.copy_aoti_torch_copy_)r r r) rrsr`rr.rr*rr+)r`r4rMrrs rvrsInplaceCopyFallback.__init__Qsr     +.   ##F1I$6$6$89GG++D1  ""4(rxcX4Vs/sHo@RU5PM nnU4n[[UR5S9UU5nU$s snfr)rr rr)rrAr@r rrMrrs rvrInplaceCopyFallback.createcsS14 ; 1##A& ;% $ cnn. /     rIrorr`)r r)rrrrr!r* rr4rrsr rrr rs@rvr r ?s?<+)  )$rxr cJ\rSrSrSrS SjrS SjrS SjrS SjrS Sjr Sr g )MutatingFirstArgExternKernelior c/SUR5Q[[UR5QnUR UR 5SSR U5SUR35 g)Nc3@# UHoR5v M g7frqr r s rvr7MutatingFirstArgExternKernel.codegen..vs 9[!!##[r rrr)rMrr rrr> rr )r`r# argrefss rvr* $MutatingFirstArgExternKernel.codegentsl 9T[[ 9 t)) *  ##%&a '(:';1W^^URSR5/$rr rgs rvr4/MutatingFirstArgExternKernel.get_mutation_namesr rxc[5$rqr/rgs rvr5MutatingFirstArgExternKernel.get_unbacked_symbol_defsrrxcgrrrgs rvhas_side_effects-MutatingFirstArgExternKernel.has_side_effectsrrxrNr>rIror) rrrrr!r* rr4rr rrrxrvr r os  +rxr c,^\rSrSrSU4SjjrSrU=r$)ResizeStorageBytesic(>[U[5(dS5e[TU] S[ UR 5S9UR U/5U4S9 [RRUR55 [RRU5Ul [RRU5 SUlSUl[RR R#UR$R55 g)NzTODO: dynamic shapesr)rz"inductor_ops.resize_storage_bytes_z&torch::inductor::resize_storage_bytes_)rrrsrrsrrr(r`rr.rr*rr+r r never_reuse_buffersr@r3)r`variablerrs rvrsResizeStorageBytes.__init__s(C((@*@@(   h113 4    +#+  ##H$5$5$78GG++D1  ""4("FG ##'' (>(>(@Arx)r rr r>r rs@rvr r s B Brxr c6^\rSrSrSU4SjjrSSjrSrU=r$)SetSourceTensorKernelic>UR5 [TU] UR5X/S[R R RRS9 [RRRURR55 [RRRUR55 [RRRUR55 UR5n[![#US9X5[![#US9X 5/Ulg)Nz!torch.ops.aten.set_.source_Tensor)r r r)rrrsrrr^r- set_ source_Tensorr`rr r@r3rrrKrr )r` self_tensorstorage_tensorrrs rvrsSetSourceTensorKernel.__init__s$$&   % % '  )B ++99  ##'' (8(8(A(A(CD ##''(?(?(AB ##'' 8**, :V4k H :V4n K! rxcvURSR5URSR5/$rr rgs rvr:2SetSourceTensorKernel.get_inputs_that_alias_outputs/ A'')4;;q>+B+B+DEErxr r>ro)rrrrrsr:rr rs@rvr r s "FFrxr cp^\rSrSrSrS SjrS SjrSSjrSSjrSSS .SU4S jjjr S r U=r $)ScatterFallbackiz This needs to be a custom class to handle mutation properly. This class handles both aten.scatter_ and aten.scatter_reduce_. It also handle the case `src` being a scalar properly. c URSn[RR(aSSS.nX#;aX2nUR(aSUR 5upEnO$SUR 5upEUR SnURUX@R SXV/URURURUUR55 g) NrrErD)r@multiplyc3@# UHoR5v M g7frqr r s rvr*ScatterFallback.codegen..sJk2244kr c3@# UHoR5v M g7frqr r s rvrr sEA--//r r5r) rr`rr/ src_is_tensorrMrgenerate_scatter_fallbackr r r )r`r# rget_operator_enumrurr@s rvr* ScatterFallback.codegensX& 77  (-6 B **2   JdkkJOQsEEJQ$$Q'C)) ""1%u 2   # #        ! rxcgrrrgs rvrScatterFallback.should_allocaterrxc>URSR5/$rr rgs rvr4"ScatterFallback.get_mutation_namesr rxc[5$rqr/rgs rvr(ScatterFallback.get_unbacked_symbol_defsrrxNTr include_selfc d>[U[5UlUR(a&X$U4Vs/sHoRU5PM n nU4n O$X$4Vs/sHoRU5PM n nX54n [T U]S[ UR5S9URU 5U XgS.[U5SS/US9 [RRUR55 [RRU5Ul[RR!U5 gs snfs snf)Nrr rr )r r r )rrrmr rrrsrrr(rr`rr.rr*rr+) r`r rur(rr@rr rtensorsrrs rvrsScatterFallback.__init__s(Y7   78oFo))!,oGF FM78jAj))!,jGA JM   alln -    (  <";/+3^*D#   ##AJJL1GG++D1  ""4(%GBs D(D-)rr r>rIror)r(rsrrOr rrrr rs@rvr r sV  .+!%!!) !)!)!) !)!)rxr cX^\rSrSrSrS SjrS SjrS SjrS SjrS U4Sjjr Sr U=r $) IndexPutFallbackizI This needs to be a custom class to handle mutation and indices properly cSUR5tp#n/n[U5n[UR5HdupxURUbUR [ U55 M1UR [ RRR5 Mf UR"UR5X%U/UR5Q76 g)Nc3@# UHoR5v M g7frqr r s rvr+IndexPutFallback.codegen..s&Rk':':'<' r ) r`r# rur valid_indicesriter_valid_indicesrrs rvr* IndexPutFallback.codegens&Rdkk&R#]!-0dll+DA||A*t$678qww33<<= , ++  "A 9=9P9P9R rxcgrrrgs rvr IndexPutFallback.should_allocaterrxc>URSR5/$rr rgs rvr4#IndexPutFallback.get_mutation_namesr rxc[5$rqr/rgs rvr)IndexPutFallback.get_unbacked_symbol_defsrrxc >X0lUVs/sH ofcMUPM nnX$/UQVs/sHo RU5PM nnSn [T U] S[ WR 5S9UR U5U4SU US9 [RRURSR55 [RRU5Ul [RRU5 gs snfs snf)Naoti_torch_index_put_outrzaten.index_put_)r r r r)rrrrsrrr(r`rr.rMrr*rr+) r`r rurr accumulaterr r r rs rvrsIndexPutFallback.__init__s $+=GqG =342M}2MN2MQ%%a(2MN4   alln -    ( M0+#   ##DKKN$;$;$=>GG++D1  ""4(>NsC9C9C>)rrr>rIrorr rs@rvr r s&  +))rxr c.\rSrSr\S5rSSjrSrg) DeviceCopyi+cUR5(dU[SUR555(a0[RR (dUR U5$[RRU5 [RRUR55 [S5 U4n[[UUR5UR5S9UR!U5/U5$)Nc3Z# UH!o[RR;v M# g7frq)r`rrV r@s rvr$DeviceCopy.create..0sG4Fq***4Fs)+zDeviceCopy in input programr)r+rrtr6 aot_inductoruse_runtime_constant_foldingr1r`radd_device_inforrOr rrrr)rrurr rs rvrDeviceCopy.create,s GA4D4D4FGGG''DD''/ / '  /78%  kkmZZ\    q ! "   rxcUR5n[U5S:XdeUR(a2URUSURR 5US5 gURUSUR 5US5 g)Nrrr5)r rrr r)r`r# rs rvr* DeviceCopy.codegenDs{  "4yA~~     ' 'Q));;=tAw   ' 'Q1G1G1I4PQ7 SrxrNr>)rrrrr rr* rrrxrvr r +s  .Trxr cX^\rSrSrSrS SjrS SjrS U4SjjrS SjrS Sjr Sr U=r $) riOz3 The result of a call to aten._local_scalar_dense. c[5$rqr/rgs rvrsDynamicScalar.get_readsTrrxcgrrrgs rvrDynamicScalar.should_allocateWrrxc>UR5 [TU] S[[R "S5S9UR U/55 XlX lgNrr) rrrsrrrr(symkeypath)r`r r r3rs rvrsDynamicScalar.__init__ZsG   *ELL$78$:M:Mtf:U  rxc.[UR/5$rq)r0r rgs rvr&DynamicScalar.get_unbacked_symbol_defsbs488*%%rxc&URU5 grq)codegen_dynamic_scalarr) s rvr* DynamicScalar.codegenes&&t,rx)r r rhrIr>r) rrrrr!rsrrsrr* rr rs@rvrrOs&&--rxrcf^\rSrSrSrS SjrS SjrS U4SjjrS SjrS SSjjr S Sjr S r U=r $)riiz- The result of a call to aten._assert_scalar c[5$rqr/rgs rvrsAssertScalar.get_readsnrrxcgrrrgs rvrAssertScalar.should_allocateqrrxcv>[TU]S[[R"S5S9/5 XlX lgr )rrsrrrscalarrn )r`r rn rs rvrsAssertScalar.__init__ts3   ell51 2    rxcgrrrgs rvr AssertScalar.has_side_effectsrrxc.[URU5$rq)rr r!s rvr"!AssertScalar.get_free_symbol_usess ];;rxc [R(dg[[UR SS955n[ R R(a^SUS3n[ R RRURSS9nURSUSURSUS 35 g[ R RRURSS9nURS US 35 URS [UR5S35 URUR5S 35 g)NFrLzstd::to_string(r)rzif (!(z()) { throw std::runtime_error("Expected z but received " + z); }zif not (z):z raise RuntimeError(z = None)r6scalar_assertsrF rE r"r`rr/ rcodegen_cpp_sizevarr rrn codegen_python_sizevarr r)r`r# symbol symbol_strsizevars rvr* AssertScalar.codegens$$$  d44454IJK 77  *6(!4Jgg**>> e?G    !J488*Tfgqfrrwx gg**AA eBG    4 5    7TXX7GqI J    19 :rx)rn r rhrIr>r`)rr) rrrrr!rsrrsr r"r* rr rs@rvrris+ <;;rxrc*\rSrSr%S\S'S\S'Srg)ExternKernelNodeirrzexport_schema.NoderrNrrrxrvr- r- s I rxr- c^\rSrSrSrSSS.SU4SjjjjrSU4SjjrSSjrSSjrS r \ S 5r S r S r SS jrSrSSjr\ SSj5r\S5rU4SjrSrU=r$)FallbackKerneliz A class that represents a fallback kernel for handling operators that are not directly support by inductor. It currently supports functional ops, view ops, inplace aten ops, and mutating ops that are auto-functionalizable. Nr c>^[T T]U[U5[U5US9 STlUTl[ U[ RR[ RR45(dSUS[U5S35eUTl UTl Uc0OUTl [RR!TR"5 /Tl/Tl[ TR[ RR5(agSTRR)5;agTRR*n[ R,R.R1TR5(a-TR&R3USR555 gUR6(a[9U5(d[;SU35eTRTR<TR>5upS U4S jjn [ R,R.RAXU5H upU "X5 M g) Nr Fz#Fails to create FallbackKernel for rz not supported_c10d_functionalrz'NYI: Can't generate FallbackKernel for c">^[TR[R5(a[U[[ 45(de[ R"TR5(a[U[ [45(aeUcgTRcgSUU4Sjjn[ R"TR5(aUbUH nU"U5 M gg[ R"TR5(deU"U5 g)Nc>TRRUR55 TRR(a<TR R[ [UR5S9UT55 ggr) alias_namesrr alias_infois_writer rKrr)rinfor`s rv add_aliasPFallbackKernel.__init__..handle_aliasing_and_mutation..add_alias sZ  '' 5??++))00&z'H!TR,rxr>) rrrrListTyperr library_utilsis_tensor_like_typer6 is_tensorlist_like_type)r8 rr9 optional_tensor_argr`s` rvhandle_aliasing_and_mutation=FallbackKernel.__init__..handle_aliasing_and_mutations$))U^^44!#e}555500;;&cE4=9999{&  44TYY???/2+!"5603#%88CCCC#rxr>)!rrsruse_runtime_dispatchr rrrr r r8 rr rP rr`r warn_fallbackr r5 r rr _libraryr mutates_and_returns_first_argrr is_mutablerrrMr zip_schema)r`r4rrd nontensor_argsrP rr schemarr@ r8 rrs` rvrsFallbackKernel.__init__s   +  . !  %*!!2  %% ..    X 14<. W  X ","Nb  d556'))+ d&& (F(F G G  !1!1!6!6!8 8 !!)) >>   = =d>N>N O O    & &{1~'>'>'@ A    %;F%C%C%9&B **4;;8J8JK  :--88vNID ( 3Orxc@>[TU]5nUR[RR R La]URHMn[U[5(dMUR[R"UR555nMO U$rq)rrr r_prims rng_primsgraphsafe_run_with_rng_staterrrrT with_readr7rr)r`rrrs rvrFallbackKernel.get_read_writesswg-/   u||55RR R))c>22"-"7"7$,,S\\^<#K* rxc nURUR5UR[USS55$Nr )(codegen_unbacked_symbol_defs_for_outputsrrFrr) s rvcodegen_unbacked_symbol_defs+FallbackKernel.codegen_unbacked_symbol_defs's0?? MMOT\\749Ld+S  rxc[USS5=n(aC[[RRR U5nUceUR 5$[5$rR rr+r`rrrrr0r`r resolveds rvr'FallbackKernel.get_unbacked_symbol_defs,Z '.A4 H H  H0  **,=H' ''==? "< rxcF[R"SS55nURVs/sHo!"UR55PM nnUR X0R 5upE[ RR(a[UR[RR5(aURXE5n[URR R"U5VVs/sH8upb[ RR$R'X&R(5PM: nnnO9UVs/sH,n[ RR$R'U5PM. nnUR*R-U5 U$s snfs snnfs snf)Nc*\rSrSr%S\S'SSjrSrg))FallbackKernel.codegen_args..Shimi7r refcUR$rq)r_ rgs rvr 2FallbackKernel.codegen_args..Shim.__repr__;s xxrxrNr[)rrrrrr rrrxrvShimr^ 7s H rxrb )rp dataclassrMrrP rr`rr/ rrr rr r r rr r rr r rrR)r`rb rurd rrparams rvr FallbackKernel.codegen_args6sH       =AKKHKqtA//12K H**;8J8JK  77  :d.>.> @U@U#V#V..t.Zs#$=K**433^rr5) rrrrrr_ rrrr0rrWr)rd rj rnon_torch_bind_tensor_argsrdevices device_setrs rvrh FallbackKernel.find_deviceMs$ J 1:a+IQ J # &3>S;C..BR's~~';GS1:  nell 3 3!(( ( ntUm 4 4#$=K$J-7AJ&&vJGA7|q qz!!&++&&!M"1: + K TBs!D)D)D.D. D3 D3c[UR[RR5(ag[ UR5R 5$r)rrr rr r8 r$rF rgs rvr FallbackKernel.has_side_effectsgs< d&& (F(F G Gt//0;;==rxcUR$rq)r5 rgs rvr:+FallbackKernel.get_inputs_that_alias_outputlr~rxcP[UR5S::deUR$r)rr rgs rvr4!FallbackKernel.get_mutation_namesos'4&&'1,,,"""rxc [RSUR5UR5 [ U[ 5(deUR URUR5upURX5nURVs/sHnUR"U40UD6PM nnURn[RR(d/UQUQ$[SS5nUR!XQU5nSn[ U["R$R&R(5(a#UR+USUS5R,n OUR.R,n [1U 5S:XaBUR2(a UR2O UR4n U SR6n U"X5/n O:[9XR25V Vs/sHupU"U R6U5PM n n n[;UR5[<R>"URRA5UU 0S9S9n[RRBREU5 /UQUQ$s snfs snn f)a ProxyExecutor Design Note We export the ExternFallbackNodes (for custom ops) into a serialized file and run it with a host side proxy executor to address the ABI problem This is currently only implemented for fbcode. Eventually, we will also make this work for OSS. Detailed design doc can be found at https://docs.google.com/document/d/1wC4DOZFaYym2t1Esz0X5yxlLI3RDnSiyRbUus3bkJ64/edit?usp=sharing z4Extern kernel node added for node %s with target %s.Nc d[U[R[R45(aUn[U[[ 45(a[ U5S:XdeUSn[U[R5(a=[RR[R"UR5S9S9$Ube[RRSS9$[U[R5(a{[UR5[R5(aN[RRUVs/sH%n[R"UR5S9PM' snS9$[U[R5(a[UR5[R5(aUc8[RR[RRSS9S9$[RR[RR[R"UR5S9S9S9$[U[R 5(a[RRUS 9$[#S [%U535es snf) Nr5rrl) as_tensorT)as_none) as_tensors)as_optional_tensor)as_intzUnsupported return type )rrr TensorTypeNoneTyperrr export_schemaArgumentrTensorArgumentrr; getElementType OptionalTypeOptionalTensorArgumentIntType RuntimeErrorr) return_typerArs rvhandle_single_outputFFallbackKernel.export_extern_kernel_node..handle_single_outputsO+(8(8%..'IJJftUm44v;!+++ )Ck5+;+;<<(1188"/">">CLLN"S9;&;(11888FFK88Z**,e.>.>>>%--44$* #)C&44#,,.I#) 5 K););<<**,e.>.>BB>(1188+8+O+O+V+V$(,W,9 )1188+8+O+O+V+V&3&B&B%+__%6',W,9K77$--44F4CC"%=d;>O=P#QRR5 s ,J-rr5)rrMrFmetadata)rr)#rrrr rrr/ rP rMrr r r r`raot_moderserialize_inputsrr\ torchbind CallTorchBindrI returnsr rrFr r rr- r} rbrextern_kernel_nodesr)r`rrrordered_kwargsr serializernamed_argumentsr r rFr output_arguments return_schemarArs rvexport_extern_kernel_node(FallbackKernel.export_extern_kernel_nodess. B MMO    $////**4;;8J8JK **4899 9  ! !# 0 09  !!ww+T+N+ +*46 $55fFK- S^ fe55??MM N NmmDGT!W5==Gnn,,G w<1 '+lldll8M8MG!!*..K 4[ JK  .1,,-G -G)M%]%<%[U[R5(aT"UR55$[U[R5$rq)rrrr r NumberType)r is_numbers rvr )FallbackKernel.codegen..is_numbers=a!3!344$Q%5%5%788!!U%5%566rxc3J># UHnTR"U40TD6v M g7frq)r )rr rr`s rvr)FallbackKernel.codegen..s(?))!6v6?s #c3z># UH0up[U[5=(a T"UR5v M2 g7frq)rrcomplexr )rrrSr s rvrr s2,DDA1g&A9Q[[+AADs8;cH>/TR5QTR5Q$rq)r r rgsrvr(FallbackKernel.codegen.."s"F$++-F0C0C0EFrx)rz torch.JitTyperr))r r0 rrrr r r`rr/ torchgen.aoti.fallback_opsr rrrrB r6r custom_ops_to_c_shimsrP rMrrrr rBrr r r& r ,generate_fallback_kernel_with_runtime_lookuprr rFr generate_fallback_kernelr4rr r rT ) r`r# rr r args_iter exported_argsr rs ` @@rvr* FallbackKernel.codegens(!!   v %fejj&;&;<< <<ww""Lv;&;;KKa15D-    -fejj&;&;<< << WW f11GGG  % GG  65::#8#899-- 7 ..t{{D>+C+CD,)D % W%  $ $ ::.Js!K{,s"3"3{rqrrr0 z"Not sure where to find device infoc J>^^[T[[45(a/[T5"UUU4Sj[ [ T5555$[T[ 5(a<TR5VVs0sHup#UT"UT[T5U4/-5_M snn$[T[R5(a}[TRT5TT5n[R(dT(d[T5(d3[R R"R%UR&5 U$[T[(5(aT$[T[R*5(aTR,R.$TbS[T5S35egs snnf)Nc3Z># UH nT"TUT[T5U4/-5v M" g7frq)r)rrgenerate_outputrrAs rvrAFallbackKernel.create..generate_output..ks7$/$F1Iw4<:K9L/LMM/s(+zFallbackKernel output type z is not supported)rrrrrrrrrrr_ MultiOutputr r6 assume_unaligned_fallback_outputr]r`rr0r@rrsSymIntrr) rArrrrrr has_unaligned_inputpackeds `` rvr .FallbackKernel.create..generate_outputis[&4-00F|$"3v;/$FD))%+LLN$2g$v,9L8M.MNN$2FELL11!((0 ;;*,V44GG--11#((; FC(( FELL11{{'''~1$v,?PQ~3s6%F)r- *_fused_moving_avg_obs_fq_helper_functionalr`rrrro rBrh rrrr\ r r rrrrrrrF)rrrrfake_incorrect_kernelscontextrj rd re rP r rrFr r r s` @@@rvrFallbackKernel.create9sj"&"Q"Q!S!'!EAGG  ;= ""6;D;F;  ! "!K{!KKn=* E++55CC  \\%(F  !&)"3 F ?? ?6!0"3 F  D"."5 geT2 3 3$FN&YFNcWs E E#c >[TU]5$rq)rr rs rvr FallbackKernel.apply_constraintsw'))rx)r5 rr r r rP rB rqr>rgrro)rAr)rrrrr!rsrrT rr rrrh r r:r4r r* r r rr rr rs@rvr/ r/ sj4j4 j4j4X   .2>  #m(^L3\  VVp**rxr/ cN^\rSrSrSrS SjrS SjrSS.S U4SjjjrSrU=r $) ComplexViewiz9View a complex number as two dtyped numbers or vice versacgrrrgs rvrComplexView.should_allocaterrxc>URSR5/$rr rgs rvr:(ComplexView.get_inputs_that_alias_outputs A'')**rxNr0 c *>[TU]UUUUUUS9 g)Nr0 )rrs)r`r4rrd rH rP r rs rvrsComplexView.__init__s)      /  rxrrIror>) rrrrr!rr:rsrr rs@rvr r s)C+    rxr c*\rSrSr%S\S'SSjrSrg)rirTrcUR$rqrrgs rvrMultiOutputLayout.get_devicerrxrNrQ)rrrrrrrrrxrvrrs rxrcp^\rSrSrSSjrS S U4SjjjrS S SjjrS SjrS SjrSr U=r $)r icURU5 UR(d#URU5 URU5 ggrq)codegen_multi_output!skip_size_stride_alignment_checksr r r) s rvr* MultiOutput.codegens:$$T*55  % %g .  * *7 36rxc>[TU]SX/S5 [RR U5Ul[RR U5 X0lX@lgrY) rrsr`rr*rr+rr )r`r4rrr rs rvrsMultiOutput.__init__sK vw3GG++D1  ""4( 1R.rxc>URSRU5$r)rMr"r!s rvr" MultiOutput.get_free_symbol_usess{{1~22=AArxc|[UR5S:Xa#[URS[5(agg)Nr5rTF)rrMrrrrgs rvrMultiOutput.should_allocates/ t{{ q t{{1~'8 9 9rxcURVs/sHIn[U[5(dM[UR 55S:dM9UR 5PMK sn$s snfr)rMrrr/ rr:r)r`rOs rvr:(MultiOutput.get_inputs_that_alias_outputs\{{ "#~. C4467!; CLLN"   sA"A" A")rrr r>r`)r4rrzlist[tuple[Any, ...]]rrrkrIro) rrrrr* rsr"rr:rr rs@rvr r sd4+0 S S' S  S S%*B!B !B   rxr c\rSrSr%SrS\S'S.SjrS/SjrS0SjrS1Sjr S2S jr S3S jr S4S5S jjr S6S jr S7SjrS8SjrS.SjrS7SjrS9S:SjjrS;SjrSSjrS?SjrS@SjrSASjrSBSjrSCSjrS.SjrS.SjrSDSjrSESjrS3S jr SES!jr!SBS"jr"S9SFS#jjr#SGS$jr$SHS%jr%S4SIS&jjr&\'SJS'j5r(SKS(jr)SJS)jr*SLS*jr+\'S+5r,S3S,jr-\-r.S-r/g )Mr8iz; TensorBox / StorageBox allow in-place mutation of Tensors rnr3c6URR5$rqr\rgs rvr!MutableBox.has_exceeded_max_readsr^rxc6URR5$rqrLrgs rvrMutableBox.get_devicerJrxc6URR5$rqr2rgs rvrMutableBox.make_loaderyy$$&&rxc6URR5$rq)r3rrgs rvrMutableBox.make_indexerrfrxc6URR5$rq)r3rrgs rvrMutableBox.get_striderJrxc6URR5$rqrQrgs rvrMutableBox.get_namerSrxNc8URRU5$rq)r3rrs rvrMutableBox.has_large_inner_fnsyy++I66rxc8URRU5$rqrXrs rvrMutableBox.mark_reuserZrxc6URR5$rqrdrgs rvrMutableBox.realize_hintrfrxc6URR5$rq)r3rrgs rvrMutableBox.unwrap_view r rxc6URR5$rq)r3rrgs rvrMutableBox.is_input_bufferyy((**rxc6URR5$rq)r3rrgs rvrMutableBox.freeze_layoutsyy&&((rxc8URRX5$rq)r3rrs rvr*MutableBox.freeze_layout_with_stride_ordersyy88NNrxc8URRU5$rq)r3rrs rvr(MutableBox.freeze_layout_with_fill_ordersyy66u==rxc8URRU5$rq)r3r r s rvr (MutableBox.freeze_layout_with_same_ordersyy66v>>rxc8URRX5$rq)r3rrs rvr+MutableBox.freeze_layout_with_exact_stridessyy99-WWrxc6URR5$rq)r3rrgs rvrMutableBox.get_read_writes$r rxc6URR5$rqrrgs rvrsMutableBox.get_reads'rnrxc6URR5$rqrrgs rvrMutableBox.num_reads*rnrxc6URR5$rqrhrgs rvrMutableBox.get_storage_numel-rjrxc6URR5$rqrrgs rvr%MutableBox.get_reduction_type0rrxc6URR5$rqrrgs rvr(MutableBox.get_reduction_size3rrxc6URR5$rqrlrgs rvr+MutableBox.is_extern6rnrxc6URR5$rq)r3r.rgs rvr.MutableBox.is_no_op9rSrxc8URRU5$rqrrs rvr1MutableBox.constant_to_device<syy++F33rxc6URR5$rq)r3r4rgs rvr4MutableBox.get_mutation_names?rrxc6URR5$rq)r3r7rgs rvr7MutableBox.get_operation_nameBrrxc6URR5$rq)r3r:rgs rvr:'MutableBox.get_inputs_that_alias_outputEsyy5577rxc6URR5$rqr`rgs rvrMutableBox.realizeHrbrxc8URRU5$rqr5r!s rvr"MutableBox.get_free_symbol_usesKsyy--m<r_r`rarcrdrfrgrhrirjrNrlrnrorkr?rBrPrHrGrJ)0rrrrr!rrrrrrrrrrrrrrrr rrrsrrr%r(r+r.r1r4r7r:rr"rtrrrur4rrrrrr rrrxrvr8r8sO L2&'(&$7+('+)7<OO/3O O >?DIX+X<@X X +%%-..%$4..8#%*=!= != *+3++&+$ "Hrxr8c$\rSrSr\S5rSrg)rmicX[U[5(aU$[[U55$rq)rrrrmr(r3s rvrTensorBox.creates% d1 2 2KD)**rxrN)rrrrrrrrrrxrvrmrms++rxcX\rSrSrS SjrSrS SjrSSjrS SjrSr SSjr S r S r g )r(ic[UR[[45(a5URR 5[ R R;$gr)rrr3rr8rr`rrrgs rvrStorageBox.is_input_buffers= dii+!? @ @99%%'177+?+?? ?rxc[UR[5=(a5 URR5[R R ;$rq)rrr3r.rr`rrV rgs rvrpStorageBox.is_module_buffers9 tyy> 3 : ""$(9(99 rxc [UR[[[[ [ 45(aURR5$[UR[[[[45(d[UR55eURR5nURR5n[S[URR!5URR#5URR%5S9URS9Ul[&R(R+UR5URl[&R(R/UR5 UR0URlXRlX RlURR,$)Nrr)rrr3rr'rr8r%rrrtrrrr|rxrrrrr`rr*rr+rUrYrW)r`rYrWs rvrStorageBox.realizesS  II   99%%' '$))iD$%GHH $ IIK  Hii//1 II++- "!yy++-ii))+YY'')   00;  ""499- LL  + ' yy~~rxc[UR[[45(a:URR 5R S:aUR 5 ggg)z< Called on buffers we expect to be forced to realize later. r5N)rrr3rrtrnontrivial_read_countrrgs rvrStorageBox.realize_hintsI tyy9i"8 9 9 **,BBQF LLNG :rxc[UR[5=(a8 UR5[R :=(d UR 5$rq)rrr3rrr6realize_acc_reads_thresholdrrgs rvr!StorageBox.has_exceeded_max_readss@$))Y/ NN vAA A )&&( rxcr^US:a[UR[[45(a[ UR5(a9URR 5mSS/n[ U4SjU55(agUR5[R:=(d UR5$g)zR A heuristic to decide if we should realize a tensor that is used multiple times. r5expsigmoidc3@># UHoTR;v M g7frq)used_ops)rruopcounts rvr5StorageBox.should_realize_on_reuse..s@iG,,,isTF) rrr3rrtrrrBrr6realize_reads_thresholdr)r`r heavy_opsrC s @rvshould_realize_on_reuse"StorageBox.should_realize_on_reuses 19DII 9/EFFdii  ))446"I. @i@@@ 6#A#AA-**, rxcRURU5(aUR5 ggrq)rG rrs rvrStorageBox.mark_reuses!  ' ' . . LLN /rxc6URR5$rqrrgs rvrStorageBox.num_readsrnrxr/ NrIrNr>r^) rrrrrrprrrrG rrrrrxrvr(r(s+  B $%rxr(c8\rSrSr%S\S'S\S'SrS\S'S rg) SubgraphirrrR graph_moduleNzOptional[GraphLowering]rr)rrrrrrrrrxrvrN rN s I&&%)E ")rxrN cUVs/sH*n[U[5(aUR5OUPM, nn[[ SU555[U5:$s snf)Nc38# UHn[U5v M g7frq)r )rr*s rvr'_has_aliased_buffers..s;7"V**7rq)rrr8rrr0)buffersr*s rv_has_aliased_buffersrT sdF!+6? C CO  z;7;; [TU]SUUS9 Xl[RR U5Ul[RRU5 gr )rrsr= r`rr*rr+)r`r= rX r4rs rvrsInvokeSubgraph.__init__sO   ! GG++D1  ""4(rxcj^SSKJn [RRnSnUR R S5=n(a USSSnO-URSSnUVs/sHoR SPM nnUVs/sHoRU5PM nn/n [U5HGup[U [5(aU RU 5 M-U RU"XU 55 MI U nURcz[RRURUURS9Ul[R "UR5 URR""U6 SSS5 URR$n Sn UH*n [U [5(aMU R'5n O U ce[)UU[+U S9S 9mS U4S jjn[U 5VVs/sHunnU"UU5PM n nnU TlU $s snfs snf!,(df  N=fs snnf) Nr5)constrain_to_fake_tensoreager_input_valsrrrrHr; rC r)r= rX r4c >[U[[45(aU$[[ UR 5UR 5UR5UR5UR5RS9T[U4/SS9$)NrT)r ) rrrrr r6rrrrrr7r)rAindinvoke_subgraphs rv create_output,InvokeSubgraph.create..create_output@s:N"OPP "%002$..0#__.%002%00299 $C[M6:  rx)rArnra rs)r r] r`rrJrFrrrrrrrrr< rO rrA rB graph_outputsrrV rrF)rr= rX r] rJ fake_operandsr^ fx_operandsru new_operandsroperandrFrrc rrArb s @rvrInvokeSubgraph.creates 6ww++  +00445GH H  H,Q/3M'++AB/K4?@KqVVE]KM@ 3;;(Q%%a((; %h/LC'#899##G,##,WC6HI 0  >> !WW22((,&mm3HN $$X^^4""M25....Gg'<== ++- !!!($F3   ">Gw=OP=O 6=+=OP")AA <(54JQs"HHH2H/ H,c&URU5 grq)codegen_invoke_subgraphr) s rvr* InvokeSubgraph.codegenUr$ rx)rr= )r= rN rX zlist[TensorBox]r4rrr)r= rN r>)rrrrr!r= rrX rFrsr rr* rr rs@rvrV rV sq$(H '*.H'.+/G (/ )  ),; )EV )  )NN`..rxrV c^\rSrSr%SrS\S'SrS\S'SrS\S'SrS\S 'Sr S \S 'SU4S jjr \ SS j5r SSjr SSjrSrU=r$) ConditionaliYNr^ predicate7Optional[list[Union[TensorBox, ShapeAsConstantBuffer]]]rX rW true_subgraphfalse_subgraphrY rFc>XlX lX0lX@l[ U/U-5upx[ T U]SUUUS9 UbX`l[RRU5Ul [RRU5 gN)rr4rMr) rp rX rr rs _split_by_sym_typerrsr r`rr*rr+) r`rp rX rr rs r4r sym_argsrd rs rvrsConditional.__init__as# *, 2I;3I J "   (%6 "GG++D1  ""4(rxcURU5nUVs/sHoPRU5PM nn[RRRSnUVs/sHoUR SPM nnX#4HnURbM[RR URUURS9Ul[R"UR5 URR"U6 SSS5 M URRn URRn SU 4SU 44H&up[U 5(dM[SU SU 35e [U 5[U 5:XdX45e[[!X55Hun upUR#5UR#5:Xd XU45eUR%5UR%5:Xd XU45eUR'5R(UR'5R(:XaMXU45e [+SU/U-55n[-[RR.R0[RRR R3S S55nUcS 5e[5UUUU[7US 9US 9nSS jn[[!U [RRR S55V VVVs/sHun unn[9[;UR#5UR%5UR=5Vs/sH nU"U5PM snUR?5Vs/sH nU"U5PM snUR'5R(S9U[@U 4/5PM n nnn nU Ul!U $s snfs snf!,(df  GMq=fs snfs snfs snnnn f)Nrrr_ true_fnfalse_fnzVOutput aliasing is currently not supported in compiled torch.cond. The outputs of the z% subgraph of torch.cond are aliased: c3p# UH,n[U[5(aMUR5v M. g7frq)rrrr)ros rvr%Conditional.create..s, +a!67 ALLNN+s66r zcannot determine devicer)rp rX rr rs r4r c\[U[5(aU$URR$rq)rrrsrr)rs rv _maybe_expr'Conditional.create.._maybe_exprs"!S!!66;; rxr)rzUnion[int, torch.SymInt]rzUnion[int, sympy.expr])"rr`rrJrrFr< rO rrA rB re rT rrrrrrrr7rF r+rrrro rr r6rrrrF)rrp rz r{ rX rurg rf r= true_outputs false_outputsrrFrt_of_orr conditionalr rA merged_outputrs rvrConditional.create}s%%i0 2:;(Q%%a((;gg**//3 0;< 1 < +H~~%!"!6!6,,#0"*--"7" ((8NN&& 698,}}22  44 (,7*m9TUMD#L11$**./TU\T]_V< C $66U8UU6&s<'GHMAz>>#s~~'77 F!# F7==?cmmo5 D} D5>>#**cnn.>.E.EE TPS} TEI  [8+   6 GG   & & GG % % ) )*=t D !<#<<!!!#$F3/   &/8L!''"6"6";";E"BC/ /**FM !,,. **,4A4F4F4HI4Hb+b/4HI6C6J6J6LM6LKO6LM!,,.55   /  && Y<<98bJM  s<OO$OAO8 O..O8 O3/O8  O+ . O8 c URU5 URUR5UR[ US055 grR )codegen_conditionalrS rrFrr) s rvr* Conditional.codegens9##D)88 MMOT\\749Lb+Q rxc[USS5=n(aC[[RRR U5nUceUR 5$[5$rR rW rX s rvr$Conditional.get_unbacked_symbol_defsr[ rx)rs rrX rp rr r )rp rnrX -list[Union[TensorBox, ShapeAsConstantBuffer]]rr rN rs rN r4rr z,Optional[dict[sympy.Symbol, pytree.KeyPath]]rr)rp rmrz rN r{ rN rX r r>r)rrrrrp rrX rr rs rFrsr rr* rrr rs@rvro ro Ys"&I&HLHEL(,M%,)-N&-+/G (/))@) ) ! ) " )H) )8TTT T @ TTl   rxro c/n/nUHFn[U[5(aURUR5 M5URU5 MH X!4$rq)rrrrr)r non_sym_argsrw rs rvrv rv sOLH c0 1 1 OOCHH %    $   !!rxc^\rSrSr%SrS\S'SrS\S'SrS\S'SrS\S'Sr S \S 'SU4S jjr \ SS j5r SS jr SrU=r$) WhileLoopiNrq carried_inputsadditional_inputsrW cond_subgraph body_subgraphrY rFc>XlX lX0lX@l[ X-5upg[ TU]SUUUS9 [RRU5Ul [RRU5 gru ) r r r r rv rrsr`rr*rr+) r`r r r r r4rw rd rs rvrsWhileLoop.__init__su-!2** 2>3U V "  GG++D1  ""4(rxc SSKJn SSjn[RRR Sn[RRR SnXx-n U V s/sHoR SPM n n UV s/sHoR SPM n n UV s/sHoR SPM n n UV s/sHoRU 5PM nn U"X<5nUV s/sHoRU 5PM nn U"XM5nX4-nX4HnURbM[RRURU URS9Ul[R"UR5 URR"U 6 XLaZ[URR5[U 5:XdeU"URRU 5URlSSS5 M URRnURRn[U5(a[!SU35e[U5S :XdU5eUSn[#U[$5(dMUR'5[(R*:XdU5e[UR-55S:XdU5e[U5S:dS 5eUSR/5nUce[U5[U5:Xd UU45e[1[3UU55HununnSS jnU"UR-5UR-55 U"UR55UR555 UR/5UR/5:Xd UUUU45eUR'5UR'5:Xd UUU45eUR75R8UR75R8:XaMUUU45e [;UUUU[=US 9S 9nURb=[#URR>[(R@RB5(deU"URR>U 5Sn[EU5nUVs/sHnUUPM nn5VVs0sHunnUU;dMUU_M nnnURG5VVs/sHuunn[I[KUR/5UR'5UR-5UR55UR75R8S9U[LU4/5PMw nnnUUl'UV s/sHn [QU RRU U5PM sn Ul*[WU5n![WU5n"[Y[U55Vs/sHnUU;a [[U"5O [[U!5PM! n#nU#5Hkun nU R]5[RR^;dM4[RR`RcUR]55 Mm U#$s sn fs sn fs sn fs sn fs sn f!,(df  GM=fs snfs snnfs snnfs sn fs snf)Nr)check_input_alias_and_mutationc ,[U5[U5:Xde/n[X5Hhup4[U[R5(a3UR [ RX4R5SS95 MWUR U5 Mj U$)NFr') rrrrrr_ rrr r) tensor_boxes fake_tensorsretrfks rv_require_exact_strides0WhileLoop.create.._require_exact_stridess|$L(99 99Cl9b%,,//JJ$:: 5; JJrN:Jrxrrr_ zOutput aliasing is currently not supported in compiled torch.while_loop. The outputs of the body_fn subgraph of torch.while_loop are aliased: r5z9torch.while_loop is assumed to have at least one operand.c~[X5H.up#[RRR X#5 M0 grq)rr`rrr) lhs_exprs rhs_exprslhsrhss rv_guard_list_equals,WhileLoop.create.._guard_list_equalsqs-!$I 9HCGG$$11#;!:rxr)r r r r r4rr)r 'list[TensorBox | ShapeAsConstantBuffer]r z,list[Union[int, torch.SymInt, torch.Tensor]]rr )r Sequence[Union[int, Any]]r r rr)2torch._higher_order_ops.utilsr r`rrJrrFrr< rO rrA rB rre rT rrrrrrrrrrrrrr7r rmodulefx GraphModuler0rr r6rrFrKr4r rE rrF rrr r@)$rcond_fnbody_fnr r r r fx_carried_inputsfx_additional_inputs fx_all_inputsrufake_all_inputsfake_carried_inputsfake_additional_inputs all_inputsr= cond_outputs body_outputsrrrrBbor while_loop mutated_idxsmutated_idx_setrrEr real_outputsrArO outputs_itermutated_inputs_iter all_outputss$ rvrWhileLoop.createsn Q A F 5 "GG0055b9 ww3388<)@ 2?@-Q66%=-@6GH6Gvve}6GH9M!N9MA&&-9M!N8FG1++A.G/T;LM;La..q1;LM2  $7  *H~~%!"!6!6,,#0"*--"7" ((8NN&&8 *"8>>#?#?@C/E  8N$NN88/8498+2}}22 }}22  - - XXdWeg  < A%3|3% O!233;;=EJJ. 1 1.qzz|$) ,1 ,):" G "A))+!!!>"c,&77W.,9WW7$S%FGKAxB <4 <4 < < r{{}bkkm < r}}  @==?bmmo5 J2r67J J5<<>R\\^3 @aR[ @3==?))R]]_-C-CC PaR[ PCH")/!!$F3  }}(Z MM %(("6"6. .   6 MM /  %\25DE_c*S/_E&l3 3S/) CH3  " ,113   4 V !,,. **,*!,,.!,,.55    4   * &' % 3::sJ 7%' # L) ">2S./ /*-)?D$ %T,EW W/  NK8HC||~!5!55 ++// ?9SAH!NGM98ZF  '  sP(Z Z$ZZ*Z?A8Z"8Z5 Z:(Z:A<[![$&[ " Z2 c&URU5 grq)codegen_while_loopr) s rvr* WhileLoop.codegens""4(rx)r r r r r) r r r r r rN r rN r4rrr)r rN r rN r r r r r>)rrrrr rr r r rFrsr rr* rr rs@rvr r sNRNKRQUNU(,M%,(,M%,+/G (/)E)I) )  ) " ) )0fffF f I ffP))rxr cV^\rSrSrSSS.S U4SjjjjrS U4SjjrS SjrSrU=r$) riNr0 c t>[T U]UUUUUSUS9 SSKJn UV s/sH&n [ U [ 5(a U R OU PM( n n U"U/UQU Q7U5n U ceXl[RRRU S5Ul U[RRU 'gs sn f)N)rr r)get_effect_key) rrstorch._higher_order_ops.effectsr rrrr effect_typer`r effectful_opsrprev_effect_buffer) r`r4rrd rH rP rr r rSuncovered_argsr rs rvrsEffectfulKernel.__init__s      /   CGR FQz!_55AGG1 [TU]5nURbGURR [ R "URR555 U$rq)rrr rr@r7rr)r`rrs rvrEffectfulKernel.get_read_writessUg-/  " " .    ! !$$T%<%<%E%E%GH rxcgrrrgs rvr EffectfulKernel.has_side_effectsrrx)r r rqr>rgrI) rrrrrsrr rr rs@rvrrs622 22@rxrc\rSrSrSrg)r| irNrrrxrvr| r| srxr| c`\rSrSr%S\S'S\S'S SjrSSSjjrSS jrSS jrSS jr S r g)rirr+Union[FakeScriptObject, torch.ScriptObject]rcUR$rqrlrgs rvrTorchBindObject.get_namerrxNcUR$rqrlrs rvr!TorchBindObject.codegen_referencerrxcUR$rqrrgs rvrX TorchBindObject.get_valuerrxc[UR[R5(a UR$URR$rq)rrrr ScriptObjectreal_objrgs rv get_real_objTorchBindObject.get_real_obj s3 djj%"4"4 5 5:: ::&& &rxcUR5n[UR55n[R"U5SnUVs/sHEn[ U[ R5(dM$UR5UR5-PMG nn[R"[RUS5$s snfr)r r__obj_flatten__rG rS rrrr_ r, numelrroperatorr@)r`real_script_obj flat_dict flat_elemsru flat_sizess rv get_buf_bytesTorchBindObject.get_buf_bytes s++-88:; ((3A6   !U\\* )ANN qwwy (   j!<<  s #B;.%B;rr[rqrP)rr )rztorch.ScriptObjectri) rrrrrrrrX r r rrrxrvrrs& I 66' =rxrcB\rSrSr%S\S'S\S'S SjrS S SjjrS rg) rT i rrrTrcUR$rqrlrgs rvrGeneratorState.get_name rrxNcUR$rqrlrs rvr GeneratorState.codegen_reference rrxrr[rqrP)rrrrrrrrrrxrvrT rT  s I rxrT cj\rSrSrS SjrS SjrS S Sjjr\S Sj5r\S Sj5r Sr g)_CollectiveKerneli cgrrrgs rvr!_CollectiveKernel.should_allocate! rrxcgrrrgs rvr "_CollectiveKernel.has_side_effects$ rrxNcZ[UR5[RRLdS5eURnUR R UlUR RVs/sH!o3R(dMUR PM# snUl gs snf)Nz,Setting cpp kernel needs a valid op_overload) rr rr r r rr r r r )r`r rrus rvr %_CollectiveKernel.set_cpp_kernel_name) sD$$%)>)>> : >!!%~~22#NN44. 4q FAFF4. *. s 7B(B(c <[RR UR"X/UQ70UD6unnnnn SSS5 W (a USU 35eWHn U R 5 M USR 5n U"[ U S9UUWW5n [R"U5n U RRU Vs/sHn[[ U S9X5PM sn5 U RRU Vs/sHoR5PM sn5 SU;a]U RR[[ U S9USU 55 U RRUSR55 gg!,(df  GNQ=fs snfs snf)Nrrrr)r`rrro rrrrG tree_leavesr rrKr5 rr)rrrMrr_example_outputrd re rP r tensor_argrr inpsrrOs rvcreate_inplace _CollectiveKernel.create_inplace: sWW  ""6CDCFC  ! %E2C1D&EE$%J    &Q**, f %      !!&)&&OS Tt^Jf5s Ct T !!T"BTc<<>T"BC F?  # # * *z8&-P     % %fUm&<&<&> ? 9 . U#CsFF>F Fc  [RR UR"X/UQ70UD6unnnnn SSS5 W (a USU 35eWHn U R 5 M [ W[ 5(aURXe5n U"[U S9UUWW5n [U5V Vs/sH(up[URU5U [ U 4/5PM* snn U l [U RU5H_up[R(d[!U5(aM,[RR"R%UR&5 Ma U R$U"URU5UUWW5n [R(d[!U5(d3[RR"R%U R&5 U /U l U $!,(df  GN=fs snn f)Nrr)r`rrro rrrrrh rrr r rFrr6r r]r0r@r)rrrMrrrj rd re rP r r rr rr9rs rvcreate_out_of_place%_CollectiveKernel.create_out_of_placew sWW  ""6CDCFC  ! %F3D2E&FF$%J    & nd + +__[AF!0 F"+>!:  ";IA ((0AYK "; FN #6>>>B ::BSCCGG--11#((; C >> !$$^4 F66>O??))--fkk:$XFNMa *sG-3/G?- G<)r r rIrqr )rM!Union[TensorBox, list[TensorBox]]rr)rMr ) rrrrrr r r r r rrrxrvr r sV  "$@>$@ $@$@x3>33rxr cF^\rSrSrSr\SSj5rSU4SjjrSrU=r $) _WaitKerneli c&URSn[U[5(aURS/$[U[5(aGURSn[U[5(a!URSup4URU/$/$/$r)rMrrr r r)r`rOcollrrs rvget_volatile_reads_WaitKernel.get_volatile_reads skk!n c, - -JJqM? " [ ) )::a=D$ 122Q C())IIrxcv[RR URX5unnnnnSSS5 W(a USU35eU"[ UR 5S9UWWW5nUR R[[ UR 5S9X(55 g!,(df  N}=f)Nrr) r`rrro rrr rrK) rrrOr rd re rP r r s rv create_wait_WaitKernel.create_wait s WW  ""6/  ! %E2C1D&EE$ cnn. /       && :S^^-=> L ! s B** B8c>[TU]5nUR5nUH@nURR [ R "UR555 MB U$rq)rrr rr@r7rr)r`rvolatile_readsvrrs rvr_WaitKernel.get_read_writes sSg-/ 002 B    ! !,"6"6r{{}"E F!rxr)rOrmrrrg) rrrrr r r rrr rs@rvr r s&*  *rxr cV[U[[45(a [U5$[U[[ 45(a5[ [R"5nUHnU[U5-nM U$[U[R5(a [U5$[ 5$rq) rrr.rr(rrr0rrr rr_ rrrs rvr r s!h%&&$Q'' At} % % u|| $ &A ,Q/ /A Au|| $ $$Q''|rxcV[U[[45(a [U5$[U[[ 45(a5[ [R"5nUHnU[U5-nM U$[U[R5(a [U5$[ 5$rq) rrr.rr'rrr0rrr rr_ r s rvr r s!h%&&A At} % % u|| $ &A #A& &A Au|| $ $A|rx)rurrr)rrrr)rrrr])r Sequence[int]rz&Callable[[Sequence[_T]], Sequence[_T]])rz&Callable[[Sequence[_U]], Sequence[_V]]rz&Callable[[Sequence[_T]], Sequence[_U]]rr)rur)rrrr}rq)rz(Sequence[Union[int, torch.SymInt, Expr]]rzOptional[ShapeEnv]rr )rSequence[Union[int, Integer]]rr rC)ruz Literal[None]rrrr)rurnrrrr)rur^rrrzOptional[torch.Tensor])rzOptional[Sequence[_T]]rz Optional[Sequence[Optional[_T]]])ruz2Union[IRNode, OutputSpec, torch.device, None, str]rrO)ruz&Union[IRNode, torch.device, None, str]rr)ruzUnion[Buffer, TensorBox]rrsrr)r%rZr&rZr'rZrr)r9Union[TensorBox, BaseView]r:z"Sequence[Union[int, torch.SymInt]]rr )rHrR rr)rMrrre)rzUnion[Expr, Sequence[Expr]]rrFrr_)r_rrrFr^rrrb)rurnrr)TFNFN)rurnrrr%rr&'Optional[Sequence[Union[int, Integer]]]rrrr rztuple[StorageBox, Layout])rurnr&r rr)rrnrr)rrZr'rZrr)rrFrrs)rS rArr)rrS rz-tuple[list[ShapeAsConstantBuffer], list[Any]])rrrr (- __future__rrsrprrloggingr textwraprWrcollections.abcrrrrrenumr r r r r rrrrrrtyping_extensionsrrr unittest.mockrrrrrtorch._export.serde.schema_exportserderI r} torch._library.utilsrD r r< torch._loggingrtorch.fxtorch.utils._pytree_pytreerG torch._dynamo.utilsrtorch._export.serde.serializer*torch._higher_order_ops.auto_functionalizertorch._inductorrtorch._prims_commonrr r!r"r#torch._subclasses.fake_tensorr$%torch.fx.experimental.symbolic_shapesr%r&r'r(r)r*r+r,r-r.torch.utils._ordered_setr0torch.utils._sympy.functionsr1r2r3torch.utils._sympy.symbolr4rr6r7codegen.commonr8r9r:r;r<r=r>r?r@ loop_bodyrA ops_handlerrBrCrDrEruntime.benchmarkingrF runtime.hintsrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r] virtualizedr^r_r`"torch._library.fake_class_registryra torch.fx.noderbcodegen.cuda.cuda_templatercrrdrerrr  __version__rr ImportErrorrfrgrhrsrirrj getLoggerrrrr- rrrorwrc r{rrrrrrrrrrrrrrrrr r+r?rJrPrnrwrrrr4rGrrrt INNER_FN_TYrhrxrrrrrr"r5r-r/r3rrrrr9r8r#r5rWrbrmryr}rrr6rrrrrr+r)rrrr.rrrr%rCrrPrimitiveInfoTyper]r~rrrr'rrrr r rrKr r r r5 rU r r r r r r r r rrr- r/ r rr r8rmr(rN rT rV ro rv r rr| rrT r r r r rrxrvr? s " 99:   =< ''22,,$$(?M#:   0LL*"  NN-:.*)C"8$%%L)$''NJ  T] T] T]CI&)&C,-)- !   8??4 8 yy~~'T k    sDk!12K8STU   i ) d#$$$D44,!$TX  1 >P   TX  1 >P   N N O O.2  &*  8 !% >9 > >;(' 0     .#G &#G /#G #GL'x,x,vUHHHV~ F~ ~ B&   @ i  F | $ y ! y !   u =) <8JN<N<N +<NBF<N<N~i  i  i  X7AB 7S97St#1#L[ +[ |D 5D D P   U5UUp !<@=A 9 9 99: 9  9 ; 99x : $\ v\ \ ~KKK\((((V79(7979t(:j;jjZRhRRj%%%POAOAd6"K|KK$ S| S S '9 <77QTZQTQTh&$I7VI7XPfP2T$%{$%N.Q*Q*hUV]DU &fi & & & K [ &6& F  UG4_G4G4T B _B JK>K\#udCeCeT?>>B h9hVUM <M M `U l  F / (&" &"RV.;L;|% % P - $/ \/ dulup&)|&)T-,-`<6B5B"F-F,H)lH)V,)|,)^!T!TH-L-46;<6;rU k*&k*\U .  < & ,& V SSSl+ +T%T%nU*v** LUg.\g.g.TUH ,H H V "  "2 "UI) I)I)X,n,^ 6 =l==@\KK\2#2p   oCNJs&e e"!e"