
    7h                   
o   % S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKJr  S SKJrJr  S SKJrJrJrJrJrJrJr  S SKJr  S SKJr  S SKrS SKrS SKrS SK rS SK!J"s  J#r$  S SK%J&r&  S S	K'J(r(  S S
K)J*r*  S SK+J,r,J-r-J.r.J/r/J0r0J1r1J2r2J3r3J4r4J5r5J6r6  S SK7J8r8J9r9  S SK:J;r;  S SK<J=r=  S SK>J?r?J@r@JArAJBrB  SSKCJDrD  SSKEJFrFJGrGJHrHJIrI  SSKJJKrKJLrL  SSKHJMrMJNrNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrVJWrWJXrXJYrYJZrZ  SSK"J[r[J\r\J]r]J^r^J_r_J`r`JaraJbrbJcrcJdrdJereJfrf  SSKgJhrhJiri  \(       a  SSKjJkrk  \" S5      rl\" S5      rm\=" SS/5      rn\R                  " \p5      rq0 rrS\sS'   0 rtS\sS '   \=\R                  R                     " 5       rw\R                  R                  rx\R                  R                  ry\R                  R                  rz\=\R                  R                     " 5       r{\=\R                  R                     " \R                  R                  /5      r~\=\R                  R                     " 5       r0 rS!\sS"'   \R                  GR                  rS# rGS_S$ jrGS`S% jrS& rS' rS( rS) r\" \xGR                  \xGR                  \xGR                  \xGR                  \xGR                  \xGR                  \xGR                  \xGR                   \xGR"                  \xGR$                  \xGR&                  \xGR(                  \xGR*                  \xGR,                  /5        \GR.                  \GR0                  \GR2                  \GR4                  \GR6                  \GR8                  \GR:                  \GR<                  \GR>                  \GR@                  \GR>                  \GRB                  \GRD                  S*.rGSaS+ jrS, rS- rGSbS. jrS/ rS0 r            GScS1 jrS2 r  GSdS3 jrS4\1GRZ                  S4\r4   GSeS5 jjrS6 rGSfS7 jr     GSgS8 jrGShS9 jrGShGSiS: jjr\" \R                  R                  SS;9S< 5       r}\" \zGRh                  SS;9GSiS= j5       rS4S>.GSiS? jjr\" \xGRn                  GRp                  SS;9GSiS@ j5       rS4S4SA.GSjSB jjr\" \zGRv                  SS;9GShGSjSC jj5       rSSD\1GRZ                  S4SSS4S4SE jrSF r\" 5          GShSG jr\" \xGR                  S4SSH9SI 5       r\" \xGR                  S4SSH9SJ 5       r\" \xGR                  \xGR                  \xGR                  \xGR                  \zGR                  /5      SK 5       r\" \xSL5      (       a  \" \xGR                  5      " \5        \" \xGR                  SS;9GSkSM j5       r\" \xGR                  SS;9GSkSN j5       r\" \xGR                  /5      GSkSO j5       r\" \xGR                  5      SP 5       r\" \xGR                  5      SQ 5       r\" \xGR                  5      SR 5       r\" \xGR                  5      SS 5       r\" \xGR                  GR                  5      ST 5       r\" \xGR                  5      SU 5       r\" \xGR                  SS;9SV 5       r\" \zGR                  SS;9SW 5       r\" \xGR                  SS;9SX 5       r\" \xGR                  5      SY 5       r\" \xGR                  SS;9\" \xGRn                  SS;9\" \xGR                  SS;9SZ 5       5       5       r\" \xGR                  SS;9S[ 5       r\" \xGR                  SS;9GSlS\ j5       r\" \xGR                  SS;9GSkS] j5       r\" \xGR                  SS;9GSkS^ j5       r\" \xGR                  SS;9GSkS_ j5       rGSmS` jr\" \GR                  SS;9                GSnSa j5       r\" \GR                  SS;9SSb.                 GSoSc jj5       r\" \GR                  GR                  SS;9              GSpSd j5       r\" \GR                  GR                  SS;9SSb.               GSqSe jj5       r\" \GR                  GR                  SS;9              GSrSf j5       r\" \GR                  GR                  SS;9SSb.               GSsSg jj5       r\" \xGR                  5      GSmSh j5       r\" \xGR                  SS;9GStGSuSi jj5       r\" \xGR                  SS;9GStGSuSj jj5       r\" \xGR                  SS;9GStGSuSk jj5       r\" \xGR                  SS;9Sl 5       r\" \xGR                  SS;9GSmSm j5       r\" \xGR                  SS;9GSmSn j5       r\" \xGR                  SS;9GSmSo j5       r\" \xGR                  SS;9Sp 5       r\" \xGR                  SS;9Sq 5       r\" \xGR                  SS;9Sr 5       rGSmSs jr\" \xGR                  5      GSvSu j5       rGSwSv jr\GR                  Sw 5       rGSkGSxSx jjrGSkGSxSy jjrGSwGSySz jjrGSzS{ jrS| r\" \R                  GR                  GR                   SS;9S} 5       Gr \" \xGR                  SS;9S~ 5       Gr\" \xGR                  SS;9S 5       Gr\" \xGR                  GR                  SS;9S 5       Gr\" \xGR                  5      S 5       Gr\GR                  " S5      S 5       GrS Gr	\" \xGR                  GR                  5      Gr\" \xGR                  GR                  5      Gr\" \xGR                  GR                  5      Gr\" \xGR                  GR                  5      Gr\" \xGR"                  5        \" \xGR                  5      S 5       Gr
\" \xGR                  5      S 5       Gr\" \GGR$                  SS;9S 5       Gr\" \GGR(                  SS;9GS{S j5       Gr\" \GGR,                  SS;9S 5       Gr\" \GGR0                  SS;9S 5       Gr\" \GGR4                  SS;9S S.GS|S jj5       Gr\" \GGR"                  SS;9S S.         GS}S jj5       GrGS~S jGrGSS jGr\" \xGR>                  GR@                  SS;9S4S4SSS.             GSS jj5       Gr\" \xGRB                  \1GRD                  S;9S4S4S.       GSS jj5       Gr!S Gr#S Gr$S Gr%S Gr&S Gr'S Gr(S Gr)S Gr*\" \xGRV                  5        \" \xGRX                  5        \" \xGRZ                  5        \" \xGR\                  S4S9  \" \xGR^                  GR                  S4S9  \" \xGR`                  5        \" \xGRb                  S4S9  \" \xGRd                  S4S9  \GRf                  GRi                  5       (       a  \" \xGRj                  S4S9  \" \xGRl                  5        \" \xGRn                  5        \" \xGRp                  5        \" \xGRr                  GR                  5        \" \xGRt                  GR                  5        \" \xGRv                  5        \" \xGRx                  GRz                  5        \" \xGR|                  GR                  5        \" \xGR~                  GR                  5        \" \xGR                  5        \" \xGR                  S4S9  \" \xGR                  G\#5        \" \xGR                  G\)5        \" \xGR                  G\#5        \" \xGR                  G\$5        \" \xGR                  G\$5        \" \xGR                  G\$5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  G\#5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  G\$5        \" \xGR                  5        \" \xGR                  G\#5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  GR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  GR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  GR                  5        \" \xGR                  GR                  S4S9  \" \xGR                  G\#5        \" \GR                   GR                  GR                  5        \" \GR                   GR                  GR                  5        \" \GR                   GR                  GR                  5        \" \xGR
                  5        \" \xGR                  5        \" \xGR                  G\$5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  5        \" \xGR                  GR                  5        \" \xGR                  GR                  G\*S4S9  \" \xGR                   GR                  G\*S4S9  \" \xGR"                  GR                  G\*S4S9  \" \xGR$                  GR                  G\*S4S9  \" \xGR&                  GR                  G\*S4S9  \" \xGR(                  GR                  G\*S4S9  \" \xGR*                  GR                  G\*S4S9  \" \xGR,                  GR                  G\*S4S9  \" \xGR.                  GR                  G\*S4S9  \" \xGR0                  GR                  G\*S4S9  \" \xGR2                  GR                  G\*5        \" \xGR4                  GR                  G\*5        \" \xGR6                  GR                  G\*5        \" \xGR8                  GR                  G\*5        \" \xGR:                  5        \" \xGR<                  SS;9GShS j5       Gr\" \xGR>                  5      SS.S j5       GrS Gr\" \xS5      (       a  \" \xGRB                  5      " G\5        \" \zGRD                  5      S 5       Gr\" \xGRF                  SS;9GSS j5       Gr\" \xGRH                  SS;9GSS j5       GrS Gr\" \GR                  \xGRL                  /5      SSSS4S.S j5       r\" \GRN                  5      GSfS j5       Gr\" \GRP                  5      S 5       Gr\" \xGRT                  5      S 5       Gr\" \xGRV                  5      S 5       Gr\" \xGRX                  5       GSSSS.S jj5       GrS Gr\" \xGR\                  SS;9S 5       GrS Gr\" \GR`                  \xGR`                  /5      SSSSSSS.S j5       GrS GrS Gr\" \xGRf                  5      " G\" G\5      5      GrG\" G\" S5      5      GrG\" G\" S 5      5      GrS Gr\" \xGRn                  5      SSSSS.S j5       Gr\" \xGRp                  5      SSSSS.S j5       Gr\" \xGRr                  5      SSSSS.S j5       Gr\" \zGRt                  GR                  5      S 5       Gr\" \GRv                  \xGRv                  /5      S 5       Gr\" \xGRx                  SS;9GShS j5       Gr\" \xGRz                  SS;9GSS j5       GrS Gr GSwS jGrS GrGSwS jGr\" \xGR                  SS;9S 5       Gr\" \xGR                  SS;9S 5       Gr\" \xGR                  SS;9GShS j5       Gr\" \xGR                  5      GShS j5       GrS GrS Gr\" \xGR                  SS;9GShS j5       Gr\" \GGR                  SS;9GShS j5       GrGShS jGr\" \xGR                  GR                  S4S9Gr\" \xGR                  GR                  S4S9Gr\" \xGR                  SS;9S 5       Gr\" \xGR                  SS;9S 5       Gr\S 5       Gr\" \xGR                  SS;9GSkS j5       Gr\" \xGR                  SS;9GSS j5       GrSSDS.       GSS jjGr\" \xGR                  SS;9SS.GSS jj5       Gr\" \xGR                  SS;9GSS j5       Gr\" \xGR                  SS;9GSS j5       Gr\" \xGR                  SS;9GSS j5       Gr\" \xGR                  SS;9SDS.GSS jj5       Gr  GS     GSS jjGr\" \xGR                  GR                  5      GSkGSS jj5       Gr\" \xGR                  GR                  5      GSkGSS jj5       Gr\" \xGR(                  GR                  5       GSf   GSS jj5       r\" \xGR*                  GR                  5       GSf   GSS jj5       r\" \xGR                  GR                  5         GS     GSS jj5       Gr\" \xGR                  GR                  5         GS     GSS jj5       GrS Gr\" \zGR                  GR                  5      S 5       Gr        GSS jGr\" \xGR                  SS;9GSmS j5       GrGSS jGrGSS jGrGSS jGr GSS jGrSS.S jGrS GrSS.S jGrS Gr\" \zGR                  SS;9 GShS j5       Gr          GSS jGr\" \zGR                  SS;9S 5       GrS Gr\" \xGR                   SS;9    GSS j5       r\" \xGR"                  SS;9    GSS j5       r\" \xGR$                  GR                  S4S9Gr\" \xGR$                  SS;9S 5       rGSS jGrS GrS GrS Gr\" \xGR                  GR                  S4S9Gr\" \xGR                  5      S 5       Gr\" \xGR                  GR                  S4S9Gr\" \xGR                  5      S 5       GrS Gr\" \xGR                  5      S 5       Gr\" \xGR                  5      S 5       GrS Gr\" \xGR                  GR                  5       GSS j5       Gr\" \xGR                  GR                  S4S9Gr\" \xGR                  GR                  S4S9Gr\" \xGR                  SS;9     GSGS  j5       r\" \xGR                  SS;9     GSGS j5       GrGS Gr\" \xGR                  GR                  S4S9Gr\" \xGR                  SS;9 GSkGS j5       r\" \xGR                   GR                  S4S9Gr\" \xGR                   SS;9 GSkGS j5       Gr GS GrGS GrGSkGSGS jjGrGS Gr\" \xGR                  5      GSSGS	.GS
 jj5       GrGS GrGS GrGS Gr	GS Gr
\" \xGR                  \zGR                  /5      GSkSS4GS.GS jj5       Gr\" \xGR                  5      GSkSS4GS.GS jj5       GrGS Gr\GS 5       Gr\" \xGR                   GR"                  S4S9Gr\" \xGR                   GR&                  S4S9Gr\" \xGR                   GR*                  S4S9Gr\" \xGR                   SDGS9GS 5       GrGShGS jGr\" \xGR0                  5      GS 5       Gr\" \xGR2                  SS;9GShGS j5       Gr\GS 5       Gr\GS 5       Gr\" \xGR8                  SDGS9GSkGS j5       Gr\" \xGR<                  /SDGS9GS 5       GrGSGS jGr\" \zGR8                  /SDGS9GS 5       Gr \" \xGRB                  \xGR8                  GR@                  /SD\1GRD                  SH9GS 5       Gr\" \xGRF                  \zGRF                  /SDGS9GS  5       Gr#\" \xGRH                  \zGRH                  /5      GSSGS	.GS! jj5       Gr%\" \xGRL                  GR                  5      Gr'\" \xGRP                  GR                  5      Gr)\" \xGRT                  GR                  5      Gr+\" \xGRX                  GR                  5      Gr-\" \xGR\                  GR                  5      Gr/\" \xGRL                  5      GSfGS" j5       Gr&\" \xGRP                  5      GSfGS# j5       Gr(\" \xGRT                  5      GS$ 5       Gr*\" \xGRX                  SS;9GSkGS% j5       Gr,\" \xGR\                  SS;9GSkGS& j5       Gr.\" \xGR`                  5      GSSGS	.GS' jj5       Gr0\" \xGRb                  5      GSGS( j5       Gr2\" \xGRf                  SS;9GSGS) j5       Gr4\" \xGRj                  SS;9GSGS* j5       Gr6\" \zGRn                  5      " G\" GS+5      5        \" \xGRp                  5      " G\" GS,5      5      Gr9\" \xGRt                  5      " G\" GS-5      5      Gr;\" \xGRx                  5      " G\" GS.\GR6                  GS/95      Gr=\" \xGR|                  5      " G\" GS0\GR6                  GS/95      Gr?\" \xGR                  SDGS1GS29Gr@\" \xGR                  GR                  S4S9GrA\" \xGR                  GR                  SS;9SStS4GS3.GS4 j5       GrB\" \xGR                  GR                  SS;9GSGS5 j5       GrYGSfGS6 jGrCGSGS7 jGrDG\C" \xGR                  5      GrEG\D" \xGR                  5      GrFG\C" \xGR                  5      GrGG\C" \xGR                  5      GrH\" \xGR                  5      GrIG\D" \xGR                  5      GrJG\D" \xGR                  5      GrK\" \xGR                  5      GrL\" \xGR                  SDGS89GrMG\D" \xGR                  5        G\D" \xGR                  5        \" \xGR                  5      GrP\" \xGR                  5      GrQ\" \xGR                  5      GrR\" \xGR                  GS9GS:9GrS\" \xGR                  5      GrT\" \xGR                  5      GrU\" \xGR                  5      GrVG\C" \xGR                  5        G\C" \xGR                  5      GrX\" \xGR                  \1GRD                  S;9" G\X5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\D" \xR                  5        \" \xGR                  SSD\GRB                  GS;9Gr]\" \xGR                  SSD\GRB                  GS;9Gr^\" \xGR                  SSD\GRB                  GS;9Gr_\" \xGR                  SSD\GRB                  GS;9Gr`\" \xGR                  5      Gra\" \xGR                  5      Grb\" \xGR                  5      " G\a5        \" \xGR                  5      " G\b5        \" \xGR                  5      Gre\" \xGR                  5      GrPG\C" \xGR                  5      Grf\" \xGR                  5        \" \xGR                  GS<GS:9Grh\" \xGR                  5        \" \xGR                  \GRB                  GS/9  \" \xGR                  5      " G\e5        \" \xGR                  \GRB                  GS/9  \" \xGR                  \GRB                  GS/9  \" \xGR                  \GRB                  GS/9  \" \xGR                  \GRB                  GS/9Grn\" \xGR                  \GRB                  GS/9  \" \xGR                  \GRB                  GS/9  G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                  5        G\C" \xGR                   5        SGS=GKGJGrGJGr  GS> GrG\ HR  GrG\" \xG\5       H  u  GrGrGr\" G\G\G\G\GS?9  M     G\" \zG\5       H  u  GrGrGr\" G\G\G\G\GS?9  M     MT     \" \xGR                  GR                  G\@SDGS89Gr\" \xGR                  GR&                  G\@SDGS89Gr\" \xGR                  GR@                  G\@SDGS89  \" \xGR                  GR                  G\5      Gr\" \xGR                  GR@                  G\5        \" \xGR                  GR&                  G\5      Gr\" \xGR                   GR                  G\M5        \" \xGR                   GR&                  G\M5        \" \xGR"                  GR                  G\e5        \" \xGR$                  GR                  G\P5        \" \xGR&                  GR&                  G\5        \" \xGR&                  GR                  G\5        \" \xGR&                  GR(                  G\5        \" \xGR*                  GR                  G\5      Gr\" \xGR*                  GR@                  G\5        \" \xGR*                  GR&                  G\5      Gr\" \xGR0                  G\K5        \" \xGR2                  G\E5        \" \xGR4                  GR                  G\a5        \" \xGR4                  GR&                  G\a5        \" \xGR6                  GR                  G\b5        \" \xGR6                  GR&                  G\b5        \" \xGR8                  GR                  G\a5        \" \xGR8                  GR&                  G\a5        \" \xGR:                  GR                  G\b5        \" \xGR:                  GR&                  G\b5        \" \xGR<                  G\f5        \" \xGR>                  G\h5        \" \xGR@                  G\5        GS@ GrG\" \xGRD                  GR                  \xGR                  GR                  G\5        G\" \xGRD                  GR&                  \xGR                  GR&                  G\5        G\" \xGRF                  GR                  \xGR                  GR                  G\5        G\" \xGRF                  GR&                  \xGR                  GR&                  G\5        G\" \xGRH                  GR                  \xGR*                  GR                  G\5        G\" \xGRH                  GR&                  \xGR*                  GR&                  G\5        GSA GrG\" \xGRL                  G\@5        G\" \xGRN                  G\Q5        G\" \xGRP                  G\R5        G\" \xGRR                  G\S5        G\" \xGRT                  G\T5        G\" \xGRV                  G\U5        G\" \xGRX                  G\V5        G\" \xGRZ                  G\5        G\" \xGR\                  GR@                  G\5        G\" \xGR\                  GR^                  G\5        G\" \xGR`                  G\]5        G\" \xGRb                  G\^5        G\" \xGRd                  G\_5        G\" \xGRf                  G\`5        G\" \xGRh                  G\M5        G\" \xGRj                  G\I5        G\" \xGRl                  G\J5        \" \xGRn                  5      " G\Q5        \" \xGRp                  5      " G\R5        \" \xGRr                  5      " G\T5        \" \xGRt                  5      " G\U5        \" \xGRv                  5      " G\V5        G\" \xGRx                  \xGRn                  5        G\" \xGRz                  \xGRp                  5        G\" \xGR|                  \xGRr                  5        G\" \xGR~                  \xGRt                  5        G\" \xGR                  \xGRv                  5        \" \xGR                  5      GSfGSB j5       Gr\" \xGR                  GR                  5      GSC 5       Gr\" \xGR                  GR                  5      GSD 5       Gr\" \xGR                  5      GSE 5       Gr\8GR                  " 5        H  u  GrGr\" \9" G\5      5      " G\5        M      \" \GR                  5      GSF 5       Gr\" \xGR                  5      GSG 5       Gr\" \R                  GR                  GR                  5      GSH 5       Gr\" \R                  GR                  GR                  5      GSI 5       Gr\" \R                  R                  GR                  GR                  5      GSJ 5       Gr\" \R                  GR                  GSK5      (       a;  \" \R                  GR                  GR2                  GR                  5      GSL 5       Gr\" \R                  R                  GR                  5      SS.GSM j5       GrS GSNGKGJGr  \" G\5        \" \*5      GSO 5       Gr\" \R                  GR                  GR                  SS;9GSP 5       Gr\" \R                  GR                  GR                  SS;9GSQ 5       Gr\" \R                  GR                  GR                  SS;9GSGSR j5       Gr\" \R                  GR                  SS;9SGSS.GSGST jj5       Gr\" \(SS;9    GSGSU j5       Gr\" \R                  R                  GR                  GR                  5      GSV 5       Gr\" \R                  GR                  GR                  SS;9GSW 5       GrSGSXGKGJGr  G\" 5         \" \GGR                  SS;9GSY 5       GrSGSZKEGJGr  \D" G\5        SGS[KEGJGr  G\GR                  " 5         G\GR                  " 5         SGS\KEGJGr  G\GR                  " 5         SGS]KEGJGr  G\GR                  " 5         \GR                  GSGS^ j5       Grg(      )annotationsN)defaultdict)IterableSequence)AnyCallablecastOptionalTYPE_CHECKINGTypeVarUnion)	ParamSpec)patch)counters)associative_scan_op)triton_kernel_wrapper_mutation)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)free_unbacked_symbols)
OrderedSet)CeilDivFloorDivIdentityModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)	DtypeView
ExpandViewIndexingConstantIRNode	is_tritonOnlineSoftmaxReductionops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)ceildivdecode_device
is_dynamicis_gpuis_pointwise_useis_view,needs_fallback_due_to_atomic_add_limitationspad_listlike#register_op_dtype_propagation_rules#register_op_requires_libdevice_fp64sympy_productuse_scatter_fallback)opsV)ReductionType_T_Pztorchvision::roi_alignzaten::index_addz8dict[Union[Callable[..., Any], str], Callable[..., Any]]	loweringsz9dict[torch._ops.OpOverload, Optional[Callable[..., Any]]]_maybe_layout_constraintsz2dict[torch._ops.OpOverload, torch._ops.OpOverload]inplaceable_foreach_opsc                     [         R                  R                  R                   H=  n U R                   H*  nUR                  S:X  a  UR
                  [        ;   a  M)      g   M?     g)Ncall_functionTF)rJ   graphcurrent_nodeusersoptargetforeach_ops)nodeusers     R/var/www/fran/franai/venv/lib/python3.13/site-packages/torch/_inductor/lowering.pycur_node_has_non_foreach_usersr\   x   sI    $$**JJDGG.DKK;4N  +
     c                   [        [        5      nSn[        U 5       H  u  p4[        U[        5      (       d  SnU4n[        U6 (       + =(       d    [        R                  nS nU H4  n[        U[        5      (       d  M  UR                  R                  5       n  O   Uc   S5       eU(       a  Uu  nXU4   R                  X445        M     U$ )NFTz.foreach op should have at least one tensor arg)r   list	enumerate
isinstancer   r?   r)   #combo_kernel_foreach_dynamic_shapesr:   data
get_deviceappend)	arg_pairsoutunpack_argsiargsuse_foreachdevicets           r[   group_foreach_argsrn      s    
d
CKY'$))K7DD!!OV%O%O 	 A!Y''**,  !S#SS!GT[!"))1)4 (  Jr]   c                ~    [        U [        R                  R                  5      (       d  gU [        ;   a	  [        U    $ g)zHGet layout constraints. Returns None if there are no layout constraints.N)ra   torch_ops
OpOverloadrO   )fns    r[   maybe_layout_constraintsrt      s3    b%**//00	&&(,,r]   c                   U [         R                  R                  R                  :X  a  [        $ U [         R                  R                  R
                  :X  a  [        $ U [         R                  R                  R                  :X  a  [        $ U [         R                  R                  R                  :X  a  g [        SU  35      e)NzUnknown layout constraint tag: )rp   _CTagneeds_exact_stridesconstrain_to_fake_tensorsneeds_contiguous_stridesrequire_contiguous_stridesneeds_fixed_stride_orderconstrain_to_fx_stridesflexible_layoutAssertionError)tags    r[   tag_to_layout_constraintr      s    
ehhll...((
ehhll333))
ehhll333&&
ehhll***
:3%@
AAr]   c                .    U (       d  [        SU 35      eg )Nzinductor does not support NotImplementedError)condmsgs     r[   
assert_nyir      s    !$>se"DEE r]   c                j  ^  [        T [        [        [        [        45      (       a  T  Vs/ s H  n[        U5      PM     sn$ [        R                  T 5        [        T [        R                  R                  5      (       a.  [        R                  U 4S jT R                  5        5       5        g g s  snf )Nc              3  <   >#    U  H  n[        TU5      v   M     g 7fN)getattr).0overloadrs   s     r[   	<genexpr>,add_needs_realized_inputs.<locals>.<genexpr>   s      %
2@hGB!!.   )ra   r_   settupler!   add_needs_realized_inputsneeds_realized_inputsaddrp   rq   OpOverloadPacketupdate	overloads)rs   xs   ` r[   r   r      s    "tS%455689b)!,b99b!"ejj1122$$ %
24,,.%
 	
 3 :s   B0c                    [        U [        R                  R                  5      (       a*  U R	                  5        H  nU[
        [        X5      '   M     g U[
        U '   g r   )ra   rp   rq   r   r   rO   r   )rs   
constraintr   s      r[   add_layout_constraintr      sF    "ejj1122H?I%gb&;< ' )3!"%r]   )r   r(   r&                     	   
         c                p    [        U [        5      (       d  U $ U [        ;   d   SU  S35       e[        U    n U $ )Nzid z missing from DTYPE_ID_LOOKUP)ra   intDTYPE_ID_LOOKUPdtypes    r[   decode_dtyper      s@    eS!!O#Os5'1N%OO#E"ELr]   c                   [        U [        5      (       a9  [        U R                  5       5      =(       d    [	        U R                  5       5      $ [        U [
        R                  5      (       a  U R                  SL $ [        U [        5      $ )NT)	ra   r:   r   	get_dtyper   sympyExpr
is_integerr   r   s    r[   is_integer_typer      s^    !Y.Q2B1;;=2QQ	Auzz	"	"||t##!S!!r]   c                ~    [        U [        5      (       a  [        U R                  5       5      $ [        U [        5      $ r   )ra   r:   r   r   boolr   s    r[   is_boolean_typer     s-    !Y..!T""r]   c                ^    S nU Vs/ s H
  o2" U5      PM     nn[        USU 06u  pVU$ s  snf )Nc                    [        U [        [        R                  45      (       a  U $ [	        U R                  5       5      n[        R                  " S/U-  U R                  5       S9$ )Nr(   r   )	ra   r   r   Basiclenget_sizerp   zerosr   )inpdims     r[   construct_input+get_promoted_dtype.<locals>.construct_input  sJ    cFEKK011Jclln%C;;sSy@@r]   type_promotion_kind)r   )r   rj   r   arginps_r   s          r[   get_promoted_dtyper     s?    A -11DSOC DD1!4Q=PQHAL 2s   *c                R   [        U [        [        45      (       d  U /n O[        U 5      n [        U 5       Hm  n[        U[        R                  R
                  5      (       d  M.  UR                  5        H+  n[        X5      nU[        ;  d  M  U R                  U5        M-     Mo     U $ r   )
ra   r_   r   rp   rq   r   r   r   rN   re   )aten_fnrs   r   other_fns       r[   get_overloadsr     s}    ge}--)w-7mb%**5566LLN"209,NN8, +  Nr]   c                    [        U [        R                  R                  5      (       a  XR                  ;   $ [        U [        R                  R
                  5      (       a  XR                  5       ;   $ gNF)ra   rp   rq   r   _qualified_op_namerr   name)rV   	namespaces     r[   in_namespacer   )  sQ    "ejj11221111	B

--	.	.GGI%%r]   c           	       ^ ^^^ [        T 5       VVs/ s H  u  pV[        U[        5      (       d  M  UPM      nnnTR                  5        VV	s/ s H  u  p[        U	[        5      (       d  M  UPM      n
nn	U(       d  U
(       d  T T4$ U(       d  U(       Ga  U(       a  [        R
                  mO{T  Vs/ s H=  n[        U[        [        R                  45      (       d  [        US5      (       d  M;  UPM?     nnUR                  S TR                  5        5       5        [        USU06mU(       a  T US      OTU
S      R                  5       mUU4S jnT  Vs/ s H
  o" U5      PM     snm TR                  5        VV	s0 s H  u  pX" U	5      _M     sn	nmU(       GaF  [        [        [         R"                  " U 4S jU 5       U4S jU
 5       5      5      6 n[        US   R%                  5       5      n['        X~S [)        U5       5       H
  u  pVUT U'   M     ['        X[)        U5      S  5       H
  u  pUTU'   M     [+        [)        T 5      5       HD  n[        T U   [,        R.                  5      (       d  M'  [0        R2                  " T U   U5      T U'   MF     T HD  n[        TU   [,        R.                  5      (       d  M'  [0        R2                  " TU   U5      TU'   MF     T T4$ s  snnf s  sn	nf s  snf s  snf s  sn	nf )Nr   c              3  L   #    U  H  n[        US 5      (       d  M  Uv   M     g7f)r   N)hasattr)r   as     r[   r   !transform_args.<locals>.<genexpr>I  s     !T_7@S!!_s   $	$r   r   c                   > [        U [        5      (       a  [        U T5      $ [        U [        R                  5      (       a   [        R                  " U R
                  TTS9$ U $ )Nvaluer   rl   )ra   r:   to_dtyper+   Constantr   )r   rl   r   s    r[   promotetransform_args.<locals>.promoteT  sJ    #y))U++C--{{%OO
r]   c              3  .   >#    U  H
  nTU   v   M     g 7fr    )r   ri   rj   s     r[   r   r   c  s     3lT!Wl   c              3  .   >#    U  H
  nTU   v   M     g 7fr   r   )r   kkwargss     r[   r   r   d  s     71VAYr   )r`   ra   r:   itemsrp   r   r   r   r   r   extendvaluesr   rd   broadcast_tensorsr_   	itertoolschainr   zipr   ranger+   r   r0   create)rj   r   	broadcastr   convert_input_to_boolri   r   args_indicesr   vkwargs_indicesr   promoting_argsr   broadcastedsizerl   r   s   ``              @@r[   transform_argsr   1  s    #,D/N/$!Z95MA/LN$*LLNONDAjI6NaNNOV|3 JJE
 Aa&%++!67771g;N    !!!TV]]_!TT&$7E &2Da!vnQ>O7P
*, 	
	 %))Dq
D),2LLN;NDA!WQZ-N;'3l377
 KN++-.2EC4E&FGDADG HC4E4G(HIDAF1I J s4y!A$q'2;;//$++DGT:Q " A&)R[[11&--fQi>q	  <} OO2 *;s.   K+K+K1-K13:K71K7K<Lc                   ^ [         R                  " T5      U4S j5       n[        U 5      n[        R	                  U5        [
        R	                  [        R                  X25      5        U$ )a  
Add a foreach lowering to lowerings dict.

Arguments:
    aten_fn: torch.ops.aten.* fn we are lowering
    decomp_fn: alternate implementation on our IR
    broadcast: True to apply broadcasting to tensor inputs
    type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
    convert_input_to_bool: some logical ops require inputs are converted to bool
c                 P   > [        U 5      S::  d   eT" U 0 UD6n[        U5        U$ )Nr&   )r   r;   )rj   r   rg   	decomp_fns      r[   wrapped+_register_foreach_lowering.<locals>.wrapped  s/    4yA~~((C
r]   )	functoolswrapsr   rX   r   rN   dictfromkeys)r   r   r   aten_fnss    `  r[   _register_foreach_loweringr   y  sV     __Y   W%Hx T]]856Nr]   c                   ^ ^^^^ [         R                  " T5      U UUUU4S j5       n[        T 5      m UR                  [        R                  T U5      5        U$ )a  
Add a lowering to lowerings dict

Arguments:
    aten_fn: torch.ops.aten.* fn we are lowering
    decomp_fn: alternate implementation on our IR
    broadcast: True to apply broadcasting to tensor inputs
    type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
    convert_input_to_bool: some logical ops require inputs are converted to bool
c                   > [        U 5      n [        U5      nSn[        U 5      S:X  a.  [        U S   [         [        45      (       a  Sn[        U S   5      n [        S T 5       5      (       d,  [        S UR                  5        5       5      (       a   S5       e[        XTTT5      u  pU(       a  U /n T" U 0 UD6n[        U5        U$ )NFr(   r   Tc              3  X   #    U  H   o[         ;   =(       d    [        US 5      v   M"     g7f)_c10d_functionalN)	fallbacksr   )r   rs   s     r[   r   6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s&      
PW"9_DR1C DDPWs   (*c              3  *   #    U  H	  oS :H  v   M     g7f)rg   Nr   r   r   s     r[   r   r    s     =}!:}   zout= ops aren't yet supported)
r_   r   r   ra   r   allanykeysr   r;   )	rj   r   unpackedrg   r   r   r   r   r   s	       r[   r   #_register_lowering.<locals>.wrapped  s    t*!%ft9>ja4-@@HQ=D 
PW
 
 
 =v{{}=== /= &)%8:O
 6D((C
r]   )r   r   r   r   r   r   )r   r   r   r   r   lowering_dictr   s   `````  r[   _register_loweringr    sO    & __Y   : G$Gw89Nr]   Fc           	     <    [         R                  " [        U UUUUS9$ )z#
Shim to support decorator syntax.
)r   r   r   r  )r   partialr  )r   r   r   r   r  s        r[   register_loweringr    s)     /3# r]   c                   / n[         R                  " [        U 5      [        U5      [        R                  R
                  S9 GHf  u  p4[        R                  R                  R                  R                  [        R                  " US5      SS9(       a  UR                  U5        Me  [        R                  R                  R                  R                  [        R                  " US5      SS9(       a  UR                  U5        M  [        R                  R                  R                  X45        [        [        R                  " U5      R                   5      [        [        R                  " U5      R                   5      :  a  UR                  U5        GMU  UR                  U5        GMi     [#        [        U5      5      $ )z
Broadcasting logic based on symbolic shapes.

We give the shapes 0 and 1 concrete values, while all other shapes
are symbolic sympy formulas.
)	fillvaluer(   Tsize_oblivious)r   zip_longestreversedr   SOnerJ   rS   sizevars	shape_envevaluate_exprEqre   guard_equalsr   expandfree_symbolsr   )r   boutputr   ys        r[   broadcast_symbolic_shapesr$    s%    F%%hqk8A;%''++V77%%33HHQN4 4 
 MM!WW''55HHQN4 6 
 MM!GG))!/5<<?//03u||A7S7S3TTa a  W &!""r]   c                  ^ Ub
  Ub   S5       eUc  Uc  [         R                  n[        S U  5       5      (       d  U $ [        S U  5       5      (       a2  U=(       d    [	        U SU06mU4S jnU  Vs/ s H
  oC" U5      PM     sn$ [        S U  5       5      n/ nU  GH!  n[        U[        [        45      (       an  UR                  [        R                  " [        R                  " XER                  5       UR                  5       S9[!        UR#                  5       5      5      5        M  [        U[$        R&                  5      (       ad  UR                  [        R                  " [)        XER                  5       UR                  5       S9[!        UR#                  5       5      5      5        GM  UR                  U5        GM$     U$ s  snf )	NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3  l   #    U  H*  n[        U[        R                  [        [        45      v   M,     g 7fr   )ra   r   r   r   floatr  s     r[   r   $promote_constants.<locals>.<genexpr>  s%     HAz!ekk3677   24c              3  l   #    U  H*  n[        U[        [        [        R                  45      v   M,     g 7fr   )ra   r   r'  r   r   r  s     r[   r   r(    s%     
DV:a#uekk233Vr)  r   c                   > [        U [        R                  5      (       a  [        R                  " U T[        S 5      S9$ [        R                  " U T[        S 5      S9$ )Nindexr   rl   r   )ra   r   r   r+   r1   r>   r   r   r   s    r[   
const_func%promote_constants.<locals>.const_func  sM    !U[[))**5t1D  {{%d@STTr]   c              3  ~   #    U  H3  n[        U[        [        [        R                  45      (       d  M/  Uv   M5     g 7fr   )ra   r:   r0   r+   r   r  s     r[   r   r(    s&     WA:a)Z1U#Vaas   .=	=r   r,  )r   DEFAULTr	  r  r   nextra   r   r'  re   r0   r   r+   r   r   get_device_or_errorr_   r   r   r   r1   )inputsoverride_return_dtyper   r/  r   exrg   r   s          @r[   promote_constantsr8    s    (,?,G OG $)<)D=EEHHHH

DV
DDD% 
);*
)<*
	U (..v!
1v..	WW	WB
Ca#u&&JJ!!KK||~b>T>T>V '	 5;;''JJ!!$||~b>T>T>V '	 JJqM) , J3 /s   3Gc                4   ^ ^^^^^ S S.SUU UUUU4S jjjnU$ )Nalphac           	     &  >^^^^	^
^ Tb%  [        S T 5       5      (       a  T(       a   eT" T6 $ [        TT5      mT(       a'  U b#  U S:w  a  [        T5      m[        TS   U 5      TS'   OU b   eT Vs/ s H  o"R	                  5       PM     snm	TS   R                  5       mT=(       d    TS   R                  5       mTSS   Hh  n[        U[        R                  5      (       a  M$  [        T5      [        UR                  5       5      :X  a  ML   ST ST SUR                  5        35       e   [        R                  [        R                  4m
[        R                  S L=(       a    [!        [        R                  SS 5      S L=(       am    [        R                  R"                  R$                  S L=(       a@    [        R                  R"                  R$                  R'                  SS	5      =(       a    TT
;   mUUUUU	U
UU4S
 jnT(       d_  S nT H=  n[)        UR+                  5       R,                  5      (       d  M-  UR+                  5       n  O   U(       d  TS   R+                  5       nT=(       d    Wn[.        R0                  " UTUTS9$ s  snf )Nc              3  f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fr   ra   r2   r3   )r   r   s     r[   r   0make_pointwise.<locals>.inner.<locals>.<genexpr>5  s&      /
BH3JsF#6	#6&   /1r(   r   zndim mismatch  rT   low_precision_pointwise_barrierFc                &  > [        U 5      [        T5      :X  d   SU  ST 35       eT[        R                  :X  a  Tb  T" T Vs/ s H
  o" U 5      PM     sn6 $ / n[        T5       Hi  u  p1U" U 5      nT
U   R	                  5       nT(       a1  UT;   a+  [
        R                  " XESS9n[
        R                  " Xe5      nUR                  U5        Mk     T	" U6 nT(       a-  [
        R                  " UTSS9n[
        R                  " UT5      $ U$ s  snf )Nzwrong ndim rB  F)use_compute_types)r   rp   r   r`   r   rI   r   re   )r-  loadinputs_loaded	inp_indexrg   	inp_dtypedowncastr   emulate_precision_castsrs   r5  loaders	low_pr_fpoverride_fn_when_input_boolrangess          r[   inner_fn/make_pointwise.<locals>.inner.<locals>.inner_fnW  s   u:V,LE7!F8.LL,

"'B'N2W4UWTT%[W4UVV "'0'9OIu+C &y 1 ; ; =I.9	3I#&<<RW#X!ll8?!((- (: -(*  #||C%PH<<%88
# 5Vs   Drl   r   rP  rO  )r	  r8  r_   mulmake_loaderr   r   ra   r+   BaseConstantr   rp   bfloat16float16rJ   rS   r   rT   metagetr@   rd   typer7   r   )r;  r5  r   otherrP  rl   ri   r   rK  rL  rM  rO  allow_alphars   override_devicerN  r6  triton_fallbacks    `     @@@@@r[   innermake_pointwise.<locals>.inner4  sF   &3 /
BH/
 ,
 ,
 #"?"F++"6+@A UaZf U3r
= =,23Fq==?F3##%%>)<)<)>ABZEeR__55V I : At1VHAenn.>-?@A    ^^U]]3	GG4 #6dB#$$))5# $$))--.OQVW# " 	 	 	, F!,,.--..\\^F  --/ *F	
 	
m 4s   3J)r5  r:   r   )rs   r6  r]  rN  r\  r^  r_  s   `````` r[   make_pointwisera  ,  s     )- I
 I
V Lr]   c                $   ^ ^ SS.SUU 4S jjjnU$ )Nr(   r:  c                8  > [        [        R                  R                  R                  5      S:H  =(       d<    [        R                  R                  R
                  [        ;   =(       d
    [        5       nS nU H"  n[        U[        [        45      (       d  M   Un  O   Uc   S5       e/ nU HO  n[        U[        [        45      (       d   UR                  U/[        U5      -  5        M>  UR                  U5        MQ     [        [        U6 5      nS /[        U5      -  nUR                  5        H  u  u  pn
/ nU
 H  u  nnT(       a	  T" USU 06nOT" U6 nXU'   [        R                  R                  U[         R"                  5      (       d  MT  U	(       d  M]  U(       d  Mf  UR%                  5         UR                  UR'                  5       5        M     U(       d  M  [        R                  R)                  U5        M     [+        S U 5       5      (       d   eU$ )Nr   z1at least one input must be a list to a foreach opr;  c              3  (   #    U  H  oS Lv   M
     g 7fr   r   r  s     r[   r   8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>  s     2'QD='   )r   rJ   rS   rT   rU   rW   inplace_foreach_opsr\   ra   r_   r   re   rn   r   r   has_featureBackendFeatureFOREACHrealizeget_operation_nameregister_operation_listr  )r;  r5  realize_outputsa_list_inputinputbroadcast_inputsgroupsoutputsrl   rk   groupoperation_list
output_indrj   r"  r\  pw_fns                  r[   r_  %make_foreach_pointwise.<locals>.inner  s   $$**+q0 0ww##**.AA0-/ 	 E%$//$  ' 	
?	
'
 EedE]33 ''#l2C(CD ''.	  $C)9$:;&3|,,,2LLN(!V5(*N "D66F"D\F&,
# GG''0F0FGG#'NN$"))&*C*C*EF   ~//?- -;0 2'22222r]   )r5  zlist[list[TensorBox]]r   )rw  r\  r_  s   `` r[   make_foreach_pointwisery    s    45 4 4l Lr]   c                   ^^ U R                  5       mTT:X  a  U(       a  [        U 5      $ U $ UU4S jn[        UTS9" U 5      $ )Nc                0   > [         R                  " U TTS9$ )N)	src_dtype)rI   r   )r   r   r|  s    r[   	_to_dtypeto_dtype.<locals>._to_dtype  s    ||Au	::r]   r6  )r   clonera  )r   r   copyr}  r|  s    `  @r[   r   r     s@    IEuQx&Q&; )5A!DDr]   r   c                   SSK Jn  UnU R                  nU" U[        R                  S9n[        R
                  " U5         UR                  " U6   SSS5        UR                  nU(       d   e[        U5      nS/[        U5      -  n	UR                  5        H  u  u  pn/ nU Hw  u  nnXU'   [        R                  R                  U
[        R                  5      (       d  M?  U(       d  MH  UR                  5         UR                  UR!                  5       5        My     U(       d  M  [        R                  R#                  U5        M     [%        S U	 5       5      (       d   eU	$ ! , (       d  f       GN= f)a)  
This lowers an invocation of foreach_map
The way this works is that an arbitrary N-arg func is provided by the user, looped over by the
polyfill with the same semantics as a foreach op (a loop applying an n-ary function to n args)
and then traced into a subgraph by dynamo.
This code allows us to inline the subgraph into the main graph lowering using the PontwiseSubgraphLowering.
The graph outputs represent the vertically fused sequence of ops, and then register_operation_list
below registers the buffers as horizontally fuseable in the scheduler.
r(   )PointwiseSubgraphLowering)root_graph_loweringNc              3  (   #    U  H  oS Lv   M
     g 7fr   r   r  s     r[   r   _foreach_map.<locals>.<genexpr>  s     .g}grf  )subgraph_loweringr  graph_modulerJ   rS   set_graph_handlerrungraph_outputsrn   r   r   rh  ri  rj  rk  re   rl  rm  r  )subgraphrj   r   r  r5  gmpw_subgraphsub_outputsrr  rs  rl   rk   rt  ru  rv  r"  s                   r[   _foreach_mapr    s2    =F			B+BAGGLK	
		[	)  
* ++K;,Ffs;''G(.$u$& 
"(Jww""6>+A+ABB{{ %%f&?&?&AB  >GG++N; )7 .g.....N3 
*	)s    E%%
E4c                D   UR                   (       d  U R                  5       R                   (       ag  U R                  5       (       a*  [        XS9n[        R
                  R                  X 5        U$ [        [        R                  R                  SS9" X5      $ [        XSS9$ )Nr   Fadd_to_fallback_setTr  )
is_complexr   r   
empty_liker+   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr   )r   r   dsts      r[   _convert_element_typer    s~    1;;=33::<< Q,C""))#1J#**22  A4((r]   r  c                  U R                  5       nX1:X  a  U(       a  [        U 5      $ U $ S nU" U5      nU" U5      nXV:w  a)  [        [        R                  R
                  5      " X5      $ [        [        R                  " X5      5      $ )Nc                    U R                   (       a   [        R                  " U 5      R                  $ [        R                  " U 5      R                  $ r   )is_floating_pointrp   finfobitsiinfor   s    r[   _get_primitive_bitwidth1to_dtype_bitcast.<locals>._get_primitive_bitwidth	  s5    "";;u%***;;u%***r]   )	r   r  r  atenviewr   r:   r/   r   )r   r   r  x_dtyper  src_bitsdst_bitss          r[   to_dtype_bitcastr    ss    kkmGuQx&Q&+ 'w/H&u-H		0::))!344r]   c                8   UR                   (       d  U R                  5       R                   (       a`  [        R                  " [        R
                  R                  [        R                  R                  R                  R                  X5      5      $ [        X5      $ r   )r  r   r:   r   r+   ComplexViewrp   rI   r  r  r   r  r.  s     r[   _view_dtyper    sa    1;;=33NN!!%)).."5"5";";QF
 	
 A%%r]   r  non_blockingc                   [        U5      nU R                  5       U:X  a  U(       a  [        U 5      $ U $ [        R                  " [
        R                  R	                  XU5      5      $ r   )r>   rd   r  r:   r   r+   
DeviceCopy)r   rl   r  r  s       r[   	to_devicer  !  sM    6"F||~uQx&Q&BMM00LIJJr]   c                    [        XSUS9$ )NTr  )r  )r   rl   r  s      r[   _device_putr  (  s    QTEEr]   Tc	                    U=(       d    U R                   n[        U5      n	[        XU5        Ub  [        U5      n[        U	UUUUS9n	[	        U UUUS9" U	5      n	[        [        U5      (       a  [	        [        [        U5      SUS9" U	5        U	$ )z3A pointwise function that maps ops.{name} to inputsN)r6  rN  r\  r^  )r   r   r   )r   r   )__name__r5   rE   ra  r  r   r  r   )
r   r   r   r   r   r6  rN  r\  r^  rs   s
             r[   register_pointwiser  -  s     #7##D	T	B'#8 #.&12M&N#	
3$?'
B 
/3	

 	

B udE4  $"7	
 		
 Ir]   c                 2  ^^ Sn [        S5      mU4S jnU4S jn[        U5      [        U[        R                  S9/mU4S jn[	        [
        R                  5      " U5      n[        [        U 5      (       a  [	        [        [        U 5      SS9" U5        U$ )z2A pointwise function that maps ops.frexp to inputsfrexpc                    > T" U 0 UD6S   $ Nr   r   rj   r   r  s     r[   frexp0register_frexp.<locals>.frexp0_      d%f%a((r]   c                    > T" U 0 UD6S   $ Nr(   r   r  s     r[   frexp1register_frexp.<locals>.frexp1b  r  r]   r  c                 0   > TS   " U 0 UD6TS   " U 0 UD64$ Nr   r(   r   )rj   r   pw_fnss     r[   rs   register_frexp.<locals>.fnj  s*    ay$)&)6!9d+Ef+EEEr]   Nr  )
r5   ra  rp   int32r  r  r  r   r  r   )r   r  r  rs   r  r  s       @@r[   register_frexpr  Z  s    D E)) 	vvU[[AF
F 





B udE4  $	
 	 Ir]   c                .    [        XS9n[        X5      nU$ )Nr\  )ry  r   )r   pointwise_lowering_fnr\  rs   s       r[   register_foreach_pointwiser  |  s    
 
  5	OB	#G	0BIr]   )r   r   c           
     *   S n[        U[        [        45      (       a  [        U5      " U5      n[        U[        [        45      (       a  [        U5      " U5      nXU/n[	        US   US   [
        R                  S9n[        U5       VVs/ s H  u  pg[        U[        5      (       d  M  UPM      nnn[        U[        U Vs/ s H  odU   PM	     sn6 5       H	  u  pgXtU'   M     [        [        U5      5       H]  n[        XF   [        R                  5      (       d  M&  [        R                   " XF   [#        XHS      R%                  5       5      5      XF'   M_     ['        X5S9" US   [)        US   U5      [)        US   U5      5      $ s  snnf s  snf )Nc                 (    [         R                  " U 6 $ r   )rI   whererj   s    r[   rs   where.<locals>.fn  s    yy$r]   r(   r&   r  r   r  )ra   r'  r   constant_liker   r   r2  r`   r:   r   r   r   r   r+   r   r0   r   r_   r   ra  r   )	r   r   r!  rs   rj   r   ri   r   indicess	            r[   r  r    sX     !eS\""!Q!eS\""!QQ<DQa.M.U.UE 't_I_TQ
1i0Hq_GIG.'0J'Qa'0JKLQ M3t9dgr{{++ ''d1:6F6O6O6Q1RSDG  ":Q$q'5)8DGU+C  J0Js   F
,F
Fc                    [        U 5      S:X  a)  [        U S   [        [        45      (       a  [	        U S   6 $ [
        R                  " [        U  Vs/ s H  oR                  5       PM     sn/ 5      n/ nU  Hg  nUR                  5       n[        U5      [        U5      :w  d   [        S [        XB5       5       5      (       a  [        X5      nUR                  U5        Mi     U$ s  snf )Nr(   r   c              3    #    U  GHC  u  p[         R                  R                  R                  R	                  [
        R                  " US 5      SS9=(       aL    [         R                  R                  R                  R	                  [
        R                  " US 5      SS9(       + =(       d    [         R                  R                  R                  R	                  [
        R                  " US 5      SS9(       + =(       aG    [         R                  R                  R                  R	                  [
        R                  " US 5      SS9v   GMF     g7f)r(   Tr  N)rJ   rS   r  r  r  r   r  r   r   r!  s      r[   r   $broadcast_tensors.<locals>.<genexpr>  s	     ,
& +! GG$$..<<At =   GG,,66DDAt E  	 ((22@@At A    ((22@@At A " +s   EE)r   ra   r_   r   r   r   reducer$  r   r	  r   r  re   )r5  r   rW   rs  sizess        r[   r   r     s    
6{aJvay4-@@ &),,(//!&#A&QJJL&#A2 F G

u:V$ ,
& E*',
 )
 )
* q!Aq1 2 N9 $Bs   C$
c                    U $ r   r   r   s    r[   nopr    s    Hr]   
lift_freshc                   [        U [        5      (       d   eUc)  [        [        R                  " U R                  5      5      $ [        U[
        [        R                  45      (       a)  [        R                  R                  R                  U5      O[        S U 5       5      n[        [        U R                  5       5      U5      n[!        [        U[        5      (       d  U4OU5      n/ n[#        U R                  5       5       H_  u  pEXB;   aD  [        R                  R                  R%                  [        R&                  " US5      SS9(       a  MN  UR)                  U5        Ma     X0R                  5       :w  a  [+        X5      $ U $ )Nc              3  t   #    U  H.  n[         R                  R                  R                  U5      v   M0     g 7fr   )rJ   rS   r  evaluate_static_shaper   ds     r[   r   squeeze.<locals>.<genexpr>  s(     Jc177##99!<<cs   68r(   Tr  )ra   r:   r9   r   rc   r   r   r   rJ   rS   r  r  r   r   r   r   r!   r`   r  r  re   r  )r   r   dims	new_shaper  ss         r[   squeezer    s"   a####
{++AFF344 cC,-- 	
..s3JcJJ 
 C

-s
3CJsE$:$:sfDDI!**,'I  ..uxx1~d.SSQ ( "+jjl!:4AAr]   c                *    [        [        X5      5      $ r   )r  r  )r   r   s     r[   squeeze_copyr    s    !!r]   c                    [        X5      n[        U [        5      (       d   e[        U[        5      (       d   eUR                  U l        U $ r   )r  ra   r:   rc   r   r   vals      r[   squeeze_r    sA    
!/Ca####c9%%%%XXAFHr]   c                    [        U 5      (       a  [        U S[        R                  S9$ [	        S5      n[        U[        R                  S9" U 5      $ )NFr   isinfr  r   	full_likerp   r   r5   ra  r   rs   s     r[   r  r    @    qE44	W	B"EJJ?BBr]   c                    [        U 5      (       a  [        U S[        R                  S9$ [	        S5      n[        U[        R                  S9" U 5      $ )NFr   isnanr  r  r  s     r[   r  r    r  r]   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nceilr   r  r5   ra  r  s     r[   r  r    s0    qQx	V	B"a  r]   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nfloorr  r  s     r[   r  r    0    qQx	W	B"a  r]   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nroundr  r  s     r[   r  r    s0    qQx!b!!$$r]   c                p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Ntruncr  r  s     r[   r  r    r  r]   c                *   [        U /5      u  n [        U [        R                  5      (       a   [        R
                  " U [        U5      5      $ [        U [        5      (       d   e[        U[        [        45      (       d   e[        U R                  5       5      [        U5      :X  a  U $ [        U R                  5       5      (       d  [        R                  R                  R                  [        U R                  5       5      5      nUS:  aT  [        U5      (       dD  U R!                  [        R                  R                  R                  [        U5      5      U-  5        [        [        R
                  " U R"                  [        U5      5      5      $ r  )r8  ra   r+   rU  r0   r   r   r:   r_   r   r    rJ   rS   r  size_hint_or_throwrG   
mark_reuserc   )r   r  x_size_products      r[   r  r  %  s   aS!DQ!R__%%  E%L11a####edE]++++QZZ\eEl* ..))<<!**,'
 A&;E&B&BLL  33M%4HI!" Z&&qvvuU|<==r]   c                    [        U5      nU H  nSX4'   M	     U n[        U5       H  u  pgUS:w  d  M  [        XV5      nM     [        XQ5      $ NrA  )r_   r`   	unsqueezer  )r   shapebroadcast_dimensionsr  broadcast_dimensionr   idxr   s           r[   broadcast_in_dimr  @  sS    UA3!#  4 	
AA,7!!A  !r]   c                4    [        XR                  5       5      $ r   )r  r   )r   r#  s     r[   	expand_asr  N  s    !ZZ\""r]   c                  ^^^ [        U R                  5       5      m[        T5      [        T5      :  aK  [        R                  R
                  /[        T5      [        T5      -
  -  T-   m[        U [        T5      5      n [        T5      [        U R                  5       5      :X  d   e[        U R                  5       5      nSn[        [        T5      5       H  nTU   S:X  a  SnX$   TU   -  X$'   M     U(       a&  [        X R                  5       U R                  5       S9$ [        S [        TT5       5       5      (       a  [        [        X5      5      $ UUU4S jn[        T5      (       d  [        U5      (       d|  [         R"                  R$                  R'                  [)        T5      5      nUS:  aD  U R+                  [         R"                  R$                  R'                  [)        U5      5      U-  5        U R-                  5       m[.        R0                  " U R                  5       U R                  5       U[        U5      S9$ )NFr   Tr   rl   c              3  H   #    U  H  u  pUS :H  =(       d    US :H  v   M     g7fr(   Nr   r  s      r[   r   repeat.<locals>.<genexpr>e  s$     
A*@$!AFa1f*@s    "c                   > [        U 5      [        T5      :X  d   e[        U 5      n [        [        T5      5       HI  nTU   S:w  d  M  TU   S:X  a  [        R                  R
                  X'   M5  [        X   STU   5      X'   MK     T" U 5      $ r  )r   r_   r   r   r  Zeror%   )r-  ri   old_sizerepeatsx_loaders     r[   rP  repeat.<locals>.inner_fnj  s}    5zS\)))Us7|$AqzQA;!#$ww||EH.uxHQKHEH % r]   rR  )r_   r   r   r   r  r  r  r   emptyr   rd   r  r   r  r  r    rJ   rS   r  r  rG   r  rT  r7   r   )	r   r  new_sizezero_tensorri   rP  old_size_productr  r  s	    `     @@r[   repeatr  S  s   AJJL!H
7|c(m#GGKK=CL3x=$@AHLDN#w<3qzz|,,,,AJJL!HK3w< 1:?KkGAJ. !
 X[[]1<<>JJ

A#gx*@
AAAVA())	 !**3H3R3R77++>>}X?VWa LL  33M(4KL#$
 }}H||~kkmH~	 r]   c                    [        U [        5      (       d   e[        U[        [        45      (       d   e[        [        R
                  " U R                  U5      5      $ r   )ra   r:   r_   r   r<   r   rc   )r   r  s     r[   r  r    sH     a####edE]++++T[[/00r]   c                    [        U [        5      (       d   e[        U[        [        45      (       d   e[        [        R
                  " U R                  [        U5      5      5      $ r   )ra   r:   r_   r   r6   r   rc   )r   r  s     r[   permuter"    sL    a####dT5M****[''d<==r]   c                    [        U [        5      (       d   e[        XS5      n[        [        R                  R                  U R                  XX4US95      $ )Nr   clamp)ra   r:   _validate_dimr+   	SliceViewr   rc   )r   r   startendstepr%  s         r[   slice_r+    sH    a####

"CR\\((Se(TUUr]   c                   [        U [        5      (       aC  [        U R                  [        R                  5      (       a  U R                  R                  5       n U R                  5         [        R                  " U 5      (       d  [        SU  S35      e[        R                  " U 5      u  pE[        R                  " UR                  UR                  U Vs/ s H  n[        R                  " U5      PM     snU Vs/ s H  n[        R                  " U5      PM     sn[        R                  " U=(       d    S5      5      n[        [        R                  " XGS95      $ s  snf s  snf )Nzunrealized as_strided(z, ...)r   rc   layout)ra   r:   rc   r+   BaseViewunwrap_viewrk  is_storage_and_layoutr   as_storage_and_layoutFixedLayoutrl   r   r   r  ReinterpretView)r   r   stridestorage_offsetstorage
old_layoutr  
new_layouts           r[   
as_stridedr:    s    !YJqvvr{{$C$CFF IIK##A&&!$:1#V"DEE2215G"&'$Qa$'"()&Qa&)^(q)J R''WHII	 	()s    E; E"c                j    [        U [        5      (       d   e[        XX#5      R                  U l        U $ r   )ra   r:   r:  rc   )r   r   r5  r6  s       r[   as_strided_r<    s.    a####8==AFHr]   c                0    [        XX#5      n[        U5      $ r   )r:  r  )r   r   r5  r6  results        r[   as_strided_copyr?    s    8F=r]   c                  ^ ^^^ / mSnT  H1  nTR                  X"UR                  5       T   -   45        TS   S   nM3     T  Vs/ s H  o3R                  5       PM     snmUU UU4S jn[        T S   R                  5       5      nTS   S   UT'   [        R
                  " T S   R                  5       T S   R                  5       UUS9$ s  snf )Nr   rA  c           	       >^
^ [         R                  " U T   [        R                  5      n/ n/ n[	        [        T5      5       GHH  m
T
S:X  a%  [         R                  " S[        R                  5      O*[         R                  " TT
   S   [        R                  5      n[         R                  " TT
   S   [        R                  5      n[         R                  " X5      n[         R                  " X5      nT
S:X  a  UnO+T
[        T5      S-
  :X  a  UnO[         R                  " Xg5      nUR                  U5        [        U 5      m[        TT   TT
   S   -
  5      TT'   UR                  [         R                  " UU
UU4S jS5      5        GMK     US   n	[	        [        T5      S-
  SS5       H!  m
[         R                  " UT
   UT
   U	5      n	M#     U	$ )Nr   r(   c                    > TT    " T5      $ r   r   )ri   idx_loadinputs_loaderss   r[   <lambda>1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>  s    N1-h7r]           rA  r&   )rI   
index_exprrp   int64r   r   constantgeltand_re   r_   r$   maskedr  )r  idx_dimmasksmasked_loadsr(  r)  
start_condend_condmasknext_valri   rC  r   r5  rD  inputs_rangess             @@r[   rP  pointwise_cat.<locals>.inner_fn  s   ..S5;;7s6{#A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvg+HAvc&kAo%!xx
5LLCyH %Xc]]15Ea5H%HIHSM

75 $D  #Fq("b1AyyaQH 2 r]   rR  )re   r   rT  r_   r7   r   rd   r   )r5  r   prev_endr   rP  r  rD  rV  s   ``    @@r[   pointwise_catrY    s    9;MHh3<<>#3F(FGH $R(  4::6Coo'6:N. .` F1I&&()H!"%b)HSMay##%Qi!!#	 k ;s   C	c           	       ^^^^^^^^	^
 [        TR                  5       5      S:X  d   S5       e[        TR                  5       5      S:X  d   S5       eU R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       eT[        U R                  5       5      :  d!   S[        U R                  5       5       35       eU R                  5       mTR                  5       m	TR                  5       m
UUUUUUU	UU
4	S jn[        R                  " U R                  5       TUU R                  5       S9$ )Nr(   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                h  >	 U T
   4nT" U 5      nT" U5      nT" U5      n[        TT[        R                  S9u  pVTR                  [        R                  :w  a%  [        R
                  " U[        R                  5      nTR                  [        R                  :w  a%  [        R
                  " U[        R                  5      n[        R                  " U5      n[        R                  " X'-  5      U-   n[        R                  " U[        R                  " Xh5      5      n	[        R
                  " U	T5      $ Nr   )_create_constantsrp   float32r   rI   r   r  
reciprocalr  maximumminimum)r  channel_idxrp  scale
zero_pointqminqmax	inv_scaler  clampedaxisr   input_loader	quant_max	quant_minscalesscales_loaderzero_pointszero_points_loaders             r[   rP  ;quantized_decomposed_quantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
&y)5==Q
<<5==(LL6E+j%++>JNN5)	ii)*Z7++dCKK$:;||GU++r]   rR  )r   r   r   rp   rV  r   rb  rT  r7   r   rd   )rp  rq  rs  rm  rp  ro  r   rP  rn  rr  rt  s    `````` @@@r[   )quantized_decomposed_quantize_per_channelrv    s?    v !Q&=(==&{##%&!+G-GG+ENN*.??- 
FuGXFYZ- #enn&'' 
!#enn&6"7!89' $$&L&&(M$002, ," !~~	 r]   )	out_dtypec               ~  ^^^^^	^
^ [        TR                  5       5      S:X  d   S5       e[        TR                  5       5      S:X  d   S5       eU R                  5       U:X  d   SU SU R                  5        35       eT[        U R                  5       5      :  d!   S[        U R                  5       5       35       eTc  [        R                  mU R                  5       m	TR                  5       m
TR                  5       mUU	UUU
UU4S jn[        R                  " U R                  5       TUU R                  5       S9$ )	Nr(   r[  r\  Expecting input to have dtype , but got dtype: r^  c                  > U T   4nT" U 5      nT
" U5      nT" U5      nT	R                   [        R                  :w  a%  [        R                  " U[        R                  5      nTR                   [        R                  :w  a%  [        R                  " U[        R                  5      n[        R
                  " [        R                  " U[        R                  5      U5      U-  n[        R                  " UT5      nU$ r   )r   rp   rb  rI   r   sub)r  rf  rp  rg  rh  r  rm  rn  rw  rq  rr  rs  rt  s         r[   rP  =quantized_decomposed_dequantize_per_channel.<locals>.inner_fnU  s    4ylS!k*'4
<<5==(LL6E-j%--@Jggcll5%--8*EMll3	*
r]   rR  	r   r   r   rp   rb  rT  r7   r   rd   )rp  rq  rs  rm  rp  ro  r   rw  rP  rn  rr  rt  s    ```   ` @@@r[   +quantized_decomposed_dequantize_per_channelr  7  s-    v !Q&=(==&{##%&!+G-GG+??% 
(/@AR@ST% #enn&'' 
!#enn&6"7!89' MM	$$&L&&(M$002  !~~	 r]   c                  ^^^^ U R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       eU R                  5       mUUUU4S jn[        R                  " U R                  5       T[        R                  " U[        U5      [        U5      S9U R                  5       S9$ )Nr]  c                @  > T
" U 5      n[        SU-  U[        R                  S9u  pB[        R                  " X4-  5      U-   n[        TT[        R                  S9u  pg[        R
                  " [        R                  " XV5      U5      n[        R                  " UT	5      $ )N      ?r   )ra  rp   rb  rI   r  re  rd  r   )r  rg  rh  rp  rk  r  ri  rj  rl  r   rn  ro  rp  s            r[   rP  Bquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn  s    S! 1%K5==!
	 ii)*Z7&y)5==Q
++ckk#4d;||GU++r]   rg  rh  rR  )r   rp   rV  r   rb  rT  r7   r   rd   r   r  r'  r   r   )rp  rg  rh  rp  ro  r   rP  rn  s      ``` @r[   0quantized_decomposed_quantize_per_tensor_defaultr  l  s     ENN*.??- 
FuGXFYZ- $$&L, , !""E%LS_
 ~~ r]   c               j  ^^ U R                  5       U:X  d   SU SU R                  5        35       eTc  [        R                  mU R                  5       mUU4S jn[        R
                  " U R                  5       T[        R                  " U[        U5      [        U5      S9U R                  5       S9$ )Nry  rz  c                   > T" U 5      n[        X[        R                  S9u  p[        R                  " [        R
                  " U[        R                  5      U5      U-  n[        R
                  " UT5      nU$ r`  )ra  rp   rb  rI   r|  r   )r  rg  rh  rp  r  rn  rw  s        r[   rP  Dquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fn  sY    S!-eu}}Uggcll5%--8*EMll3	*
r]   r  rR  )r   rp   rb  rT  r7   r   rd   r   r  r'  r   r   )	rp  rg  rh  rp  ro  r   rw  rP  rn  s	         ` @r[   2quantized_decomposed_dequantize_per_tensor_defaultr    s     ??% 
(/@AR@ST% MM	$$&L !""E%LS_
 ~~ r]   c                >  ^^^^^^^^	 U R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       eU R                  5       mTR                  5       mTR                  5       m	UUUUUUUU	4S jn[        R                  " U R                  5       TUU R                  5       S9$ )Nr]  r   r(   expect scale as scalar tensor"expect zero_point as scalar tensorc                  > T	" U 5      nT" [        TR                  5       5      S:X  a  SOS5      nT" [        TR                  5       5      S:X  a  SOS5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      n[
        R                  " U[
        R                  " U5      -  5      U-   n[        TT
[        R                  S9u  pV[
        R                  " [
        R                  " XE5      U5      n[
        R                  " UT5      $ )Nr(   r   r   r   )r   r   r   rp   rb  rI   r   r  rc  ra  re  rd  )r  rp  _scale_zero_pointr  ri  rj  rl  r   rn  ro  rp  rg  scale_loaderrh  zero_point_loaders           r[   rP  Aquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKiiv 667+E&y)5==Q
++ckk#4d;||GU++r]   rR  )r   rp   rV  r   rb  r   r   rT  r7   r   rd   )
rp  rg  rh  rp  ro  r   rP  rn  r  r  s
    ````` @@@r[   /quantized_decomposed_quantize_per_tensor_tensorr    sf    ENN*.??- 
FuGXFYZ- u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  $$&L$$&L"..0, , !~~	 r]   c                 ^^^^^	^
 [        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       eU R                  5       U:X  d   SU SU R                  5        35       eTc  [        R                  mU R                  5       mTR                  5       m	TR                  5       m
UUUU	UU
4S jn[        R                  " U R                  5       TUU R                  5       S9$ )	Nr   r(   r  r  ry  rz  c                h  > T" U 5      nT" [        TR                  5       5      S:X  a  SOS5      nT
" [        TR                  5       5      S:X  a  SOS5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nT	R                  [        R                  :w  a%  [
        R                  " U[        R                  5      n[
        R                  " [
        R                  " U[        R                  5      U5      U-  n[
        R                  " UT5      nU$ )Nr(   r  r   )r   r   r   rp   rb  rI   r   r|  )r  rp  r  r  r  rn  rw  rg  r  rh  r  s        r[   rP  Cquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKggcll5%--8+FOll3	*
r]   rR  r~  )rp  rg  rh  rp  ro  r   rw  rP  rn  r  r  s    ``   ` @@@r[   1quantized_decomposed_dequantize_per_tensor_tensorr    sU    u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  ??% 
(/@AR@ST% MM	$$&L$$&L"..0
 
 !~~	 r]   c                |  ^^^^^^^ U S   R                  5       R                  S:H  nU(       a  [        S U  5       5      (       au  U  H  nUR                  5         M     [        S U  5       5      (       a  [	        [
        R                  /U Q76 u  p[        [
        R                  R                  5      " X5      $ [        U 5      S:X  a  [        U S   5      $ [        U S   US5      n[        U S[        R                  06nU  Vs/ s H  n[        Xe5      PM     n nSS jmS mUUU4S	 jm[!        U4S
 jU  5       5      nSUU4S jjm["        R$                  (       a  ['        X5      $ U(       a(  [)        [*        R,                  R/                  X5      5      $ UU4S jmSnSmSS jm[        U 5      U::  d8  [        U 5      ["        R0                  ::  a  [        UU4S jU  5       5      (       a  [        U4S j[2        R4                  R6                   5       5      n	[!        U4S jU  5       5      =(       a    U	n
[        U4S jU  5       5      =(       a    [!        U4S jU  5       5      (       + nU
(       d  U(       a  U(       d  ['        X5      $ [)        [*        R,                  R/                  X5      5      $ s  snf )Nr   cpuc              3     #    U  H5  oR                  5       [        R                  [        R                  4;   v   M7     g 7fr   )r   rp   int8uint8r   rp  s     r[   r   cat.<locals>.<genexpr>  s*      DJ5ejj%++66Fs   =?c              3  Z   #    U  H!  n[        UR                  5       5      S :H  v   M#     g7f)r   N)r   r   r  s     r[   r   r  &  s!     >ves5>>#$)vs   )+r(   r   c                $   [        U [        5      (       aO  [        U R                  [        R                  5      (       a  U R                  R                  5       $ U R                  $ [        U [        R                  5      (       a  U R                  $ U $ r   )ra   r:   rc   r+   r/  r0  
StorageBoxr   s    r[   unwrap_tensorcat.<locals>.unwrap_tensor3  s_    a##!&&"++..vv))++vva''66Mr]   c                    [        U [        R                  5      =(       a$    [        U R                  [        R                  5      $ r   )ra   r+   ComputedBufferrc   r8   )rm   s    r[   is_reductioncat.<locals>.is_reduction?  s)    !R../TJqvvr||4TTr]   c                  > [        U [        [        R                  45      (       a  T" T" U 5      5      $ T" U 5      =(       dD    [        U [        R                  5      =(       a#    [        U4S jU R                  5        5       5      $ )Nc              3  n   >#    U  H*  nT" [         R                  R                  U5      5      v   M,     g 7fr   )rJ   rS   
get_buffer)r   readcan_fuse_reductions     r[   r   2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>H  s/      .D #177#5#5d#;<<.s   25)ra   r:   r+   r  r7   r	  get_read_names)rm   r  r  r  s    r[   r  cat.<locals>.can_fuse_reductionB  sk    a)R]]344%mA&677O !R\\*  ,,. 	
r]   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r   rm   r  s     r[   r   r  O  s     B6a.q116   c                X  > [         R                  " U 5      (       a;  [         R                  " U SS9u  p[         R                  R	                  U5      (       + $ [        U [        [         R                  45      (       a  T" T" U 5      5      $ [        U [         R                  5      (       a  gg)NF)freezeT)	r+   r1  r2  ConcatKernelcan_realize_into_without_copyra   r:   r  r7   )r   r7  r   should_lower_cat_inputr  s      r[   r  #cat.<locals>.should_lower_cat_inputQ  s     ##A&&11!EBJGDDWMMMa)R]]344)-*:;;a&&r]   c                `  > [        U [        [        R                  45      (       a  T" T" U 5      5      $ [        U [        R                  5      (       d  gU R                  5       R                  nU R                  5        H+  nUT" [        R                  R                  U5      5      -  nM-     U$ r  )ra   r:   r+   r  r7   inner_fn_opcountnum_opsr  rJ   rS   r  )r   countr  op_countr  s      r[   r  cat.<locals>.op_counth  s    a)R]]344M!,-- !R\\**""$,,$$&DXagg00677E ' r]   r   r&   c                p    U [         R                  R                  [         R                  R                  4;   $ r   )r  catr  constant_pad_ndrV   s    r[   additional_pointwise_ops%cat.<locals>.additional_pointwise_ops}  s(    dhh&&(<(<(D(DEEEr]   c              3  :   >#    U  H  nT" U5      T:*  v   M     g 7fr   r   )r   rm   MAX_SIMPLE_OP_COUNTr  s     r[   r   r    s     CFq22F   c              3  <   >#    U  H  n[        UT5      v   M     g 7fr   )rA   )r   user  s     r[   r   r    s#      
+ S":;;+r   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r   r   r  s     r[   r   r    s     >v&s++vr  c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r  s     r[   r   r    s      "
39C"3''6r  c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   r  s     r[   r   r    s     <V(++Vr  )r   zUnion[TensorBox, ir.StorageBox]return	ir.IRNode)r  r   rV   torch._ops.OpOverload)rd   rZ  r  rk  require_channels_lastr  r  r  r  r   r  r&  r   r   r2  r   r	  r)   force_pointwise_catrY  r:   r+   r  r   max_pointwise_cat_inputsrJ   rT   rU   )r5  r   
cpu_devicerp  r   r   r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_usehorizontal_fuse_catr  r  r  r  r  r  r  s               @@@@@@@r[   r  r    sD   %%',,5Jc DJ  
 EMMO >v>>>-dhh@@IF 0 01&>>
6{aVAY
q	3
*C	%D%L%LE /55fshs"fF5
U

 B6BB  !!V)) //<==$ !"F 6{//	V77	7CFCCC 
~~++
 
 >v>>Q> 	 " "
39"
 
 =<V<<< 	 "5>O --R__++F899K 6s   J9c                  ^^^
^^ U R                  5       m[        T5      m[        TTS9m[        TTS9m[        TT:g  UU4S j5        [        R
                  R                  R                  [        R                  " US5      5      nU(       a\  [        R
                  R                  R                  [        R
                  R                  R                  TT   U-   TT   5      S5      nO[[        R
                  R                  R                  [        R
                  R                  R                  TT   TT   U-
  5      S5      nSm
U(       a  U* S4m
OSU4m
[        T5       VVs/ s H  u  pgUTT4;  d  M  UPM     nnnUR                  U5        U
UUUU4S jn	[        [        R                   R#                  XU	5      5      $ s  snnf )N)r  rankc                    > ST  ST 3$ )Nz(diagonal dimensions cannot be identical z, r   )dim1dim2s   r[   rE  diagonal.<locals>.<lambda>  s     HbQUPVWr]   r   )r   r   c                   > U S   nS/[        T	5      -  nSn[        T5       H2  nUT:X  a  UTS   -   X$'   M  UT:X  a  UTS   -   X$'   M'  X   X$'   US-  nM4     U[        T	5      S-
  :X  d   eU$ )NrA  r   r(   r&   )r   r   )
r  diag_idxoriginal_idxcur_dimr  base_idxr  r  num_dimsoriginal_shapes
        r[   	reindexerdiagonal.<locals>.reindexer  s    r7sS00xADy"*Xa["8d"*Xa["8"%,1 ! #n-1111r]   )r   r   r   r   rJ   rS   r  r  r   Ltevaluate_maxevaluate_minr`   re   r:   r+   GenericViewr   )rp  offsetr  r  offset_negative	diag_sizeri   r  r  r  r  r  r  s     ``      @@@r[   diagonalr    s   ^^%N>"H84D84D	W gg&&44UXXfa5HIOGG$$11GG))t$v-~d/C 	
	 GG$$11GG))t$nT&:V&C 	
	 HGQ<v;$^4N441$8MQ4EN	LL   R^^**5CDD' Os   &G8Gc                ,    [        [        XX#5      5      $ r   )r  r  )rp  r  r  r  s       r[   diagonal_copyr    s    %455r]   c                J    [        U 5      n[        XRX45      n[        Xa5        U$ r   )r  r  	mutate_to)rp  srcr  r  r  r"  rW   s          r[   diagonal_scatterr    s$    5\Ffd1FfMr]   c           	         [         R                  " X R                  5       U   5      n[        [	        XX"S-   5      U5      $ r  )r<   handle_negative_indexr   r  r+  )r   r   r  s      r[   selectr    s5    

$
$S**,s*;
<C6!#Qw/55r]   c                n   [        XS5      nUn[        U[        [        45      (       d^  U R	                  5       U   n[
        R                  R                  R                  [        XA-   S-
  U5      5      nU/U-  nXES-
  U-  -
  US'   / nSnU H#  nXx-   n	UR                  [        XXySS95        U	nM%     U$ )Nr   r(   rA  Fr$  )r&  ra   r_   r   r   rJ   rS   r  r  r#   re   r+  )
r   r  r   sizes_x_sizechunksr>  r(  r   r)  s
             r[   splitr     s    

"CF edE]++c"!!77V^a'/
 6!zU22r
 FEl 	fQUu=>  Mr]   c                    [        XU5      $ r   )r   )r   r  r   s      r[   split_with_sizesr     s    3r]   c                    [        XS5      n[        R                  R                  R	                  U R                  5       U   5      n[        U5       Vs/ s H  n[        XU5      PM     nnU$ s  snf r  )r&  rJ   rS   r  r  r   r   r  )r   r   r  ri   r>  s        r[   unbindr    s\    

"CWW33AJJL4EFF).v7AfQQF7M 8s   A.c                4  ^^ U R                  5       n[        U5      n[        XQ5      mUS:X  a  [        [	        U S5      US9$ UT   n[
        R                  R                  nUR                  X&5        UR                  ST5        [        Xb-
  T5      S-   nUR                  U5      S:  a,  U R                  UR                  [        X-  U5      5      5        / US T QUPUTS-   S  QUPn	UU4S jn
[        [        R                   R#                  X	U
5      5      $ )Nr   )r)  r(   c                D   > U S   U T   T-  -   n/ U S T QUPU TS-   S Q7$ )NrA  r(   r   )r  dim_idxr   r*  s     r[   r  unfold.<locals>.reindexer#  s?    b'CHtO+8Tc8G8c#'B&788r]   )r   r   r   r+  r  rJ   rS   r  	guard_leqguard_ltr#   r  r  r"   r:   r+   r  r   )r   	dimensionr   r*  r  ndimdim_sizer  new_dim_sizeout_sizer  r   s      `       @r[   unfoldr    s   JJLEu:D
4
+Cqyi1o400SzHwwHt&aHOT2Q6L""8,q0	''0CX(NO	
 EtDlDU379-=DtDH9 R^^**1	BCCr]   c                    [        XS5      n[        U R                  5       5      nUR                  U[        R
                  R                  5        [        X5      $ r  )r&  r_   r   insertr   r  r  r  )r   r   r  s      r[   r  r  *  sA    

"CQZZ\"IS%''++&r]   c                    [        X5      n[        U [        5      (       d   e[        U[        5      (       d   eUR                  U l        U $ r   )r  ra   r:   rc   r  s      r[   
unsqueeze_r  2  sB    
A
Ca####c9%%%%XXAFHr]   c                   [         R                  R                  R                  R	                  [
        R                  " U5      5      n[        U R                  5       5      nUS:  a  XU-   -  nSUs=::  a	  X2-   :  d   e   eU$ r  )	rJ   rS   r  r  r  r   sympifyr   r   )r   r   r  r  s       r[   r&  r&  ;  sn    
''


$
$
2
25==3E
FCqzz|D
Qwf}#dm#####Jr]   rA  c                    [        XS5      n[        R                  R                  R	                  U R                  5       U   5      S-  n[        XSU5      n[        XX"S-  5      n[        U[        U5      5      $ )Nr   r&   )	r&  rJ   rS   r  r  r   r+  rS  sigmoid)r   r   new_lenr   r!  s        r[   glur  D  sg    

"Cgg44QZZ\#5FG1LGqq'"Aqw!,Aq'!*r]   c                Z   ^  U(       a  [         R                  T 5        U 4S jnSUl        U$ )Nc                 |   > S n[         R                  " U[        R                  R                  " T/U Q70 UD65      $ )Nc                p    [        U [        R                  5      (       a  [        R                  " U 5      $ U $ r   )ra   r+   r2   r:   r   r   s    r[   wrap_tensors7fallback_handler.<locals>.handler.<locals>.wrap_tensorsS  s(    *4Q		*B*B9##A&IIr]   )pytreetree_mapr+   FallbackKernelr   )rj   r   r  kernels      r[   handler!fallback_handler.<locals>.handlerR  s<    	J "++226KDKFK
 	
r]   T)r  r   _is_fallback_handler)r#  r  r$  s   `  r[   r  r  N  s&    f
 $(G Nr]   c                 0    [         R                  " S5        g )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   r]   r[   _warn_complex_not_supportedr*  `  s    MMtr]   c                   U R                  5       (       a  [        5         gU R                  (       a  gU R                  [        R
                  :X  Ga  U(       d  g[        UR                  [        R                  R                  5      =(       as    UR                  [        R                  R                  [        R                  R                  [        R                  R                  [        R                  R                  4;   =(       dJ    [        UR                  [        R                  R                  5      =(       a    [!        UR                  5      (       + $ g)z0Do not support reading or writing to this tensorTF)r  r*  is_metar   rp   float8_e8m0fnura   rW   rq   rr   r  r  r  r  r  
_scaled_mmrB   )rm   rY   s     r[   unsupported_input_tensorr/  i  s    ||~~#%yyww%&&&
 t{{EJJ$9$9: 		  

""''	Y 4;;

(=(=>W74;;CW

 
	
 r]   c                L   [         R                  R                  [        R                  R
                  R                  R                  4nUb&  UR                  U;   a  U R                  5       (       a  g[        X5      (       a  gU R                  =(       a    [        R                  $ )z2Do not support writing tensor but can read from itFT)r  r  r   rp   rI   r  r  r  rW   r  r/  is_cpur)   disable_cpp_codegen)rm   rY   supported_complex_viewss      r[   unsupported_output_tensorr4    sr     					,,44 DKK+BBq||~~((882222r]   c                ^  ^  T R                   [        R                  R                  L a  gT R                  S:X  a  gT R                   [        R
                  R                  L a  gU 4S jn[        R                  " T R                  0 T R                  D6 H  nU" USS9(       d  M    g   U" T SS9$ )NFplaceholderc                  > [        U [        R                  R                  5      (       d  gSU R                  ;  a  g[
        R                  " U R                  S   5       H]  n[        U[        R                  R                  5      (       d  M.  U(       a  [        UT5      (       a    gMJ  [        UT5      (       d  M]    g   g)NFr  T)ra   rp   fxNoderX  r   tree_leaves_subclasses
FakeTensorr4  r/  )inp_out_node	is_outputrX  rY   s      r[   check_skip_conditionCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition  s    ,66)))&&|'8'8'?@DdE$5$5$@$@AA,T488 9 ,D$77 A r]   )r>  T)
rW   r  view_as_complexr  rV   lift_fresh_copyr   arg_tree_leavesrj   r   )rY   allow_cpu_inputsr?  r   s   `   r[   %fallback_node_due_to_unsupported_typerE    s    {{d**222ww- {{d**222* %%tyy@DKK@u55 A  55r]   c                  ^ U [         ;  d  U(       d
   SU  35       eU(       a  [        [        R                  " S5      5      (       a  [	        U /5      (       a  [
        R                  (       a(  U [        R                  R                  R                  ;   ds  U(       dl  [        R                  R
                  R                  (       a4  S[        R                  R
                  l        [        R                  S5        [        SU  S35      eU4S jn[!        U [        R"                  R$                  5      (       a+  U R'                  5        H  n[)        X5      nU" U5        M     g [!        U [        R"                  R*                  [        R"                  R,                  45      (       a	  U" U 5        g [/        SU  S	[1        U 5       35      e)
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                j   > [        U 5        Tb  [        U T5        [        U S S9" [        U 5      5      $ Nr  )r   r   r  r  )op_overloadlayout_constraints    r[   register_fallback(make_fallback.<locals>.register_fallback  s8    !+.(!+/@A $G[)
 	
r]   zUnsupported fallback z with type )r-   r   osgetenvr.   r)   fallback_randomrp   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningr   ra   rq   r   r   r   rr   HigherOrderOperatorRuntimeErrorrZ  )rV   rK  r)  override_decomprL  olrJ  s    `     r[   make_fallbackr\    sV   ^# 
4RD96 	4!!t$$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj1122,,.B!"/Kk* ! 
B..

0N0NO	P	P"22$k$r(LMMr]   c                P    SnU  H  nX-  nM	     [        U[        R                  S9$ )z
TorchInductor offset calculation differs from PyTorch eager offset
calculation for random ops (tl.rand vs torch.rand). In future, we should
strive for same impl for tl.rand and torch.rand.
r(   r   tensorrp   rI  )r	  numelr  s      r[   philox_rand_offsetra    s+     E	 %u{{++r]   c                @  ^^	^
^ [         R                  " UTU [         R                  R                  U 5      5      R	                  5       m
UR                  5       mUR                  5       m	UU	U
U4S jn[        R                  " UTU[        U 5      S9n[        U 5      nXx4$ )Nc                  > [         R                  " T" / 5      [        R                  5      n[         R                  " T" / 5      [        R                  5      n[         R                  " [         R
                  " T" U 5      [        R                  5      U5      n[         R                  " UU5      n[         R                  " UT5      $ r   )rI   r   rp   r  r   rH  rand)	r-  seed_index_exproffset_index_exprrand_index_exprr>  r   offset_loader
random_posseed_loaders	        r[   rP  philox_rand.<locals>.inner_fn  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**r]   rR  )
r+   r3  FlexibleLayoutcontiguous_stridesmake_indexerrT  r7   r   r_   ra  )r   seedr  r5  rl   r   rP  random_values_nodeoffset_noderh  ri  rj  s        `   @@@r[   philox_randrr    s     
,,T2	
 ln  ""$K&&(M+ + #))Dz	 %T*K**r]   c           	         [         R                  (       a\  [        R                  " [        R
                  [        R                  R                  [        R                  R                  XU5      5      $ [        S5      e)Nz&should be handled in replace_random.py)r)   rP  r   r!  r:   r   r+   r"  r  native_dropoutr  r   )r   ptrains      r[   rt  rt  "  sW    $$T%8%8%@%@!N
 	

 EFFr]   c                   [         R                  (       d/  U R                  5       [        R                  " S5      :X  d   S5       eU R                  5         [        U5      S:X  d  [        US   [        5      (       a  [        R                  R                  O[        R                  R                  n[        R                  " X /UQ76   U $ )Nr  Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r)   rP  rd   rp   rl   rk  r   ra   r'  r  
bernoulli_Tensorr+   InplaceBernoulliFallback)r   rj   rJ  s      r[   ry  ry  -  s    !!Q\\^u||E7J%J ^J IIK t9>ZQ77 	__## 
 66Hr]   c                    [         R                  (       d/  U R                  5       [        R                  " S5      :X  d   S5       e[        [        U 5      /UQ76 $ )Nr  rx  )r)   rP  rd   rp   rl   ry  r  )r   rj   s     r[   bernoulli_pr}  <  sG    !!Q\\^u||E7J%J ^J eAh&&&r]   c                    [         er   r   )r   s    r[   _foobarr  E  s    
r]   c                .    [         R                  S5        g )Nz1using triton random, expect difference from eager)rV  info)salts    r[   _warn_triton_randomr  J  s    HH@Ar]   c                 J    [        [        R                  R                  5        g r   )r  rJ   rS   creation_timer   r]   r[   warn_triton_randomr  O  s    --.r]   c                     UR                  SS 5      b  [        U 0 UD6$ [        R                  (       a  UR	                  SS 5        [        U 0 UD6$ [        S5      eN	generatorz-should have been handled in replace_random.py)rY  fallback_rand_generatorr)   rP  popfallback_rand_defaultr   rj   r   s     r[   rd  rd  [  sV    zz+t$0&777			

;%$d5f55
H
IIr]   c                     UR                  SS 5      b  [        U 0 UD6$ [        R                  (       a  UR	                  SS 5        [        U 0 UD6$ [        S5      er  )rY  fallback_randn_generatorr)   rP  r  fallback_randn_defaultr   r  s     r[   randnr  e  sV    zz+t$0'888			

;%%t6v66
H
IIr]   c                l    [         R                  " U5      n[         R                  R                  X5      $ r   )r+   get_stride_orderExternKernelrequire_stride_order)input_tensorr5  stride_orders      r[   inductor_force_stride_orderr  o  s'    &&v.L??//KKr]   c                    [        S5      e)Nz.should be handled in fuse_seed_creation_pass()r  )rl   s    r[   inductor_seedr  u  s    
I
JJr]   c                ~    [        5         [        R                  " [        R                  " U [        U5      5      5      $ r   )r  r:   r   r+   RandomSeedsr>   )r  rl   s     r[   inductor_seedsr  z  s)    BNN5-2GHIIr]   c                z   ^ ^ UU 4S jn[         R                  " T R                  5       T R                  5       U/ S9$ )Nc                N   > [         R                  " TR                  5       T5      $ r   )rI   	load_seedget_name)r   r-  seedss    r[   rP  &inductor_lookup_seed.<locals>.inner_fn  s    }}U^^-u55r]   rR  )r7   r   rd   r   )r  r-  rP  s   `` r[   inductor_lookup_seedr    s9    6 !oo	 r]   r  c                 ^^^	 [         R                  (       a   eTS;   d   e/ U Qn [        R                  nUR	                  5       n[
        R                  " XTU [
        R                  R                  U 5      US9R                  5       mUR                  5       m	UUU	4S jn[        R                  " UUU/ U QS9nUR                  5         U$ )N)rd  r  r  c                   > [        [        T5      " T" / 5      [        R                  " T" U 5      [        R                  5      5      $ r   )r   rI   rH  rp   r  )r-  moderi  rj  s    r[   rP  !inductor_random.<locals>.inner_fn  s5    sD!ONN:e,ekk:
 	
r]   rR  )r)   rP  rp   rb  r4  r+   r3  rl  rm  rn  rT  r7   r   rk  )
r   ro  r  r  r   rl   rP  r>  ri  rj  s
     `     @@r[   inductor_randomr    s    %%%%$$$$T7DMME%%'FtR..AA$GPVln  ""$K
 w	F NNMr]   c               h  ^ ^^^	 [         R                  (       a   e/ UQn[        R                  nUR	                  5       n[
        R                  " XeU[
        R                  R                  U5      US9R                  5       mUR                  5       m	UU UU	4S jn[        R                  " UUU/ UQS9$ )Nr  c           	        > [         R                  " T" / 5      [         R                  " T" U 5      [        R                  5      [         R                  " T[        R
                  5      [         R                  " T[        R
                  5      5      $ r   )rI   	randint64rH  rp   r  rI  )r-  highlowri  rj  s    r[   rP  "inductor_randint.<locals>.inner_fn  sV    }}ONN:e,ekk:NN3,NN4-	
 	
r]   rR  )r)   rP  rp   rI  r4  r+   r3  rl  rm  rn  rT  r7   r   )
r  r  r   ro  r  r   rl   rP  ri  rj  s
   ``      @@r[   inductor_randintr    s     %%%%T7DKKE%%'FtR..AA$GPVln  ""$K
 
 w	 r]   c                    U R                  5       U R                  5       S   U R                  5       S   U R                  5       S   -  U R                  5       S   4$ NrA  r   )r  r   
get_stridetbs    r[   _boundaries_helperr    sO    

b
a2==?1--
	 r]   c                H    U R                  5       U R                  5       S   4$ r  )r  r  r  s    r[   _sorter_helperr    s    ;;="--/"---r]   	out_int32rightsidesorterc          	     p  ^ ^^^
^ S nU" T 5      (       a  U" U5      (       a  Tb8  U" T5      (       d+  [        [        R                  R                  SS9" T UUTUTS9$ Ub  US:X  a  SmU(       a  [        R
                  O[        R                  m
UR                  5       mT R                  5         Tb  TR                  5         [        T R                  5       5      S:X  a  U
UU UU4S jnO
U
UU UU4S	 jnUR                  5       n[        R                  " UT
UUR                  S
9n	U	R                  5         U	$ )Nc                ^    [         R                  R                  U [        R                  5      $ r   )rJ   rS   rh  ri  	BUCKETIZEr  s    r[   rE  searchsorted.<locals>.<lambda>  s    AGG$7$7
N$$%r]   Fr  r  r  Tr(   c           
        > T" U 5      n[         R                  " U[        T5      STTTc  S O
[        T5      Tc  S S9$ SS9$ )Nr   r  sorter_indicesrI   	bucketizer  r  )r  r  index_dtyper  sorted_sequencer  values_loaders     r[   rP  searchsorted.<locals>.inner_fn	  sV    $C=="?3%~t>&3I'-~t  <= r]   c                   >^  T" T 5      nSU U4S jjn[         R                  " U[        T5      U" T5      TTTc  S O
[        T5      Tc  S S9$ U" T5      S9$ )Nc                   > U R                  5       n[        R                  " [        R                  " [
        R                  S [        US S TS S 5       5       5      T5      $ )Nc              3  .   #    U  H  u  pX-  v   M     g 7fr   r   )r   r  ri   s      r[   r   Nsearchsorted.<locals>.inner_fn.<locals>.get_flattened_index.<locals>.<genexpr>	  s     &U9Tqu9T   rA  )r  rI   rH  r   r  operatorr   r   )r  stridesr  r  s     r[   get_flattened_index;searchsorted.<locals>.inner_fn.<locals>.get_flattened_index	  sU    --/~~$$ &UWSb\3sPR89T&U  	 r]   r  )r  r:   r  )r  r  r  r  r  r  r  r  s   `  r[   rP  r  	  sm    $C  =="?3#O4%~t>&3I'-~t  <Ov;V r]   rR  )r  r  searchsortedrz  rp   r  rI  rT  rk  r   r   rd   r7   r   r	  )r  selfr  r  r  r  validate_bucketizerP  rl   r>  r  r  s   `  ` `    @@r[   r  r    s    //!$'''9&'A'A 1 1 8 8eT
 	
 DGO!*%++K$$&M 
?##%&!+
	 
		 	0 __Fzz	F NNMr]   r  r  c                 ^^^^ [        TR                  5       5      S:X  d   e[        R                  R	                  U [
        R                  5      (       a3  [        R                  R	                  T[
        R                  5      (       d)  [        [        R                  R                  SS9" U TUTS9$ TR                  5         U R                  5       nU R                  5       mU(       a  [        R                  O[        R                   mUUUU4S jn["        R$                  " UTUU R                  5       S9nUR                  5         U$ )Nr(   Fr  r  c                ^   > T" U 5      n[         R                  " U[        T5      STT5      nU$ r  )rI   r  r  )r-  r  r  
boundariesr  rn  r  s      r[   rP  bucketize.<locals>.inner_fnS	  s6    5!--z*
 r]   rR  )r   r   rJ   rS   rh  ri  r  r  r  r  rz  rk  rd   rT  rp   r  rI  r7   r   )	rp  r  r  r  rl   rP  r>  r  rn  s	    ` `   @@r[   r  r  5	  s     z""$%*** 	
E>#;#;<<GG
N,D,DEE 5 55Q:%
 	
 F$$&L!*%++K
 
 ~~	F NNMr]   c                    [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ r   )r   tree_map_onlyr+   r2   r  require_stride1r   rj   r   s      r[   require_denser  r	  s5    ''
		2??22TNLD <r]   c                    [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ r   )r   r  r+   r2   r  require_contiguousr  s      r[   r  r  y	  s5    ''
		2??55~LD <r]   c                    [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ r   )r   r  r+   r2   r  r{   r  s      r[   r{   r{   	  s7     ''
		2??==~LD <r]   c                    [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ r   )r   r  r+   r2   r  r  r  s      r[   r  r  	  s5    ''
		2??884.LD <r]   c           	     @   [        U [        R                  5      (       at  UR                  5        Vs/ s H:  n[        U[        R
                  5      (       a  UR                  R                  OUPM<     nn[        R                  R                  X5      $ [        U [        5      (       a/  U R                  5        Vs0 s H  oD[        X   X   5      _M     sn$ [        U [        [        45      (       a!  [        U 5      " S [!        X5       5       5      $ U $ s  snf s  snf )Nc              3  <   #    U  H  u  p[        X5      v   M     g 7fr   constrain_to_fake_tensor)r   r   f_as      r[   r   +constrain_to_fake_tensor.<locals>.<genexpr>	  s      
=O!$Q,,=O   )ra   r+   r2   r5  rp   SymIntrY   exprr  require_exact_stridesr   r
  r  r   r_   rZ  r   )r   fake_argr  meta_stride_exprkeys        r[   r  r  	  s    #ryy!!GOGX
GX!:a66AFFKKA=GX 	 
 44SKK#tNQhhj
NXs)#(HMBBj
 	
 
C%	'	'Cy 
=@=O
 
 	
 J


s   AD;Dc           
         [        S [        X5       5       5      n UR                  5        VVs0 s H  u  pEU[        XSU   5      _M     nnnX4$ s  snnf )Nc              3  <   #    U  H  u  p[        X5      v   M     g 7fr   r  )r   r   r  s      r[   r   ,constrain_to_fake_tensors.<locals>.<genexpr>	  s!      1MC 	!//1r  )r   r   r   r  )rj   r   	fake_argsfake_kwargsr   r   s         r[   ry   ry   	  s[      1 D JPXa)!^<<FX< Ys   Ac           
        ^ U4S jm[        U4S j[        XR                  5       5       5      nUR                  5        VVs0 s H  u  p4UT" X@R                  U   5      _M     nnnX4$ s  snnf )Nc           	       > [        U [        R                  5      (       as  [        R                  " UR                  S   R                  5       [        R                  R                  R                  5      n[        R                  R                  X5      $ [        U [        5      (       a,  U R                  5        Vs0 s H  o3T" X   X   5      _M     sn$ U $ s  snf Nr  )ra   r+   r2   r  rX  r5  rJ   rS   r  r  r  r  r   r
  )r   fx_argr  r  apply_constraints       r[   r  1constrain_to_fx_strides.<locals>.apply_constraint	  s    c299%%..E"))+QWW-=-=-G-GL ??77JJc4  LOHHJWJS)#(FK@@JWW
 Xs   ;Cc              3  8   >#    U  H  u  pT" X5      v   M     g 7fr   r   )r   r   r  r  s      r[   r   *constrain_to_fx_strides.<locals>.<genexpr>	  s       9P+#%%9Ps   )r   r   rj   r   r   fx_noderj   r   r   r   r  s        @r[   r}   r}   	  sl      9<T<<9P D EKLLNSNDAa!!^^A%677NFS< Ts   "A+c                   ^ ^ U 4S jm[        U4S j[        [        UT R                  5      5       5       5      nUR	                  5        VVs0 s H  u  p4UT" SUT R
                  U   5      _M     nnnX4$ s  snnf )Nc           
       >^ [        U[        R                  5      (       d  U$ UR                  S   nUR	                  5        Vs/ s H:  n[        U[
        R                  5      (       a  UR                  R                  OUPM<     nn[        R                  " UR	                  5       5      nU(       a=  US   S:w  a4  [        [        [        [        UR                  5       5      5      5      5      nTR                  [         R"                  R$                  :X  a  U S;   a  [        U5      S:X  d   eSnUR&                  (       d  [        R(                  R+                  X5      $ SmTR                  [
        R,                  R                   R.                  R$                  :H  =(       a    U S:H  n[        U[0        5      (       d   e[        UR                  5       5      S	;  a  U$ [        R2                  " UT5      (       a4  [        R4                  " [        R(                  R7                  U5      U5      $ [        U[        5      (       aa  UR9                  5       bP  [        R2                  " UT5      (       a4  [        R4                  " [        R(                  R7                  U5      U5      $ U(       Ga  [        UR                  5       5      n/ n	UR9                  5       n
[        [        UR                  5       5      S
-
  5       H}  n[:        R<                  R>                  RA                  X[   S5      (       d8  U
c  M9  [:        R<                  R>                  RA                  X   S5      (       d  Ml  U	RC                  U5        M     S/[        U5      -  nS
US'   S
n[        [        U5      S-
  SS5       H  nXS
-      S:w  a
  XUS
-      -  nX;   a=  [:        R<                  R>                  RA                  XS
-      T-  S5      (       a  SX'   MZ  [:        R<                  R>                  RA                  UT-  S5      (       d  [E        UT5      T-  nXU'   M     [        R(                  RG                  X5      $ [        R2                  " UT5      (       a4  [        R4                  " [        R(                  R7                  U5      U5      $ [        U[        5      (       aa  UR9                  5       bP  [        R2                  " UT5      (       a4  [        R4                  " [        R(                  R7                  U5      U5      $ U4S jn[        URH                  [        RJ                  5      (       a\  U" U5      (       dO  U" URM                  5       5      (       a4  [        R4                  " [        R(                  R7                  U5      U5      $ [        R(                  R+                  X5      $ s  snf )Nr  rA  r   )r   r   r   )r   r(   r&   r   r   r   r   r   r(   r&   c                   > [         R                  R                  R                  U R	                  5       S   5      T-  S:H  $ r  )rJ   rS   r  	size_hintr   )r   	ALIGNMENTs    r[   
is_aligned=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned.
  s4    GG$$..qzz|B/?@9LQRRRr]   )'ra   r+   r2   rX  r5  rp   r  rY   r  r  r_   r  r   r   r   rW   r  0_scaled_dot_product_efficient_attention_backwardr  is_cudar  r  rI   '_scaled_dot_product_efficient_attentionr:   is_aligned_realized_tensortry_match_insignificant_stridesrealize_inputmaybe_get_striderJ   rS   r  statically_known_equalsre   r=   r  rc   r/  r0  )r  r   r  meta_valr  r  r  effn_attn_fwd_biasr  expanded_dimsmaybe_strideri   out_stridesr5  r	  r  r  s                  @r[   r  )sdpa_constraint.<locals>.apply_constraint	  s   #ryy))J;;u%GOGX
GX!:a66AFFKKA=GX 	 
 **8??+<=L,1s3<<>/B)C DEL NNDDLLMv|$)))
 (L??77JJ 	 NNyy~~EEMMN q 	 #y))))s||~f,J((i8855--c24D 
 sF##$$&2--c9==55--c24D  CLLN+HM //1L3s||~.2377##;;<L<OQRSS ,((@@RSTT!((+ 4 $X.KKOF3x=1,b"51u%*#q1uo5F %ww''??#E*Y6  *+ ww''??@RTUVV$VY7)CF!'A! 6$ ??88JJ((i8855--c24D 
 sF##$$&2--c9==55--c24D 	S chh,,c??coo/00==55c:<L  33CFFi
s   AW c              3  @   >#    U  H  u  nu  p#T" XU5      v   M     g 7fr   r   )r   r  r   r  r  s       r[   r   "sdpa_constraint.<locals>.<genexpr>:
  s)      "DC# 	6**"Ds   rA  )r   r`   r   rj   r   r   r  s   `    @r[   sdpa_constraintr  	  sz    yGv  "+Cgll,C"D D IOWa!"a):;;FW< Xs   $A8)r)  c                   UnU R                  5       UR                  5       :w  a  [        X0R                  5       5      nU R                  5       UR                  5       :w  a  [        X0R                  5       5      nU R	                  5       UR	                  5       :w  a$  [        X0R	                  5       5      n[        U5      $ [        U5      $ r   )rd   r  r   r   r   r  r  )r  r  r  r   rg   s        r[   r  r    s    ACNN,,a*+~~3==?*Q()}}#,,.(Q(Sz8Or]   )memory_formatc          	         [         R                  " U R                  5       U R                  5       U R	                  5       [        U R                  5       5      S9$ NrR  )r7   r   rd   r   rT  r_   r   )r   r  s     r[   r  r  $  s@     ||~kkmAJJL!	 r]   c                "   / n[        U [        5      (       a  [        U R                  [        R                  5      (       a  U R                  n [        U [        R                  5      (       aL  UR                  U R                  5       5        U R                  n [        U [        R                  5      (       a  ML  [        U 5      n [        U 5      n U(       a:  U R                  n US S S2    H  n[        R                  " XS9n M     [        U 5      n U $ )NrA  r-  )ra   r:   rc   r+   r4  re   
get_layoutr  )r   reinterpret_view_layoutsr.  s      r[   clone_preserve_reinterpret_viewr#  /  s    !!YJqvvr7I7I$J$JFFB..//$++ALLN;A B..// aLaAFF.tt4F""9A 5aLHr]   rB  c               Z   ^^^ UUU4S jn[         R                  " [        U5      TUU /S9$ )Nc                @   > [         R                  " TU S   -  T-   TS9$ )Nr   r   rI   rH  )r-  r   r(  r*  s    r[   rs   iota.<locals>.fnQ  s!    ~~dU1Xo5UCCr]   rR  )r7   r   r>   )lengthr(  r*  r   rl   requires_gradrs   s    ```   r[   iotar*  G  s1    D V$x	 r]   c           	       ^^^^ U R                  5       UR                  5       :X  d   eU R                  5       m[        U TS5      m[        R                  R
                  R                  [        R                  " TS5      5      (       a  TU R                  5       T   -   m[        R                  R
                  R                  ST5        [        R                  R
                  R                  TU R                  5       T   5        [        [        UT5      U R                  5       5      nUR                  5       mUUUU4S jn[        R                  " U R!                  5       U R                  5       U[#        U R                  5       5      S9$ )Nr   c           	       > [         R                  " [         R                  " [         R                  " U T   [        R
                  5      [         R                  " T[        R
                  5      5      T" U 5      T" U 5      5      $ r   )rI   r  eqrH  rp   r  )r  r   r-  
src_loaderr  s    r[   rP   select_scatter.<locals>.inner_fnh  sX    yyFFs3x5uekk2 sOSM
 	
r]   rR  )r   rT  r&  rJ   rS   r  r  r   r  r   r	  r
  r  r  r7   r   rd   r_   )r   r  r   r-  rP  r.  r  s     `` @@r[   select_scatterr0  \  s   ;;=CMMO+++}}H
3
"Cww%%ehhua&899

S))GGq%(GGeQZZ\#%67
3$ajjl
3C"J
 
 ||~kkmAJJL!	 r]   c           	     J  ^ ^^^^^^	^
 T R                  5       UR                  5       :X  d   eT R                  5       m
[        T TS5      mT R                  5       T   m[        R
                  R                  T TTT5      u  mm[        T R                  5       5      n[        TT-
  TS-
  -   T5      UT'   [        X5      nUR                  5       m	UUUU	UUU U
4S jn[        R                  " T R                  5       T R                  5       U[        T R                  5       5      S9$ )Nr   r(   c                V  >^ T	S:X  a  TT:X  a  T
S:X  a  T" U 5      $ [         R                  " U T   [        R                  5      n[	        U 5      m[        U T   T	-
  T
5      TT'   / nT	S:w  a]  UR                  [         R                  " U[         R                  " [        R                  " T	5      [        R                  5      5      5        TT:w  a]  UR                  [         R                  " U[         R                  " [        R                  " T5      [        R                  5      5      5        T
S:w  a}  UR                  [         R                  " [         R                  " [        U T   T	-
  ST
5      [        R                  5      [         R                  " S[        R                  5      5      5        U(       d   e[        R                  " [         R                   U5      n[         R"                  " UUU4S j[%        T5      (       a  SOS5      n[         R&                  " UUT" U 5      5      $ )Nr   r(   c                    > T" T 5      $ r   r   )src_idxr.  s   r[   rE  1slice_scatter.<locals>.inner_fn.<locals>.<lambda>  s
    Jw'r]   rG  )rI   rH  rp   rI  r_   r#   re   rK  r   r  rL  r-  r%   rJ  r   r  rM  rN  r   r  )r  rO  rT  src_valr4  r   r  r)  r.  r(  r*  r   r  s       @r[   rP  slice_scatter.<locals>.inner_fn  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK0	 t$/**' ##A

 yySM
 	
r]   rR  )r   rT  r&  r   r+   r'  normalize_start_endr_   r#   r  r7   r   rd   )r   r  r   r(  r)  r*  src_sizerP  r  r.  r  s   ` ````  @@@r[   slice_scatterr:  z  s    ;;=CMMO+++}}H
3
"Czz|C H11!S%EJE3AJJL!HS5[D1H5t<HSM

C"J,
 ,
\ ||~kkmAJJL!	 r]   c                v    [        U [        [        45      (       a  [        U 5      S:  a  [	        U S   5      $ U $ r  )ra   r_   r   r   _unwrapr   s    r[   r<  r<    s0    !dE]##A
qt}Hr]   r   rl   r.  
pin_memoryc               :  ^ ^ [        US [        R                  4;   SU 35        [        U(       + S5        [        [	        T 5      [
        5      (       a  T=(       d    [        R                  mOT=(       d    [        R                  " 5       m/ n[        T [        R                  5      (       a  U U4S jnO[        T [        [
        45      (       a  U U4S jnO[        T 5      S:X  d-  [        T S   [        [
        45      (       aE  [        T 5      S::  a6  UR                  [        R                  " [        T 5      5      5        U U4S jnO3[        R                  R!                  [        R"                  " T TUS95      $ [$        R&                  " [)        U5      TUUS	9$ )
Nlayout=r>  c                2   > [         R                  " TT5      $ r   r&  r-  rc   r   s    r[   rP  tensor.<locals>.inner_fn  s    >>$..r]   c                2   > [         R                  " TT5      $ r   rI   rJ  rB  s    r[   rP  rC    s    <<e,,r]   r   r   c                   >^ ^ UUUU 4S jm[        T5      S:X  a  [        R                  " ST5      $ T" S[        T5      5      $ )Nc           	     n  > X:  d   eX-
  S:X  a  [         R                  " TU    T5      $ X-
  S-  U -   n[         R                  " [         R                  " [         R                  " TS   [
        R                  5      [         R                  " U[
        R                  5      5      T" X5      T" X!5      5      $ )Nr(   r&   r   )rI   rJ  r  rL  rH  rp   rI  )r(  r)  midbinary_searchrc   r   r-  s      r[   rI  /tensor.<locals>.inner_fn.<locals>.binary_search  s    {"{;!#<<UU;;{q(50yyFFuQx=S%++6 "%-!#+ r]   r   )r   rI   rJ  )r-  rI  rc   r   s   `@r[   rP  rC    s<      4yA~||Au-- CI..r]   r  rR  )r   rp   stridedra   r<  r   rI  get_default_dtyper   r   r'  r   re   IntegerrJ   rS   add_tensor_constantr_  r7   r   r>   )rc   r   rl   r.  r>  rO  rP  s   ``     r[   r_  r_    s+   v$..'&0BC:~|,'$-%%$2002!F$$$	/ 
D5#,	'	'	- 
Ta:d1gs|<<TaemmCI./	/( ww**LLU6:
 	
 V$	 r]   c                |    [        U [        5      (       a  Ub  [        X5      n Ub  [        X5      n U $ [	        XUS9$ )Nr  )ra   r:   r   r  r_  )rc   r   rl   s      r[   	as_tensorrP    sA    $	""D(DT*D$F33r]   c                2    [        U [        R                  S9$ r`  r^  )rc   s    r[   long_tensorrR    s    $ekk**r]   c                   SSK Jn  U" [        R                  R                  R
                  [        R                  R                  R                  S   5      nUc   e[        U5      S:X  d   U5       e[        [        UR                  5       5      5      u  p4[        R                  " X4U 5      n[        R                  R                  U5      Ul        [        R                  R!                  U5        [        R                  R                  R                  S   n[#        U[$        R&                  [$        R(                  [$        R*                  45      (       a  UR,                  R.                  $ [0        R2                  " U5      $ )Nr   )resolve_unbacked_bindingsunbacked_bindingsr(   r  )%torch.fx.experimental.symbolic_shapesrT  rJ   rS   r  r  rT   rX  r   r3  iterr   r+   DynamicScalarregister_bufferr   register_operationra   rp   r  SymFloatSymBoolrY   r  r   r  )rc   rT  rU  binding_symkeypathbufferr  s          r[   _local_scalar_denser`    s   O 2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&   %6%<%<%> ?@KkD9F''))&1FKGGv& ''


#
#E
*C#ennemmDEExx}}}}S!!r]   c                    g r   r   )rc   r   s     r[   _assert_scalarrb  <  s     r]   )rl   r.  c                   g r   r   )r   r   r5  r   rl   r.  s         r[   _assert_tensor_metadatard  F  s     r]   c                  ^^^ U m[        U [        [        45      (       d  [        TS5      (       a  TR                  m[        T[        [        45      (       a  UU4S jnO\[        T[
        R                  5      (       a  UU4S jnO5[        TR                  5       5      S:X  d   eTR                  5       mU4S jn[        R                  " UTU[        U5      S9$ )Nr   c                2   > [         R                  " TT5      $ r   rE  r-  r   r   s    r[   rP  _full.<locals>.inner_fnT  s    <<u--r]   c                2   > [         R                  " TT5      $ r   r&  rg  s    r[   rP  rh  Y  s    >>%//r]   r   c                   > T" / 5      $ r   r   )r-  value_loaders    r[   rP  rh  `  s    ##r]   rR  )ra   r   r'  r   r   r   r   r   r   rT  r7   r   r_   )
fill_valuerl   r   r   rP  r   rk  s     `  @@r[   _fullrm  M  s    Ej3,//GE74K4K%#u&&	. 
E5;;	'	'	0 5>>#$)))((*	$ Dz	 r]   c                8    [        [        U5      5      " U 40 UD6$ r   create_tensor_liketensor_constructor)r   rl  r   s      r[   r  r  k  s    0<=aJ6JJr]   c                &   ^  S S S S SS S.U 4S jjnU$ )NF)namesr   rl   r.  r>  r  c                0  > [        U S L S5        [        US [        R                  4;   SU 35        [        U(       + S5        [        U5      nU=(       d    [        R                  " 5       n[        U5      S:X  a;  [        US   [        [        [        R                  45      (       a  [        US   5      nU H$  n[        U[        R                  5      (       d  M$   e   U Vs/ s H  n[        R                  " U5      PM     nn[        TX!U5      $ s  snf )Nnamed tensorsr@  r>  r(   r   )r   rp   rK  r>   rL  r   ra   r_   r   Sizer  r   r  rm  )	rs  r   rl   r.  r>  r  r   r  rl  s	           r[   r_  !tensor_constructor.<locals>.innerr  s     	5D=/26dEMM22gfX4FGz><0v&2002t9>ja4

2KLLa>D A!!U\\2222 )-.AQ.Z55 /s   $ Dr   )rl  r_  s   ` r[   rq  rq  p  s(     6 6. Lr]   )rs  r   r.  rl   r>  r  c           	         [        U S L S5        [        U5      n[        U5      S:X  a;  [        US   [        [
        [        R                  45      (       a  [        US   5      n[        US XX4S9$ )Nru  r(   r   r   r.  rl   r>  )	r   r>   r   ra   r_   r   rp   rv  empty_strided)rs  r   r.  rl   r>  r  r   s          r[   r  r    se     u}o.6"F
4yA~*T!WtUEJJ.GHHT!W~d%v r]   c                $   ^  SSSSSS.U 4S jjnU$ )zR
Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
NF)r   rl   r.  r>  r  c                 > [        U(       + S5        [        US [        R                  4;   SU 35        Uc  U R                  5       nO[	        U5      nU=(       d    U R                  5       n[        U R                  5       5      nT" XaX#US9$ )Nr>  r@  r=  )r   rp   rK  r   r   rd   r_   r   )r   r   rl   r.  r>  r  r   creation_fns          r[   _constant_like*create_tensor_like.<locals>._constant_like  s~     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!f

 	
r]   r   )r}  r~  s   ` r[   rp  rp    s#     duTX
 
 r]   c                *    [        [        U 5      5      $ r   ro  )rl  s    r[   r  r    s    0<==r]   c                "   ^  S S S S S.U 4S jjnU$ )Nry  c                 > [        U[        [        45      (       d   e[        U(       + S5        [        US [        R
                  4;   SU 35        [        U5      =(       d    U R                  5       nU=(       d    U R                  5       nU Vs/ s H  n[        R                  " U5      PM     nn[        T[        U5      X!5      $ s  snf )Nr>  r@  )ra   r_   r   r   rp   rK  r   r   rd   r   rM  rm  r>   )r   r   r   r.  rl   r>  r  rl  s          r[   _new_constant#new_constant.<locals>._new_constant  s     $u....z><06dEMM22gfX4FGU#4q{{})1<<>*./$Qa $/Zv!6DD 0s    C	r   )rl  r  s   ` r[   new_constantr    s     tDT	E 	E r]   ry  c          	     |    Uc  U R                  5       nUc  U R                  5       n[        US UU[        U5      US9$ Nry  r   rd   rz  r>   )r   r   r   r.  rl   r>  s         r[   	new_emptyr    sF    }~V$ r]   c                  [        U [        [        45      (       d   e[        U[        [        [        S 5      45      (       d   e[	        U(       + S5        [	        US [
        R                  4;   SU 35        [        U5      =(       d    [
        R                  " 5       nU=(       d     [
        R                  " S5      R                  n[        U5      n[        SXBU S9nUR                  5         UR                  R                  n[        R                   " UR                  S/[#        U 5      -  S9Ul        [        U[$        R&                  5      (       d   eU  Vs/ s H  n[(        R*                  " U5      PM     n nU(       a'  U Vs/ s H  n[(        R*                  " U5      PM     snO[$        R,                  R/                  U 5      n[$        R0                  " UUU US9Ul        U$ s  snf s  snf )Nr>  r@  rG  r   )rl  rl   r   r   )rO  )rl   r   r   r5  )ra   r_   r   rZ  r   rp   rK  r   rL  r_  rl   r>   rm  rk  rc   dataclassesreplacer   r+   r  r   r  rl  rm  r3  r.  )	r   r5  r   r.  rl   r>  	pointwiser_  r  s	            r[   rz  rz    s    dT5M****ftUDJ78888:~|,v$..'&0BC<5#:#:#<E/u||C(//F6"F6TJI^^  F%%fkk1#D	/JFKfb//0000%)*TELLOTD*  #))&Qa&)11$7 
 NN	FM  +)s    G' G,c          	     |    Uc  U R                  5       nUc  U R                  5       n[        UUUU[        U5      US9$ r  r  )r   r   r5  r   r.  rl   r>  s          r[   new_empty_stridedr    sH     }~V$ r]   c                   U Vs/ s H,  n[         R                  R                  R                  U5      PM.     nn[	        [        [        U5      5      UR                  S9n[        R                  R                  X5      $ s  snf )N)r  )rJ   rS   r  r  sortedr   r   __getitem__r+   r  r  )r   r5  r  r  s       r[   copy_stridedr    sa    >DEfagg11!4fFE%F,&2D2DEL??//@@ Fs   3A?c                X    UR                  S5      c   S5       e[        U5      " U 40 UD6$ )Nr   z(dtype should be handled by decomposition)rY  rq  )r   rl  r   s      r[   fullr    s2    ::g*V,VV*j)$9&99r]   c                  ^^^^ [        U [        5      (       d   eUR                  5       S:X  a  [        XR	                  5       5      $ U R	                  5       m[        T5      S:H  n[        U TU5      mU(       a  [        U S/5      n S/mU R                  5       mUR                  5       mUUUU4S jn[        R                  " U R                  5       U R                  5       UUR	                  5       S9$ )Nr   r(   c                   > [        U 5      n [        R                  " T" U 5      TT   5      n[        U 5      S:X  a  U/n OXT'   T" U 5      $ r  )r_   rI   indirect_indexingr   )r  
gather_idxr   index_loaderr   r  s     r[   rs   gather.<locals>.fn1  sJ    3i**<+<d3iH
s8q=,C!H}r]   rR  )ra   r:   	get_numelr  r   r   r&  r  rT  r7   r   rd   r   )	r   r   r-  sparse_gradr  rs   r  r   r  s	    `    @@@r[   gatherr    s     a####ANN,--::<DY!^F
3
'C1qcNs}}H$$&L  ||~kkm~~	 r]   c                V  ^^^^	^
 U(       a+  [        [        R                  R                  5      " XX#U5      $ U(       a   e[	        U [
        5      (       d   e[	        U[
        5      (       d   eS[        UR                  5       5      ;   d   eU R                  5       m	UR                  5       m[        UR                  5       5      mU R                  5       m
/ UR                  5       QT
SS  QmUUUU	U
4S jn[        R                  " U R                  5       U R                  5       UTS9$ )Nr   r(   c                   > [        U 5      [        T5      :X  d   U  ST 35       eT" U S T 5      n[        R                  " UTS   5      // U TS  Q-   nT" U5      $ )Nz != r   )r   rI   r  )r  	var_index
weight_idxindices_loaderindices_ndimr  weight_loaderweight_sizes      r[   rs   embedding.<locals>.fnT  s{    3x3x=(@SEhZ*@@("3}#56	++I{1~FG K
K
 

 Z((r]   rR  )r  r  	embeddingr  ra   r:   strr   rT  r   r   r7   r   rd   )weightr  padding_idxscale_grad_by_freqsparsers   r  r  r  r  r  s         @@@@@r[   r  r  B  s    6 67[f
 	
 :fi((((gy))))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 r]   c           
     L   [        S U  5       5      (       d-   SU  Vs/ s H  o"c  M  UR                  5       PM     sn 35       e[        S U  5       5      (       a  [        S5      e[	        U 5       VVs/ s H  u  p#[        U[        5      (       d  M  UPM      nnn[        U5      S:  d   S5       eS /[        U 5      -  n[        U[        U Vs/ s H  o U   PM	     sn6 5       H(  u  p#UR                  5       U:w  a  [        S5      eX5U'   M*     XT4$ s  snf s  snnf s  snf )Nc              3     #    U  HY  nUc  M  UR                  5       [        R                  [        R                  [        R                  [        R
                  4;   v   M[     g 7fr   )r   rp   rI  r  r   r  r   ri   s     r[   r   .check_and_broadcast_indices.<locals>.<genexpr>e  sA      A 	M%++u{{EJJLLs   A#AA#z)indices must be int64, byte or bool. Got c              3     #    U  H:  oc  M  UR                  5       [        R                  [        R                  4;   v   M<     g 7fr   )r   rp   r   r  r  s     r[   r   r  l  s.      <Cq2%**ekk22Gs
   A8AzFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r  r   r	  r   r`   ra   r:   r   r   r   rd   )r  rl   ri   r   
valid_idxsnew_indicess         r[   check_and_broadcast_indicesr  d  s2       
 4G4eGq]Q[[]G4e3fg   <C   ""=>> )' 2O 2jI6N! 2JOz?QD DD&3w<'KJ 1
3S
1AJ
3S TU <<>V#%&VWWA V ""# 5f P 4Ts   DD5DDD!c	           
     ^  ^^^^^^^^^^ Sm[        TTSS  5       H  u  pX-
  S:w  d  M  SmM     [        T5       VVs/ s H  u  pUb  M
  X   PM     snnm/ TQU [        T5      [        T5      -   S  QmTS   nT(       a  TT-   mOTS U T-   TUS  -   mUUUUUUUUUU4
S jnTU4$ s  snnf )NFr(   Tr   c                  >
 [        U 5      [        T5      :X  d   e[        T5      [        T
5      :X  d   e[        T5      n/ nTS   nT(       a  SOUnSn[        TS   S-   5       H|  nXd:X  a  XQ-  nTU   c+  U[        U 5      :  d   eUR                  X   5        US-  nM=  TU   nUc   eT
U   nUR                  [        R                  " U" XXA-    5      UT	TS95        M~     / UQXS  QnTc  U$ T" U5      $ )Nr   rA  r(   r   wrap_neg)r   r   re   rI   r  )r  r  	new_indexfirst_tensor_indexstart_offsetnext_idxri   loaderr   r   indexed_sizer  indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizer  r  s            r[   rs   *index_output_size_and_inner_fn.<locals>.fn  s=   3x3{++++?#s<'8888;	+A.3q9K~b)A-.A  qz!#c(***  /A(+)))#A  ))s,2EFG#!)	 /&

^
	 %,yE(92EEr]   )r   r`   r   )r  r  r  r  r  r  r  r   r  previouscurrentri   r  r  rs   r  r  s    ````````      @@r[   index_output_size_and_inner_fnr  ~  s    $ $ 1CD"&*# E ,5W+=M+=969+=MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	 F  FD ?_ Ns   	B)B)c                    [        XU5      u  p4n[        R                  " U R                  5       U R	                  5       UUS9$ r  )index_impl_helperr7   r   rd   r   )r   r  r   r  rP  r   s         r[   
index_implr    s>    0UCK1||~kkm	 r]   c                  ^^ [        U[        [        45      (       d   eU R                  5       m[	        XR                  5       5      u  p[        U5      S:  d   S5       eU Vs/ s H  oUb  UR                  5       OS PM     nn[        XS      R                  5       5      nU R                  5       n[        [        U5      5       Vs/ s H  oQU   c  M
  X   PM     n	nU(       a  SU	;   a  SU;  a  [        S5      e[        [        U5      5       Vs/ s H  oXU   PM	     n	n[        UUUUUU	S UUS9	u  n
mUU4S jnXT4$ s  snf s  snf s  snf )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0r  c                    > T" T" U 5      5      $ r   r   )r  index_inner_fnr  s    r[   rP  #index_impl_helper.<locals>.inner_fn  s    s+,,r]   )ra   r_   r   rT  r  rd   r   r   r   
IndexErrorr  )r   r  r   r  r  ri   r  r  r  r  r  rP  r  r  s               @@r[   r  r    sL   ge}----}}H9'<<>RG~"F$FF"KRS7a-q}}TA7OS wa01::<=KZZ\F',S\':U':!ajIFI':LUl"q';KLL',S\':;':!1I':L;"@
#K- .009 T V <s   %E		EEEc                     [        XSS9$ ! [         a;    U R                  5         [        [        R
                  R                  SS9" X5      s $ f = f)NTr   Fr  )r  r   rk  r  r  r-  rz  r   r  s     r[   r-  r-    sM    
!D11 
			

 1 1uM
 	

s   	 AAAc                    [        XSS9$ )NFr  )r  r  s     r[   _unsafe_indexr    s    a..r]   c           	     .    [        [        U 5      XUSSS9$ )NTFr   may_realizeindex_put_impl_r  r   r  r   
accumulates       r[   	index_putr    s    a':Tu r]   c           	     .    [        [        U 5      XUSSS9$ )NFr  r  r  s       r[   _unsafe_index_putr    s    a':U r]   c                    UR                  5       U R                  5       :w  a  [        X R                  5       5      nU(       a  [        X5      n[        U [	        US   X 5      5      $ r  )rd   r  r   r  r  )r  r  r   r  s       r[   index_put_as_masked_fillr    sP    T__..%!23D T5U9::r]   c                z    [         R                  " [        R                  R                  R
                  XX#5        U $ r   )r+   IndexPutFallbackrJ   rS   rT   rW   r  r  r   r  s       r[   index_put_fallbackr  &  s)    ,,33TFWKr]   c           	         [        XX#SSS9$ )NTr  r  r  s       r[   
index_put_r  +  s    v4 r]   c           	         [        XX#SSS9$ )NFTr  r  r  s       r[   _unsafe_index_put_r  2  s    vD r]   c                  ^ U(       aJ  S nS mU" U 5      UR                  5       ;   a*  [        U4S jU 5       5      (       d  UR                  5         UR                  5       S:X  a  [	        U5      S:X  a  US   R                  5       [        R                  [        R                  4;   a_  US   n[        [	        UR                  5       5      [	        U R                  5       5      5       H  n[        US5      nM     [        X/X#5      $ [        R                  " 5       (       a  [        XX#5      $ U HH  n	U	c  M  U	R                  5       [        R                  [        R                  4;   d  M<  [        XX#5      s  $    U R                  5       n
[	        U
5      nU(       aQ  [        U R                  5       5      (       a3  US:X  a  [!        U S/5      n [        XX#5      n US:X  a  [!        U / 5      n U $ [#        X R                  5       5      n [%        XR'                  5       5      u  pU Vs/ s H  ob  UR+                  5       OS PM     nn[-        U [.        5      (       d   eU R                  5         US:X  a  [!        U S/5      n [1        XS      R                  5       5      n[        [	        U5      5       Vs/ s H  oU   PM	     nn[3        U
UUUUUS US9u  nn[5        UU5      n[6        R8                  " U R'                  5       U R                  5       UR+                  5       UUU(       a  SOS S	9n[6        R:                  " S [6        R<                  " U 5      US
9n[>        R@                  RC                  U5      Ul"        [>        R@                  RG                  U5        US:X  a  [!        U / 5      n U $ ! [(         a    [        XX#5      s $ f = fs  snf s  snf )Nc                n   [        U [        R                  5      (       a  U R                  n [        U [        R                  5      (       a  U R                  5       n [        U [        R                  5      (       a  U R                  n [        U [        R                  5      (       a  U R                  5       $ S $ r   )	ra   r+   r:   rc   r/  r0  r  Bufferr  r   s    r[   try_get_name%index_put_impl_.<locals>.try_get_name<  ss    !R\\**FF!R[[))MMO!R]]++FF#-a#;#;1::<EEr]   c                &   [        U [        5      (       a  [        U R                  [        R                  5      (       a  U R                  R                  5       n [        U [        R                  5      =(       a    [        U R                  [        R                  5      =(       am    [        U R                  SS 5      =(       aO    U R                  R                  R                  [        R                  R                  R                  R                  :H  $ g)Nr  F)ra   r:   rc   r+   r/  r0  r  r  r   r  rW   rp   rI   r  randpermr  )indices    r[   indice_slice_from_randperm3index_put_impl_.<locals>.indice_slice_from_randpermE  s     &),,FKK1U1U002vr}}5 V"6;;@VY=V ++22eiinn6M6M6U6UU	 r]   c              3  4   >#    U  H  nT" U5      v   M     g 7fr   r   )r   r  r  s     r[   r   "index_put_impl_.<locals>.<genexpr>S  s      E
=D6&v..Wr  r(   r   rA  r  
atomic_addrl   r   rP  rO  output_indexerscatter_moder   r.  rc   )$r  r  rk  r  r   r   rp   r   r  r   r   r  r  $are_deterministic_algorithms_enabledr  rC   r  r   r  rd   r   rT  ra   r:   r_   r  r  r+   Scatterr  MutationLayoutSHOULDREMOVErJ   rS   rY  r   rZ  )r  r  r   r  r   r  r  rT  r   r-  r  x_ndimr  ri   r  r  r  expected_vals_sizerP  scatterr_  r  s                        @r[   r  r  9  sb   	F	 !6!6!88 E
=DE
 B
 B
 NN 	aLAAJ  "uzz5;;&??qzs4==?+S-ABAT2&D C'ffII 1133!$DD !2uzz5;;6O!O%dVHH  ]]_F[FB4>>CSTTQ;qc?D!$DQ;b>Dfnn./FE"=__&#
 LSS7a-q}}TA7OSdI&&&&LLN {D1# wa01::<=K',S\':;':!1I':L;#A	$  F./F jj nn##%!%/\TG ,,T2F
 ''))&1FKGGv&{D"~Ka  E!$DDE T <s   N4 *O8O4OOr  c                   ^^^^^	 [        XSSS9u  pEmTR                  5       mU R                  5       m	UUUUU	4S jn[        R                  " U R	                  5       U R                  5       UUS9$ )NFr  c                   >^  TR                   [        R                  :w  a,  [        R                  " T" T 5      [        R                  5      nOT" T 5      n[        R
                  " UUU U4S jT5      $ )Nc                     > T" T " T5      5      $ r   r   )_unsafe_index_fnr  self_loaders   r[   rE  8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    K8H8M,Nr]   )r   rp   r   rI   r   rN  )r  mask_valr  fillrT  mask_loaderr  s   ` r[   rP  &_unsafe_masked_index.<locals>.inner_fn  sK    ::#||K$4ejjAH"3'Hzz($NPTUUr]   rR  )r  rT  r7   r   rd   r   )
r  rT  r  r	  rO  r   rP  r  r
  r  s
    ` `   @@@r[   _unsafe_masked_indexr    ss    "3UU#F ""$K""$KV V  nn	 r]   c           	         [        XS5      nU R                  5       n[        [        U5      5       Vs/ s H%  nX&   (       a  [	        X&   XV   * XV   S-
  5      OS PM'     nn[        XUSS9$ s  snf )Nr   r(   T)r  )r  r   r   r   r%  r  )r   rT  r  r   masked_valuer	  ri   clamped_indicess           r[   #_unsafe_masked_index_put_accumulater    sx    q)LJJLE s7|$$A 7>jgj58)UX\2dJ$   Q$OOs   ,A,c                X    [         R                  " U[         R                  " X 5      5      $ r   )rI   rd  re  r   minmaxs      r[   r%  r%    s    ;;sCKK/00r]   c                J    [        U 5      n[        XRX45      n[        Xa5        U$ r   )r  r:  copy_)r  r  r   r5  r6  r"  output_views          r[   as_strided_scatterr    s$    4[FV6BK	+Mr]   c                0    [        [        U 5      XU40 UD6$ r   )scatter_r  )r   r   r-  r  r   s        r[   r  r    s    E!Hc#888r]   r  include_selfc               T   [        U[        5      n[        U UUR                  5       [	        [
        R                  U(       a  UR                  5       O
[        U5      5      U(       a  UR                  5       R                  OSU5      (       a  [        R                  " U UUUUUUS9  U$ g )Nznot implr  )ra   r:   rH   r   r	   rp   r   rZ  rd   r+   ScatterFallback)rJ  r  r   r-  r  r  r  src_is_tensors           r[   scatter_fallbackr     s     sI.MU[[]#--/S	J!.J  	%	
 r]   r  c          	         US;   d   eUcV  [        [        R                  [        R                  R
                  R                  R                  5      n[        XPXX4S9nUb  U$ US:X  a  SnOUS:X  a  Sn[        XX#U5      $ )N)Nr   multiplyr!  r   sumr#  prod)
r   r  r  rJ   rS   rT   rW   _overloadnamer   scatter_reduce_)r  r   r-  r  r  rJ  fallback_results          r[   r  r    s    ....~dmmQWW-A-A-H-H-V-VW*s3
 &""	:	4e&99r]   c                .    [        [        U 5      XU5      $ r   )scatter_add_r  r   r   r-  r  s       r[   scatter_addr,  )  s    a#c22r]   c                    [        XX#S5      $ )Nr$  )r'  r+  s       r[   r*  r*  .  s    15u55r]   c                0    [        [        U 5      XX440 UD6$ r   )r'  r  )r   r   r-  r  reduction_typer   s         r[   scatter_reducer0  3  s    58SOOOr]   )r  c          
     (  ^ ^^^^ US;   d   e[        [        R                  R                  5       5      S:X  a"  S[        R                  R                  5       ;   d   S5       e[	        T[
        5      (       a  [        T T5      m[        [        R                  R                  T TUTUUS9nU(       a  U$ [	        T [        5      (       d   eS[        UR                  5       5      ;   d   e[        T R                  5       5      nUS:X  a  [        T S/5      m [	        T[        5      (       a*  [        TR                  5       5      S:X  a  [        TS/5      m[	        U[        5      (       a*  [        UR                  5       5      S:X  a  [        US/5      nUR                  5       S:X  a  T $ [        T T5      mT R!                  5         UR#                  5       m[	        T[        5      (       a  TR#                  5       OS mUUU 4S jnU UU4S	 jn	S
 n
U(       d  [$        R&                  " T R)                  5       T R                  5       U 4S jUR                  5       US S9n[$        R*                  " S [$        R,                  " T 5      US9n[.        R0                  R3                  U5      Ul        [.        R0                  R7                  U5        [$        R&                  " T R)                  5       T R                  5       U	UR                  5       UU
" U5      S9n[$        R*                  " S [$        R,                  " T 5      US9n[.        R0                  R3                  U5      Ul        [.        R0                  R7                  U5        US:X  a  [        T / 5      m T $ )N)Nr$  r%  meanamaxaminr(   twozKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r  r   r   c                   > TR                  5       n[        U5      n[        U 5      n[        R                  " T" U 5      US:X  a  SOUT   SS9UT'   U$ )Nr   r(   F)r  )r   r   r_   rI   r  )r  r	  r  indirect_idxr   r  r  s       r[   r  'scatter_reduce_.<locals>.output_indexerf  sU    5zCy11DAIq5:
S r]   c                l   > T(       a  T" U 5      $ [         R                  " TTR                  5       5      $ r   rI   rJ  r   )r  r  r  r.  s    r[   rs   scatter_reduce_.<locals>.fnp  s*    c?" <<T^^%566r]   c                    U S:X  a  gU b   eg )Nr$  r  r   r!  s    r[   backend_reduce_str+scatter_reduce_.<locals>.backend_reduce_strw  s    U? >!>r]   c                N   > [         R                  " STR                  5       5      $ r  r:  )r-  r  s    r[   rE  !scatter_reduce_.<locals>.<lambda>  s    3<<4>>3C#Dr]   r  r  )r   r  r'  r   ra   r   r  r   r5  r:   r  r   r   r  r  r&  rk  rT  r+   r  rd   r  r  rJ   rS   rY  r   rZ  )r  r   r-  r  r  r  r(  r  r  rs   r=  zero_outr_  r  r  r.  s   `` `          @@r[   r'  r'  8  s   BBBBD  **,-2T))3355U UU	6 #vc"&  !O dI&&&&C)****t}}DqyD1##y!!c#,,.&9Q&>3n%##ENN,<(=(BUQC A
c
"CLLN$$&L&0i&@&@"dJ7 ::??$.."D>>#)
 ""006

 gg--f5	""6*
 jj nn~~%'/G ,,T2F
 ''))&1FKGGv&qyD"~Kr]   c                d  ^^^^^^ U R                  5         U R                  5       mU R                  5       T* S  mU R                  5       S T*  nT Vs/ s H,  n[        R                  R
                  R                  U5      PM.     snm[        U5      T:X  d   eUn[        TU5       VVs/ s H	  u  phXh-  PM     snnm[        U5       H  u  piU	c  M
  SU	-  TU'   M     U4S jmUUUUU4S jn
[        R                  " U R                  5       U R                  5       U
/ UQUQS9$ s  snf s  snnf )Nr  c                  > [         R                  " U [        R                  5      n T(       a:  [         R                  " U [         R
                  " S[        R                  5      5      n [         R                  " U [         R
                  " U[        R                  5      5      n [         R                  " U [        R                  5      n [         R                  " XSS9$ )N      ?Fr  )
rI   rH  rp   rb  r   rJ  rS  r   r  r  )r   rg  r   exacts      r[   scale_fn$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::r]   c                   > U T* S  nU S T*  nT
" / UQ[        UTT5       VVVs/ s H  u  p4nT	" X4U5      PM     snnnQ5      $ s  snnnf r   )r   )r  r   r!  ri   r  r   i_sizes
inv_scalesnrF  r  s         r[   rs   upsample_nearestnd.<locals>.fn  s_    H!HWaW#aW:UV:UJA$8A$':UVW
 	
Vs   ArR  )realize_hintrT  r   rJ   rS   r  r  r   r   r`   r7   r   rd   r   )r   r  scales_xrK  rE  batchri   o_sizesorg  rs   rI  rJ  rF  r  s      ``      @@@@r[   upsample_nearestndrR    s    NN}}HjjlA23GJJL1"EBIJ'Qqww55a8'JGx=AG$'$9:$9DA!%$9:Jh'%KJqM (	;
 
 ||~kkm!!!	 9 K
 ;s   3D'+D,c                    [        XU4SS9$ )Nr(   rK  rR  r   r  rq  s      r[   upsample_nearest1drW    s    avi1==r]   c                    [        XU4SSS9$ )Nr(   TrK  rE  rU  rV  s      r[   _upsample_nearest_exact1drZ    s    avi1DIIr]   c                    [        XX#4SS9$ )Nr&   rT  rU  r   r  scales_hscales_ws       r[   upsample_nearest2dr_    s     ax.BaHHr]   c                    [        XX#4SSS9$ )Nr&   TrY  rU  r\  s       r[   _upsample_nearest_exact2dra    s     ax.BatTTr]   c                    [        XX#U4SS9$ )Nr   rT  rU  r   r  scales_dr]  r^  s        r[   upsample_nearest3dre    s     ax8.LPQRRr]   c                    [        XX#U4SSS9$ )Nr   TrY  rU  rc  s        r[   _upsample_nearest_exact3drg    s     	X6!4 r]   c                .   ^  [        U 4S jU 5       5      $ )Nc              3  R   >#    U  H  n[         R                  " UT5      v   M     g 7fr   rE  )r   r   r   s     r[   r   $_create_constants.<locals>.<genexpr>	  s     6Aa''s   $')r   )r   rj   s   ` r[   ra  ra    s    6666r]   c                   ^^^ U R                  5       mU R                  5       mUUU4S jn[        R                  " U R	                  5       U R                  5       UTS9$ )Nc                   > [        U 5      n [        U 5      [        T5      :X  d   eT H  nTU   S-
  X   -
  X'   M     T" U 5      $ r  )r_   r   )r  r   r  r  r  s     r[   r  rev.<locals>.loader  sN    3i3x3u:%%%Cc
Q#(2CH  }r]   rR  )rT  r   r7   r   rd   r   )r   r  r  r  r  s    ` @@r[   revrn    sM     }}HJJLE ||~kkm	 r]   c                   S nU" 5       (       a  g[        U5      S:w  d  [        U R                  5       5      S:w  a  gU R                  5         [        U [        R
                  5      (       a  [        U R                  [        R                  5      (       a  [        U R                  R                  [        R                  5      (       dH  [        R                  (       aX  [        U R                  R                  [        R                  5      (       a%  U R                  R                  R                  (       d  gU R                  5         [        R                  " U 5      u  pEUR                  nUS   S:w  a  gUS   S:w  d  US   S:w  d	  US   S:w  a  gUS   nUS:X  a  gUS   nUR                   S   n	XU-   :  a  gU R                  R                  R                  n
UR                   S   UR                   S   U-   /nU["        R$                  R&                  U
'   [)        U UUR                  UR*                  5      n[-        USXU-   S9n[/        X5        [0        S	   S
==   S-  ss'   U$ )z
This optimization changes the semantics of padding from 'clone'
style to 'view' style.

Thanks to functionalization, this change can still maintain numerical
correctness.
c                    [         R                  R                  n U c  g[        U R                  5      n[        U5      S:X  aF  US   R                  [        R                  R                  [        R                  R                  4;   a  gg)z
Conservatively check if padding can be fused with downstream op.
1. if the downstream op is a sum, then there is little benefit to
   do inplace padding
2. if the downstream op is a matmul, doing inplace padding can
   save membw.
Tr(   r   F)rJ   rS   rT   r   rU   r   rW   r  mmr  addmm)rT   rU   s     r[   _padding_can_be_fused6inplace_constant_pad_nd.<locals>._padding_can_be_fused-  sk     ww++l(()u:?uQxGGOOJJ3
  
 r]   Nr   r&   r(   r   r   )r   r(  r)  inductorinplace_padding)r   r   rk  ra   r+   r:   rc   r  r  r)   can_inplace_pad_graph_inputInputBufferr   freeze_layoutr2  r5  r   rJ   rS   buffer_to_padded_sizer:  r  r+  fill_r   )r   paddingrl  rs  r   r.  r  npadstride0rowsizebufnamepadded_size	resized_xsliced_xs                 r[   inplace_constant_pad_ndr  "  s   (  7|qC

-2
 IIK
 q",,''!&&"--00qvv{{B$5$56622qvv{{BNN;; vv{{OO((+IAmmGqzQqzQ'!*/WQZ1_1:DqyajGkk!nG4ffkkG;;q>6;;q>D#89K-8AGG!!'*		I iQgT>JH	(Z*+q0+r]   c                  ^^^^^^^ [        U5      S-  S:X  d   e[        S U 5       5      (       a  [        U 5      $ [        R                  (       a  [        XT5      nU(       a  U$ U R                  5       n[        [        [        [        US S S2   USS S2   5      5      5      5      m[        U5      [        T5      -
  m/ mT H?  u  pVTR                  [        R                  R                  R                  U5      U45        MA     [        US T 5      n/ m[        TUTS  5       HC  u  u  pn
TR                  U
5        UR                  [        R                   " X-   U	-   5      5        ME     [        U5      [        U5      :X  d   e[#        U R%                  5       5      " T5      mUUUUU4S jmUUU4S jnU R'                  5       m[(        R*                  " U R-                  5       U R%                  5       UUS9$ )Nr&   r   c              3  *   #    U  H	  oS :H  v   M     g7fr   Nr   r   ru  s     r[   r   "constant_pad_nd.<locals>.<genexpr>  s     
#7a67r  r(   c                N  >^  / n[        T T	S  TT5       HL  u  nu  p4nUS:w  a  UR                  [        US5      5        US:w  d  M2  UR                  [        X%5      5        MN     [        R
                  " [        R                  U5      n[        R                  " UU U
4S jT5      $ )Nr   c                    > T" T 5      $ r   r   )r-  r  s   r[   rE  /constant_pad_nd.<locals>.mask.<locals>.<lambda>  s	    r]   )	r   re   range_mask_lowrange_mask_highr   r  rI   rM  rN  )r-  rT  r  r  r  r(  boundsrl  
mask_sizesrK  r  s   `     r[   rT  constant_pad_nd.<locals>.mask  s    (+E!"Ivz(J$C#faxN323qyOC89	 )K
 $/zz$ 7DDr]   c                   > [        U S T 5      n[        U TS  T5       H  u  nu  p4UR                  X#-
  5        M     [        U5      [        U 5      :X  d   eT" U5      $ r   )r_   r   re   r   )r-  r  r  r  _highbounds_precomprT  rK  s        r[   	offset_fn"constant_pad_nd.<locals>.offset_fn  sa    rO	!$U12Y!?C#SY' "@9~U+++Ir]   rR  )r   r  r  r)   rv  r  r   r_   r  r   re   rJ   rS   r  lookup_precomputed_sizer   r  r   r   rT  r7   r   rd   )r   r|  rl  rg   r  lhr  r  r  r   r  r  r  rT  r  rK  r  s     `         @@@@@@r[   r  r    s   L1"""

#7
###Qx%a*=J JJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68Nqww//GGJANO  uRay/KJ qr3T$5<<
T(9:; 4 {s5z)))q{{}-j9JE E }}H||~kkm	 r]   c                    [         R                  " [         R                  " U [        R                  5      [         R                  " [
        R                  " U5      [        R                  5      5      $ r   )rI   rK  rH  rp   rI  r   rM  )ri   r  s     r[   r  r    s@    66q%++&u}}S)5;;7 r]   c                    [         R                  " [         R                  " U [        R                  5      [         R                  " U[        R                  5      5      $ r   )rI   rL  rH  rp   rI  )ri   r  s     r[   r  r    s7    66q%++&tU[[) r]   c                T    [         R                  " [        X5      [        X5      5      $ r   )rI   rM  r  r  )ri   r  r  s      r[   
range_maskr    s#    88q  r]   c                   ^ ^^^^^^^ T R                  5       T* S  mT R                  5       mT=(       d    S/T-  mUUUUUUU U4S jnU$ )Nr   c                h  >^^ U S T*  mU T* S  m[         R                  " [        R                  [	        T5       Vs/ s H   n[        TU   TU   T
U   -   T
U   * 5      PM"     sn5      nT	(       a   [        R                  " UUUUUU4S jT5      $ [        R                  " UUUU4S jT5      $ s  snf )Nc                 .   > [        TTT S9" / TQTQ5      $ )Nr   )constant_boundary_condition)r   ihpad_fill_valueprefixr   s   r[   rE  ;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s    3A~3O"fNrNr]   c                    > T" / TQT Q5      $ r   r   )r  r  r  s   r[   rE  r    s    (>V>b>*Br]   )r   r  rI   rM  r   r  rN  )r-  ri   rT  r  r  r   rl  r  r  r|  	padding_hr   r  s      @@r[   rF  )constant_boundary_condition.<locals>.load  s    uC45\HHLQRUJWJqZ1qtil2Yq\MBJW
  JJ  
	
 D"BJO
	
 Xs   'B/
)r   rT  )	r   rl  r|  r  r   rF  r  r  r  s	   ````` @@@r[   r  r    sE     	


cTUA}}H$A39I
 
( Kr]   dilationc                  Uc  S/[        U5      -  n[        U SXA   -  -   Xa   X!   S-
  -  -
  X1   S-
  -   X1   5      nU(       Ga  [        U SXA   -  -   Xa   X!   S-
  -  -
  SX1   S-
  -  -   X1   5      n[        R                  R                  R                  US-
  X1   -  U -
  XA   -
  5      S:  a<  US-  n[        R                  R                  R                  SXU   -  U -
  XA   -
  5        [        R                  R                  R                  Xx-
  5      S:X  a.  [        R                  R                  R                  Xx5        SnXu4$ UnXu4$ )Nr(   r&   r   F)r   r#   rJ   rS   r  r  r	  r  )	r   ri   kernel_sizer5  r|  	ceil_moder  x_outx_alts	            r[   pooling_sizer    si   3W%	A
NX[KNQ,>??69q=Q	E
 '*nk[^a/01 69q=!" I
 77%%uqyFI&=&AGJ&NOSTTQJEGG&&q%)*;a*?'**LM77%%em49GG))%7I  Er]   c               l    [        X5      n [        R                  " [        R                  U 5      nUS:  $ )N   )rD   r   r  r  rS  )r  n_dimwindow_sizes      r[   %should_fallback_max_pool_with_indicesr  
  s-    {2K""8<<=Kr]   assert_fallbackc                  US:X  a  S/U-  nUS:X  a  S/U-  nU(       d  Un[        X5      n[        X%5      n[        X55      n[        XE5      n[        U [        5      (       d   e[        U5      U:X  d   e[        U5      U:X  d   e[        U5      U:X  d   e[        U5      U:X  d   e[        U R	                  5       5      US-   US-   4;   d   e[        XS9nUb  Xv:X  d   eXX4U4$ )Nr   r(   r&   r  )rD   ra   r:   r   r   r  )r   r  r5  r|  r  r  r  use_fallbacks           r[   max_pool_checksr    s    !|#+1}3;{2K&(F7*GH,Ha####{u$$$v;%w<5   x=E!!!qzz|EAI 66668RL"...<??r]   c               &  ^^^^^ U R                  5         U R                  S T*  nU R                  T* S  n[        [        T5       V	s/ s H  n	[	        X   XTTUTS9PM     sn	6 u  pU R
                  nU[        R                  L a  SO;UR                  (       a  [        S5      O[        R                  " U5      R                  n[        U5      [        U
5      -   n[        T5      (       d'  [        U5      (       d  [        S T 5       5      (       a  [        XTS9mOU R                  5       mUUUUU4S jn[         R"                  " SU U R%                  5       UUUUUS9n[         R"                  " S	U U R%                  5       [        R&                  UUUUS9n[)        UR*                  R*                  [         5      (       a  UR-                  5         [)        UR*                  R*                  [         5      (       a  UR-                  5         UU4$ s  sn	f )
Nr  F-infc              3  *   #    U  H	  oS :  v   M     g7fr  r   r  s     r[   r   )_max_pool_with_offsets.<locals>.<genexpr>L  s     ,EHqUHr  r  c                   > U S T*  nU T* S  n[        T5       Vs/ s H  nX4   T	U   -  X   TU   -  -   TU   -
  PM      nnT
" / UQUQ5      $ s  snf r   r   )r  reduction_idxr  bhri   r  r  r  r|  r5  r  s         r[   fn_inner(_max_pool_with_offsets.<locals>.fn_innerQ  s    Wuf%\ 5\
! UVAY=#3hqk#ABWQZO! 	 
 &2''	
s   %Ar  r/  
input_noderl   	dst_dtyper|  rP  rO  reduction_rangesargmax)rM  r	  r   r   r  r   rp   r   r  r'  r  r  r_   r	  r  rT  r8   r   rd   rI  ra   rc   rk  )r   r  r5  r|  r  r  r  rO  dhwr  dhw_outr   	min_valuer  r  r>  offsetsr  s     ``` `          @r[   _max_pool_with_offsetsr  -  s    NNGGGeVE
''5&'
C
 5\	

 " X "	

G GGE EJJ 	$66eFmEKK<N<R<R  E{T']*H
7||s9~~,EH,E)E)E.qG==?( ( ||~$	F ||~++$	G &++""I..',,##Y//7?s

s   Hc                    [        U5      n[        U UUUUUSS9u  pp4n[        R                  " SS9   [	        U UUUUUUS9u  pU[        U	[        R                  5      4sS S S 5        $ ! , (       d  f       g = f)NFr  r  unroll_reductions_thresholdr  )r   r  r)   r   r  r   rp   r  )
r   r  r5  r|  r  r  r  r   r>  r  s
             r[   !_low_memory_max_pool_with_offsetsr  x  s     E 1@	1-KA 
"	50
 x44 
6	5	5s   -A**
A8c                R  ^^^^^^ [        T5      mU R                  5       m[        R                  " [        R
                  " [        R                  T5      5      mUUUUUU4S jn[        R                  " U R                  5       [        R                  UU R                  5       S9nU$ )Nc                   > T	" U 5      n[         R                  " UT
5      n[        R                  " UT5      nT" X5      n[         R                  " [        R
                  " UTT* S  5      [        R                  5      $ r   )rI   r  r*   _flattened_index_to_ndrH  _flatten_indexrp   rI  )r  r  offset_sympyr  idhwincrements_to_index
input_sizer  r  offsets_loaderr  s        r[   offsets_to_indices4_pool_offsets_to_indices.<locals>.offsets_to_indices  sk    $,,V[A&==lKX"36~~))$
E670CDekk
 	
r]   rR  )r   rT  r   r  r   r  r  rS  r7   r   rd   rp   rI  r   )	r  r  r  r  r  r  r  r  r  s	    ```  @@@r[   _pool_offsets_to_indicesr    s     E((*N--	 0 0{ KLK
 
 !!#kk#!	G Nr]   c                J   ^^^^ [        U5      mUUUU4S jn[        XX&5      $ )Nc                   > U T* S  n[        T5       Vs/ s H  nX#   TU   -  X   TU   -  -   TU   -
  PM      sn$ s  snf r   r  )r  r  r  ri   r  r  r|  r5  s       r[   r  D_low_memory_max_pool_offsets_to_indices.<locals>.increments_to_index  s`    %\ 5\
! UVAY=#3hqk#ABWQZO!
 	
 
s   %=)r   r  )r  r  r  r5  r|  r  r  r  s      ``` @r[   '_low_memory_max_pool_offsets_to_indicesr    s-     E
 
 $j r]   c           
     ~    [        XX#XFS9u  pp4n[        XX#XEUS9u  p[        U	UU R                  U* S  UUU5      n
X4$ )Nr  )r  r  r  r	  )r   r  r5  r|  r  r  r  r   rg   r  r  s              r[   _max_pool_with_indicesr    si     1@	1-KA *	ELC 6	G <r]   c           
         [        XX#XESS9$ Nr&   r  r  r   r  r5  r|  r  r  s         r[   max_pool2d_with_indicesr         "	A r]   c           
         [        XX#XESS9$ Nr   r  r  r  s         r[   max_pool3d_with_indicesr    r  r]   c                `  ^^^^^^^^^^^ TS:X  a  SS/mUS:X  a  SS/nT(       d  Tm[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        U5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         U R                  5       n[        U[        5      (       a  [        UR                  R                  [        5      (       a  UR                  R                  n	[        R                  " S [        R                  " U	R                  5       U	R                  5       U	R                  5       S9U	S9n
U
R                  5         U
R                  5       nOUR                  5       nUS L=(       a    US   S:H  =(       d    US L=(       a    US   S:H  n[        S U 5       5      (       a  [!        XTTTXVU5      $ UR                  5       Gt pmU R                  5       Gt nmmUR#                  5       mU R#                  5       m[%        UR                  5       5      n['        UU4S j[)        TS   S-  5       5       5      m['        UU4S	 j[)        TS   S-  5       5       5      mTT-  nUS
:  a  [!        XTTTXVU5      $ UR                  5       mUUUUUUUUUUU4S jn[        R*                  " U R                  5       U R                  5       UUS9nU(       a  [        R,                  R/                  U5      $ U$ )Nr   r(   r&   r  )rl   r   r   r  c              3  *   #    U  H	  oS :g  v   M     g7fr  r   r  s     r[   r   3max_pool2d_with_indices_backward.<locals>.<genexpr>9  s     
$8a68r  c           
   3  z   >#    U  H0  n[        UTS    -  [        S UTS    -
  TS    -  5      -
  S5      v   M2     g7fr   r(   Nr  r   r  r  r5  s     r[   r   r  F  H      *A 	ANSQQ%7F1I$EFFJJ*   8;c           
   3  z   >#    U  H0  n[        UTS    -  [        SUTS    -
  TS    -  5      -
  S 5      v   M2     g7fr(   r   Nr  r   wr  r5  s     r[   r   r  J  r  r  r  c                  > U Gt pn[         R                  " UT-  U-   [        R                  5      nUTS   -   nUTS   -   n[         R                  " [	        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [	        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [	        UTS   5      S-   [        R                  5      n[         R                  " [	        UTS   5      S-   [        R                  5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      nS n	[        T5       GHD  n
[        T5       GH0  n[         R                  " U[         R                  " U
[        R                  5      5      n[         R                  " U[         R                  " U[        R                  5      5      n/ UQ[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TS   SS9P[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TS   SS9PnT" U5      nT" U5      n[         R                  " X5      nU	c>  [         R                  " UU[         R                  " S[        R                  5      5      n	GM  [         R                  " [         R                  " [         R                   " X5      [         R                   " X5      5      U5      n[         R                  " U[         R                  " U	U5      U	5      n	GM3     GMG     U	c   eU	$ )Nr   r(   Fr  rA  rG  )rI   rH  rp   r  r#   rd  rJ  re  r   r   r  r|  r-  r  rb  rM  rL  )r  r  r  r  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   rT  grad_loaderh_window_sizer  indices_sizer  r|  pooled_heightpooled_widthr5  w_window_sizewidths                      r[   rs   ,max_pool2d_with_indices_backward.<locals>.fnY  s+   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|8#"yyy#,,sEMM*J H 88FF2-FF2- D  #yyswwx/KXVHE , (H ###r]   rR  )ra   r:   r   r   rM  r  rc   r7   r+   r  rl  rd   r   decide_layoutr  r	  )fallback_max_pool2d_with_indices_backwardrT  r_   r  r   r   r  r  )grad_outputr   r  r5  r|  r  r  r  	gO_striderc   x_bufferx_strideis_channels_last_batch_heightr   r  r  rs   rg   r  r  r  r  r  r  r  r  s     ```               @@@@@@@@r[    max_pool2d_with_indices_backwardr    s    !|a&1}q6a####{q   v;!w<1x=Aqzz|&&& ,,.I!YJqvv{{I$F$Fvv{{$$$$(nn&]]_
 
 	 &&(%%' ,A!1A 3)A,!"3  
$8
$$$8K(w
 	
  jjlVe&1&:&:&<#Q|((*N))+KAJJL!H {1~)* M  {1~)* M
  -/KR8K(w
 	
 ##%L9 9v 

%%'##%	C 44S99
r]   c                8   ^^ U R                  5       mUU4S jnU$ )Nc           
       >^ ^^^	^
 Uu  mm	Uu  mm
Uu  pE[         R                  " [         R                  " [         R                  " TT-   [        R
                  5      [         R                  " U[        R
                  5      5      [         R                  " [         R                  " T
T	-   [        R
                  5      [         R                  " U[        R
                  5      5      5      n[         R                  " UUUU	U U
U4S jT5      $ )Nc                 ,   > T" / TQT T-   PTT-   P5      $ r   r   )h_start_indexr  iwr  w_start_indexr  s   r[   rE  3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s$    HNvN}r'9N=2;MNOr]   )rI   rM  rL  rH  rp   rI  rN  )r  
incrementsstart_indicesend_indicesh_end_indexw_end_indexrT  r!  r  r"  r#  pad_valr  s   `      @@@@r[   rF  !pad_adaptive_loader.<locals>.load  s    B'4$}#. xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
r]   rT  )r   r*  rF  r  s    ` @r[   pad_adaptive_loaderr-    s    }}H
, Kr]   c                    [         R                  " XUS9n[         R                  " XUS9n[         R                  " XUS9n[         R                  " XUS9n	XgX4$ )N)out_diminp_dim)r   r  )
start_index	end_indexh_inw_inh_outw_outr!  r(  r#  r)  s
             r[    compute_indices_adaptive_poolingr7    sU    %%k$OM##IdKK%%k$OM##IdKK}AAr]   c                \   ^^^^^^ Uu  pgUu  p[        XXgX5      u  mmmmUUUUUU4S jn
U
$ )Nc                   > U Gt p#nT" U5      nT" U5      nT" U5      nT" U5      nS n	[         R                  " [        TS   5      [        TS   5      5       H"  u  pU" UX/XW/Xh/5      nU	c  Un	M  T" X5      n	M$     U	$ r  )r   productr   )r  r  r  r  bwr!  r(  r#  r)  r>  r  r"  r  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                r[   rs    _adaptive_pooling_fn.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VWFB.*	C ~#C0 X r]   r7  )r1  r2  r>  in_sizes	out_sizesr?  r3  r4  r5  r6  rs   r<  r=  r@  rA  s     `  `     @@@@r[   _adaptive_pooling_fnrF    sF     JDLE 	)E	
 . Ir]   c                d   ^^^
^^^^ Uu  nmUu  px[        XUTXx5      u  mm
mmU
UUUUUU4S jn	U	$ )Nc                  > U Gt p#nT" U5      nT" U5      nT" U5      nT" U5      nS n	S n
[         R                  " [        TS   5      [        TS   5      5       H  u  pU" UX/XW/Xh/5      n[        R                  " X[-   T-  U-   U-   [
        R                  5      nU
c  Un
O+[        R                  " [        R                  " X5      X5      n
U	c  Un	M{  T" X5      n	M     U
$ r  )	r   r:  r   rI   rH  rp   rI  r  gt)r  r  r  r  r;  r!  r(  r#  r)  maxvalmaxindexr  r"  r  r-  r<  r=  r>  r?  r@  r4  rA  s                  r[   rs   )_adaptive_pooling_fn_with_idx.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VWFB.*	C NN#t+m;b@%++E  99SVVC%8%J~#C0) X, r]   rC  )r1  r2  r>  rD  rE  r?  r3  r5  r6  rs   r<  r=  r@  r4  rA  s     `  `    @@@@@r[   _adaptive_pooling_fn_with_idxrM    sK     JD$LE 	)dE	
! !F Ir]   c           	     
  ^ ^^ T R                  5       [        R                  :X  a  [        S5      e[	        T [
        5      (       d   e[        U5      S:X  d   eT R                  5         T R                  5       Gt p#n[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUu  pVX5:X  a  XF:X  a  [        T 5      $ US:X  d  US:X  a/  / UQUPUPn[        UT R                  5       T R                  5       S9$ X5-  S:X  a  XF-  S:X  a  X5-  XF-  /n[!        T U5      $ [#        X5-   S-
  U5      n	[#        XF-   S-
  U5      n
[%        U5      XV/-   nT R                  5       nX-  nUS:  a  ['        T U5      $ S nS n[)        UUX/X4/XV/[*        R,                  S	9m[/        [1        T 5      5      mUUU 4S
 jn[2        R4                  " T R                  5       UUUS9nU$ )Nz0'adaptive_avg_pool2d' not implemented for 'Long'r&   r   r  r(   r  c                    [        X-  U5      $ r   r#   r-  r/  r0  s      r[   r1  )_adaptive_avg_pool2d.<locals>.start_indexO      733r]   c                2    [        U S-   U-  U-   S-
  U5      $ r  rP  rQ  s      r[   r2  '_adaptive_avg_pool2d.<locals>.end_indexR  "    g-7!;WEEr]   r1  r2  r>  rD  rE  r?  c                `   > [         R                  " T" U [        T5      5      T" U T5      5      $ r   )rI   truedivr-  )r  fn_sumones_loaderr   s    r[   rs    _adaptive_avg_pool2d.<locals>.fn`  s-    {{3+A./[1I
 	
r]   rR  )r   rp   rI  rY  ra   r:   r   rM  r   rJ   rS   r  r  r  r  rd   
avg_pool2dr=   r_   fallback_adaptive_avg_pool2drF  rI   r   r-  	ones_liker7   r   )r   r  rO  r3  r4  r5  r6  o_sizer  h_kernel_maxw_kernel_maxr  r   r  r1  r2  rs   rvrZ  r[  s   `                 @@r[   _adaptive_avg_pool2drd  )  s   {{}#MNNa####{q   NNU$7711$7D7711$7DLE }QxzUaZ'5'%''V1;;=HH|qT\Q.}dm4![))DL1,u5LDL1,u5LE{e^+HKKME-KR+A{;;4F ""1.77F &il3K

 
		||~	
B Ir]   c           	       ^ ^^ T R                  5       [        R                  :X  a  [        S5      e[	        T [
        5      (       d   e[        U5      S:X  d   eT R                  5         T R                  5       Gt p#n[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUu  pVUS:X  d  US:X  aV  / UQUPUPn[        UT R                  5       T R                  5       S9[        U[        R                  T R                  5       S94$ X5-  S:X  a  XF-  S:X  a  [        e[!        X5-   S-
  U5      n[!        XF-   S-
  U5      n	[#        U5      XV/-   n
T R                  5       nX-  nUS:  a  [%        T U5      $ S nS n['        UUX/X4/XV/[(        R*                  S	9m[-        UUX/X4/XV/[(        R*                  S	9mUU 4S
 jnUU 4S jn[.        R0                  " T R                  5       UUU
S9n[.        R0                  " T R                  5       [        R                  UU
S9nUU4$ )Nz,adaptive_max_pool2d not implemented for Longr&   r   r  r(   r  c                    [        X-  U5      $ r   rP  rQ  s      r[   r1  (adaptive_max_pool2d.<locals>.start_index  rS  r]   c                2    [        U S-   U-  U-   S-
  U5      $ r  rP  rQ  s      r[   r2  &adaptive_max_pool2d.<locals>.end_index  rV  r]   rW  c           	     <   > T" U [        T[        S5      5      5      $ Nr  r-  r'  )r  inner_func_max_valr   s    r[   inner_fn_max_val-adaptive_max_pool2d.<locals>.inner_fn_max_val      !#':1eFm'LMMr]   c           	     <   > T" U [        T[        S5      5      5      $ rk  rl  )r  inner_func_max_idxr   s    r[   inner_fn_max_idx-adaptive_max_pool2d.<locals>.inner_fn_max_idx  rp  r]   rR  )r   rp   rI  rY  ra   r:   r   rM  r   rJ   rS   r  r  r  rd   
ValueErrorr=   r_   fallback_adaptive_max_pool2drF  rI   rd  rM  r7   r   )r   r  rO  r3  r4  r5  r6  r`  ra  rb  r  r   r  r1  r2  rn  rs  rc  rirr  rm  s   `                  @@r[   adaptive_max_pool2drx  t  s3   {{}#IJJa####{q   NNU$7711$7D7711$7DLEzUaZ'5'%''V1;;=H%%++allnK
 
 	
 |qT\Q.DL1,u5LDL1,u5LE{e^+HKKME-KR+A{;;4F ."1.;; 7"1.;;NN 
		||~!	
B 
		||~kk!	
B r6Mr]   c                j   ^ ^^^^^^ TT   mTT   mTT   mT R                  5       mUUUUUU U4S jnU$ )Nc                  > T" / U QTS-
  T	-
  P5      n[         R                  " UTR                  5       5      n[         R                  " T
T-
  [        R                  5      n[         R                  " TS-
  [        R                  5      n[         R
                  " [         R                  " U[        R                  5      [         R                  " U[        R                  5      5      n[         R                  " [         R                  " US5      SU5      n[         R                  " X2-   U-  5      [         R                  " X&-  5      -
  n[         R                  " U[        R                  5      n[         R                  " X55      n[         R                  " [         R                  " XU5      [        R                  " T
5      5      $ )Nr(   r   )rI   rH  r   rp   rI  rY  r   float64r  r-  r  rL  r  r   r  )r  ri   samplei_exprdiffout_sz_exprr;  seq_irT  r   in_sz	kernel_szndimsout_szsamplessamples_loaders            r[   rF  )_fractional_pooling_offsets.<locals>.load  s)    :& :%!)c/ :;7#4#4#67~~ei/=nnVaZ=LLu}}-s||K/W
 		#&&a0!U;		6?e34syy7PPUEKK0vvf*$$SYYtD%A5==QVCWXXr]   r,  )r  r  r  r  r   r  rF  r  s   `````` @r[   _fractional_pooling_offsetsr    s?    C[F#JE#I((*NY Y Kr]   c                    [        XX#SS9$ r  _fractional_max_poolr   r  r  random_sampless       r[   fractional_max_pool2dr        STUUr]   c                    [        XX#SS9$ r  r  r  s       r[   fractional_max_pool3dr    r  r]   c                  ^^^^ U R                  5         U R                  S T*  U R                  T* S  pe[        R                  " SS9   [	        T5       Vs/ s H  n[        UUUUTUS9PM     snmU R                  5       mUUU4S jnUU4S jm[        U5      [        U5      -   n	U R                  5       n
[        R                  " SU U R                  5       U
U
UU	US9n[        R                  " SU U R                  5       [        R                  U
UU	US9n[        UR                  R                  [        5      (       a  UR!                  5         [        UR                  R                  [        5      (       a  UR!                  5         [#        XU R                  T5      nX4sS S S 5        $ s  snf ! , (       d  f       g = f)	Nr  r  )r  r  r  r  r  r   c                4   > U S T*  nT" / UQT" X5      Q5      $ r   r   )r  r  r  r  r  r  s      r[   r  &_fractional_max_pool.<locals>.fn_inner  s,    5&\FOfO':3'NOPPr]   c                   > U S T*  nU T* S  n[        T5       Vs/ s H  nTU   " X#U   5      X   -   PM     sn$ s  snf r   r  )r  r  r  bdhwr  dhw_index_fnr  s        r[   r  1_fractional_max_pool.<locals>.increments_to_index  s]    5&\Fvw<D u%A QQ0=3CC%  s   ;r  r  r  )rM  r	  r)   r   r   r  rT  r_   r   r8   r   rd   rp   rI  ra   rc   rk  r  )r   r  r  r  r  rO  inp_dhwr  r  r  r   r>  r  r  r  r  r  s       `         @@@r[   r  r    s   NNWWWuf%qwwvw'77	"	5 5\

 " (&"% "

 ==?	Q	 ;k!22!! <<>(	
 ""#<<>kk(	
 fkk&&	22NNgll''33OO*!''+>
 w 
6	5

 
6	5s   F1F,.D4F1,F11
F?c           	     D  ^ ^^ T R                  5         T R                  5       Gt pVn[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUGt pXn	Xh-  S:X  a  Xy-  S:X  a  [        T Xh-  Xy-  /SS9$ [        Xh5      n
[        Xy5      nS mU4S jn[        TUX/Xg/X/[        R                  S9mUU 4S jn[        R                  " T R                  5       T R                  5       U[        U5      S9nU$ )	Nr   r(   )divisor_overridec                F    [        X-  [        R                  " U5      5      $ r   )r"   r   r  rQ  s      r[   r1  0upsample_nearest2d_backward.<locals>.start_index;  s    ug(>??r]   c                   > T" U S-   X5      $ r  r   )r-  r/  r0  r1  s      r[   r2  .upsample_nearest2d_backward.<locals>.end_index>  s    EAI99r]   rW  c                (   > T" U [        T5      5      $ r   )r-  )r  rZ  r   s    r[   rs   'upsample_nearest2d_backward.<locals>.fnJ  s    c.q122r]   rR  )rM  r   rJ   rS   r  r  r]  r=   rF  rI   r   r7   r   rd   r   r_   )r   r  r  r]  r^  r  inp_hinp_wout_hout_wra  rb  r2  rs   rc  rZ  r1  s   `              @@r[   upsample_nearest2d_backwardr  )  s    NNJJLVEGG2259EGG2259E&VE}emq0!enen=PQRR5(L5(L@: ""1.77F3 
		||~kkmJ	
B Ir]   c                "    [        U UUUUUUSS9$ )Nr&   r  _avg_poolndr   r  r5  r|  r  count_include_padr  s          r[   r]  r]  _  )     		 	r]   c                "    [        U UUUUUUSS9$ )Nr   r  r  r  s          r[   
avg_pool3dr  u  r  r]   c                  ^^^^^^^^^^^ T(       d  TmT(       d  S/T-  m[        TT5      m[        TT5      m[        TT5      m[        U [        5      (       d   e[        T5      T:X  d   e[        T5      T:X  d   e[        T5      T:X  d   e[        U R	                  5       5      TS-   TS-   4;   d   eU R                  5         U R	                  5       S T*  nU R	                  5       T* S  m[        [        T5       V	s/ s H  n	[        TU	   U	TTTU5      PM     sn	6 u  p[        T5      (       d  [        U5      (       a  [        U STS9mSnOU R                  5       mSn[        U5      [        U
5      -   nU R                  5       m[        R                  " [         R"                  T5      nUS:  a6  TS:X  a  [$        nOTS	:X  a  [&        nO[)        S
T 35      eU" U TTTUTU5      $ UUUU4S jmU(       a  U(       a5  U(       a  UOUmTR*                  (       a  ST-  mUUUU4S jnOUUUU4S jnOUUUUUUUUU4	S jn[,        R.                  " U R1                  5       TUUS9nU$ s  sn	f )Nr   r(   r&   rG  r  TFr  r   zUnknown dim: c           	     f  > U S T	*  nU T	* S  nS n[         R                  " [        T	5       Vs/ s H  n[        T
U   5      PM     sn6  HZ  n[        T	5       Vs/ s H  oSU   TU   -  Xe   -   TU   -
  PM     nnU" / UQUQ5      nUc  UnMD  [        R                  " X5      nM\     U$ s  snf s  snf r   )r   r:  r   rI   r   )r  r  r  r!  totalri   r  r   r  r   r  r|  r5  s            r[   rZ  _avg_poolnd.<locals>.fn_sum  s    UsdJ##U3Z%PZeKN&;Z%PQBBG*M*QQ4&)#be+gaj8*CM6C)C}+ R  &QMs   B)B.c                j   > [         R                  " T" U T5      [         R                  " TT5      5      $ r   )rI   rS  rJ  )r  r   rZ  rg  r  s    r[   rs   _avg_poolnd.<locals>.fn  s&    wwvc84cll5%6PQQr]   c                j   > [         R                  " T" U T5      [         R                  " TT5      5      $ r   )rI   truncdivrJ  )r  divisorr   rZ  r  s    r[   rs   r    s'    ||F3$93<<QV;WXXr]   c                h  >	 U T	* S  n/ n[        T	5       H  nX   TU   -  TU   -
  n[        R                  " UTU   -   TU   TU   -   5      nT(       d1  [        R                  " US5      n[        R                  " UTU   5      n[        R
                  " XT-
  [        R                  5      nUR                  U5        M     [        R                  " [        R                  U5      nT
R                  (       a  [        R                  " T" U T5      U5      $ [        R                  " T" U T5      U5      $ r  )r   r   MinMaxrI   rH  rp   r  re   r   r  rS  r  rY  r  )r  r  divide_factorsri   hstarthendfactordivide_factorr  r   r   rZ  r  r  r|  r5  r  s           r[   rs   r    s    cTUBN3Z*WQZ7yy+a.!8!A$:KL("YYvq1F 99T1Q40Du{{C%%f-   &,,SWWnEM&&{{6#x#8-HH<<sH 5}EEr]   rR  )rD   ra   r:   r   r   rM  r   r   r  r	  r  rT  r_   r   r   r  r  rS  fallback_avg_pool2dfallback_avg_pool3dru  r  r7   r   rd   )r   r  r5  r|  r  r  r  r   rO  ri   r5  
ceil_modeshad_paddingr  r  fallbackrs   rc  r  r   rZ  r  rg  r  s    ``` ` `          @@@@@@r[   r  r    sc    #){C0K&#&F7C(Ga####{s"""v;#w<3qzz|q#' 2222NNJJL3$E	

cTUA 3Z

 1q+vw	J

E 7||s:.q#3?==?E{T%[(HKKME""8<<=KR!8*HAX*H}SE233
 	
  *&6"K""KER R
Y Y	F 	F$ 
		||~	
B Iu

s   I-c                (  ^^^^^^^^^^^^^^ Tb  TS:w  d   S5       eT(       d  TmT(       d  SS/m[        U [        5      (       d   e[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         UR                  5       Gt nmm[        TSTTTU5      u  p[        TSTTTU5      u  pU R                  5       mTS   =(       d    TS   =(       d    U
=(       d    UmU R                  5       Gt nmm[        UR                  5       5      nUR                  5       n[        UU4S j[        TS   S-  5       5       5      m[        UU4S j[        TS   S-  5       5       5      mTT-  nUS:  a  [        U UTTTUTT5      $ UUUUU4S	 jmUUUUUUUUUUUU4S
 jn[        R                  " U R                  5       UUUS9nU$ )Nr   divisor must be not zeror&   r  r(   c           
   3  z   >#    U  H0  n[        UTS    -  [        S UTS    -
  TS    -  5      -
  S5      v   M2     g7fr  r  r  s     r[   r   &avg_pool2d_backward.<locals>.<genexpr>3  r  r  c           
   3  z   >#    U  H0  n[        UTS    -  [        SUTS    -
  TS    -  5      -
  S 5      v   M2     g7fr  r  r  s     r[   r   r  7  r  r  r  c           	     n  > [         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " [         R
                  " X5      U5      n[         R                  " [         R
                  " X5      U5      n	[         R                  " [         R                  " X5      [         R                  " [         R                  " T[        R                  5      U5      5      n
[         R                  " [         R                  " X5      [         R                  " [         R                  " T[        R                  5      U5      5      n[         R                  " U[         R                  " S[        R                  5      5      n[         R                  " U	[         R                  " S[        R                  5      5      n	[         R                  " U
[         R                  " T[        R                  5      5      n
[         R                  " U[         R                  " T[        R                  5      5      n[         R
                  " [         R                  " X5      [         R                  " X5      5      nU$ )zc
This computes the scaling factor that we will divide an element
by when `count_include_pad=False`
r   r(   )
rI   rJ  rp   r  r|  rS  re  r   rH  rd  )r  r  stride_hstride_wpad_hpad_wkernel_hkernel_wr  wstartr  wendr  heightr  r|  r5  r  s                r[   !compute_pool_size_without_padding>avg_pool2d_backward.<locals>.compute_pool_size_without_paddingJ  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<.6.6{{GGF%GGCNN65;;7?
 {{GGF%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CD 5swwt7LMr]   c                  > U Gt pnUTS   -   nUTS   -   n[         R                  " [        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [        UTS   5      S-   [        R                  5      n[         R                  " [        UTS   5      S-   [        R                  5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      nS n[        T5       GHD  n	[        T5       GH0  n
[         R                  " U[         R                  " U	[        R                  5      5      n[         R                  " U[         R                  " U
[        R                  5      5      nTb  TnO"T(       d  T(       d  TS   TS   -  nOT" X5      n[         R                  " T" / UQ[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TSS9P[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TSS9P5      U5      n[         R                  " [         R                  " X5      [         R                  " X5      5      nUc=  [         R                  " X[         R                  " S[        R                   5      5      nGM  [         R                  " U[         R                  " X5      U5      nGM3     GMG     Uc   eU$ )Nr   r(   Fr  rG  )rI   rH  r#   rp   r  rd  rJ  re  r   r   rY  r  r|  rM  rL  r  rb  )r  r  r  r  r   r  r  r  r  r  r  r  r  rg  partrT  r  r  r  r  r  r  r  r|  r  r  r5  r  s                   r[   rs   avg_pool2d_backward.<locals>.fnf  s   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=bEE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2%FF2% #"yyS\\#u}}5UVH"yyswwx/FQHS , (V ###r]   rR  )ra   r:   r   r   rM  r  rT  r_   r   r  r   fallback_avg_pool2d_backwardr7   r   rd   )r  r   r  r5  r|  r  r  r  r   _h_out
ceil_mode1_w_out
ceil_mode2r  r   r  rs   rc  r  r  r  r  r  r  r  r  r  s     ``` ``          @@@@@@@@@r[   avg_pool2d_backwardr  
  s"    #'71'<X>XX<a&k9----a####{q   v;!w<1qzz|&&&

Q%;F &eQVWiXF))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME {1~)* M  {1~)* M
  -/KR+	
 		
 8? ? ?B 
		%%'	
B Ir]   c                  ^^^^^^^^^^^^^^^^^  Tb  TS:w  d   S5       eT(       d  TmT(       d  / SQm[        U [        5      (       d   e[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         UR                  5       Gt nmmm [        TSTTTU5      u  p[        TSTTTU5      u  p[        T STTTU5      u  pU R                  5       m[        T5      =(       d    U
=(       d    U=(       d    UmU R                  5       Gt nmmm[        UR                  5       5      nUR                  5       nUU4S j[        S5       5       u  mmmTT-  T-  nUS	:  a  [        U UTTTUTT5      $ UUUUUU 4S
 jmUUUUUUUUUUUUUU4S jn[        R                  " U R                  5       UUUS9nU$ )Nr   r  )r   r   r   r   )r   r   r(   r&   c              3  r   >^#    U  H+  m[        UUU4S  j[        TT   S-  5       5       5      v   M-     g7f)c           
   3  z   >#    U  H0  n[        UTT   -  [        S UTT   -
  TT   -  5      -
  S5      v   M2     g7fr  r  )r   r  ri   r  r5  s     r[   r   0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>  sH      
. VAYQ[^);q	(I!JJANN.r  r&   N)r  r   )r   ri   r  r5  s    @r[   r   &avg_pool3d_backward.<locals>.<genexpr>  sA      3
 A	 	 
;q>A-.
 	
 	
 s   37}   c           	       > S T 5       u  p4nS T 5       u  pgnS T 5       u  pnS [        XU/X4U/XgU/5       5       u  pnS [        XU/XU/TTT/XgU/5       5       u  nnnS XU4 5       u  pnS [        UUU/TTT/5       5       u  nnn[        R                  " [        R                  " [        R                  " X5      [        R                  " UU5      5      [        R                  " UU5      5      nU$ )Nc              3  l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fr   rI   rJ  rp   r  )r   r  s     r[   r   Qavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s"     'UfQ(D(Dfr)  c              3  l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fr   r  r  s     r[   r   r    s"     MWs||Au{{;;Wr)  c              3  l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fr   r  )r   r   s     r[   r   r    s%      (
2=QCLLEKK((+r)  c              3  ~   #    U  H3  u  pn[         R                  " [         R                  " X5      U5      v   M5     g 7fr   )rI   r|  rS  )r   ru  r  pads       r[   r   r    s5      "
	c GGCGGAM3''s   ;=c           
   3     #    U  Hk  u  pp4[         R                  " [         R                  " X5      [         R                  " [         R                  " U[        R
                  5      U5      5      v   Mm     g 7fr   )rI   re  r   rH  rp   r  )r   r(  r   r   r  s        r[   r   r    sV      

'"# KK!3773>>#u{{+KS#Q 's   A3A5c              3     #    U  H?  n[         R                  " U[         R                  " S [        R                  5      5      v   MA     g7fr  rI   rd  rJ  rp   r  )r   r(  s     r[   r   r    s4      "
1 KKs||Au{{;<<1   AA	c              3     #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fr   rI   re  rH  rp   r  )r   r)  r   s      r[   r   r    s6      
K KKS^^C=>>K   A	A)r   rI   rS  r|  )pdr  r  stride_dr  r  pad_dr  r  kernel_dr  r  dstartr  r  dendr  r  r  depthr  r  r|  r5  r  s                      r[   r  >avg_pool3d_backward.<locals>.compute_pool_size_without_padding  s.   'Uf'U$HMWMe(
2=(
$H"
 x8<uU>S"


 '*(X.&u%	'	

dD"
 &1"

tT 2UFE4JK
dD GGCGGD)3774+@A3774QWCX
 r]   c                R  > U Gt pp4S [        X#U/T5       5       u  p#nS [        X#U/TT!5       5       u  pVnS [        X#U/T!5       5       u  pn
S XVU4 5       u  pVnS [        XU
/TTT /5       5       u  pn
S n[        T5       GH  n[        T5       GH  n[        T"5       GHo  nS [        XVU/XU/5       5       u  nnnTb  TnO*T(       d  T(       d  TS   TS   -  TS	   -  nO
T" UUU5      n[        R                  " T" / UQ[        R                  " [        R
                  " U[        R                  " U[        R                  " S[        R                  5      5      5      TS
S9P[        R                  " [        R
                  " U[        R                  " U	[        R                  " S[        R                  5      5      5      TS
S9P[        R                  " [        R
                  " U[        R                  " U
[        R                  " S[        R                  5      5      5      T S
S9P5      U5      n[        R                  " [        R                  " [        R                  " X5      [        R                  " UU	5      5      [        R                  " UU
5      5      nUc>  [        R                  " UU[        R                  " S[        R                  5      5      nGMB  [        R                  " U[        R                  " UU5      U5      nGMr     GM     GM     Uc   eU$ )Nc              3  .   #    U  H  u  pX-   v   M     g 7fr   r   )r   r   r  s      r[   r   2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s     A)@vq17)@r  c              3     #    U  H<  u  pn[         R                  " [        X-
  U-   U5      [        R                  5      v   M>     g 7fr   rI   rH  r#   rp   r  )r   r   r   r  s       r[   r   r  !  s:      %
>a NN8AEAIq15;;??>s   AAc              3     #    U  H8  u  p[         R                  " [        X5      S -   [        R                  5      v   M:     g7fr  r   )r   r   r  s      r[   r   r  &  s4      
. NN8A>A-u{{;;.s   A Ac              3     #    U  H?  n[         R                  " U[         R                  " S [        R                  5      5      v   MA     g7fr  r  )r   pstarts     r[   r   r  +  s4      %
5 KKQ <==5r  c              3     #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fr   r  )r   pend
pooled_dims      r[   r   r  /  s;      
%  KKcnnZEFF%r  c              3     #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fr   )rI   r   rJ  rp   r  )r   r  p_s      r[   r   r  ;  s:      "+JF R(EFF+r  r   r(   r&   Fr  rG  )r   r   rI   rY  r  re  r|  rJ  rp   r  rM  rL  r  rb  r   )#r  r  r  r  r  pdstartr   r  pdendr  r  r  pd_r  r  r  r  r  rg  r  rT  r  r  d_window_sizer  r  r  r  r  r|  pooled_depthr  r  r5  r  s#                        r[   rs   avg_pool3d_backward.<locals>.fn  s   AAaAY)@Aa%
ay+v>%
!'

Q1Iv.
e
%
"W5%
!'
$'u%m\'R%
e 'C]+ /C"*-$w7#C+"JBB (3 0*+ +AQ ?+a. P A"b" M;;#!' # 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!" !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %2*/!"  !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!"!4 7D< 88!2CFF2u4EFr5)D  '#&99 $S%--(H$ $'99T3778T3JH#Um 0 , (r ###r]   rR  )ra   r:   r   r   rM  r  rT  r	  r_   r   r   fallback_avg_pool3d_backwardr7   r   rd   )!r  r   r  r5  r|  r  r  r  r  _d_outceil_mode_dr  ceil_mode_hr  ceil_mode_wr   r  r   r  rs   rc  r  r  r  r  r  r  r  r  r  r  r  r  s!     ``` ``             @@@@@@@@@@@@r[   avg_pool3d_backwardr    s    #'71'<X>XX<k9----a####{q   v;!w<1qzz|&&&$%JJL!VUFE&q+vw	F ';F 'q+vw	F ))+Kg,K+KKK4?4H4H4J1Qm\AJJL!HKKME3
 q3/M=-  -/-?KS+	
 		
# #JU U Un 
		%%'	
B Ir]   c                F   U R                  5       n[        U[        5      (       a  U/nOU(       d  [        [	        U5      5      n[	        U5      S:X  a  [        U5      S;   d
   SU 35       e/ $ [        U5      n[        [	        U5      5       Hg  nX   S:  a'  X==   [	        U5      (       a  [	        U5      OS-  ss'   SX   s=::  a  [	        U5      :  a  ML  O  [	        U5      S:X  a
  X   S:X  a  Mg   e   [	        [        U5      5      [	        U5      :X  d   S5       eU$ )Nr   )r   r  rA  zinvalid axis: r(   zreduction axis not unique)r   ra   r   r   r   r   r_   r!   )r   rm  r   ri   s       r[   _validate_reduction_axisr  }  s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t97Q;GCIIs4y14GDG'c$i'CINtw!|LL  z$ CI-J/JJ-Kr]   c          	       ^^
^^^ Ub  [        X5      n U R                  5       m[        [           " [	        X5      5      n/ n/ m/ n/ m[        [        T5      5       HT  nXq;   a'  TR                  U5        UR                  TU   5        M/  TR                  U5        UR                  TU   5        MV     U
UUUU4S jnT(       a1  [        T5      n	T H  n[        R                  R                  X'   M!     OUn	U R                  5       m
[        U R                  5       U=(       d    U R                  5       U R                  5       UU	US9$ )Nc                  > [        U5      [        T	5      :X  d   eT(       a.  [        U 5      [        T
5      :X  d   eT Vs/ s H  o U   PM	     n n[        U 5      [        T5      :X  d   eS /[        U 5      [        U5      -   -  n[        R                  " [        TU 5      [        T	U5      5       H	  u  pEXSU'   M     T" U5      $ s  snf r   )r   r   r   r   )r-  reduction_indexri   r  r  varinner_loaderkeepdimskept_idxreduced_idxr   s         r[   r  %_make_reduction_inner.<locals>.loader  s    ?#s;'7777u:T***'/0x!1XxE05zS]***Fc%j3+??@	!% #k?"C
HC !cN
 I&& 1s   C)rl   r  r|  rP  rO  r  )r   r   r!   r   r  r   r   re   r_   r   r  r  rT  r   rd   r   )r   rm  r  r   r6  
kept_sizesreduced_sizesri   r  r  r  r  r  r   s     `       @@@@r[   _make_reduction_innerr#    s   Q::<Dc?3A<=DJHMK3t99q!  a)OOAd1g& ' ' :A''++HK  ==?L||~'81;;=++-& r]   c                $   ^ ^ SS S.UU 4S jjjnU$ )Nr   c                  > [        U UUUTS9n[        R                  " STU S.UD6n[        UR                  R                  [        5      (       a  UR                  5         U$ )Nrm  r  r   r6  )r/  r  r   )r#  r8   r   ra   rc   rk  )r   rm  r  r   r   r>  r6  r/  s         r[   r_  make_reduction.<locals>.inner  sg    &"7
 !!XAXQWXKK
 
 NNr]   r   r   )r/  r6  r_  s   `` r[   make_reductionr(    s    T    Lr]   c                   Ub  [        X5      n [        X5      n[        U R                  5       U R	                  5       4U R                  5       4U R                  5       US9$ )N)rl   dtypes	inner_fnsr   rm  )r   r&  r   rd   r   rT  r   )r   rm  r   s      r[   _make_scan_innerr,    sV    Q!D||~==?$ZZ\ r]   r   c                 ^ Ub  [        X5      n U R                  5       m[        X5      nU R                  5       nU[        R
                  [        R                  4;   a  [        U [        R                  5      n [        XU5      n[        U4S jU 5       5      n[        R                  " X`R                  5       U R                  5       S9n[        R                  " U[        UR                  5       5      5      n[        [!        XV5      U5      $ )Nc              3  .   >#    U  H
  nTU   v   M     g 7fr   r   r   ri   r   s     r[   r   mean.<locals>.<genexpr>       04a$q'4r   r,  )r   r   r  r   rp   rW  rV  r'  sum_rG   r+   r1   rd   r0   r   r_   div)r   rm  keepdimr   output_dtype
sum_resultdenomr   s          @r[   r2  r2    s    Q::<D#A,D;;=Lu~~66Q$aw'J0400Ee;;=XEeT**=*=*?%@AEC
*L99r]   c                F  ^
 Uc  SnU R                  5       m
[        X5      n[        XSS9nU(       a  UR                  5         [	        [        X5      5      n[        XaU5      n[        U
4S jU 5       5      nU(       a  [        R                  " X-
  S5      n[        R                  " XR                  5       U R                  5       S9n[        R                  " U[!        UR                  5       5      5      n[#        Xx5      n	U(       d  U	4$ U(       a  UO
[%        XQ5      nX4$ )Nr(   T)r4  c              3  .   >#    U  H
  nTU   v   M     g 7fr   r   r/  s     r[   r    var_mean_sum_.<locals>.<genexpr>   r1  r   r   r,  )r   r  r2  rk  squarer|  r2  rG   r   r  r+   r1   r   rd   r0   r   r_   r3  r  )r   rm  
correctionr4  return_meanx_meandiffsr6  r7  x_varr   s             @r[   var_mean_sum_rA    s    
::<D#A,D!4(F3q>"Ee7+J0400E		%,a0e;;=XEeT**=*=*?%@AE
"ExVGF$9F=r]   c                    [        X5      n[        XUS S S9nUS   n[        US   5      n[        U[        R
                  5      =(       a1    [        U5      [        R                  :  =(       a    [        U5      S:g  $ )Nr&  rO  r  r(   )	r  r#  rG   ra   r   rM  r   r)   r  )r   rm  r4  r   rO  reduction_numels         r[   use_two_step_variancerD    sx    #A,D"	wd$F HF#F+=$>?O?EMM2 	' 6#E#EE	'&!Q&r]   c                 ^^^^^ Tc  Sm[        XUS S S9nUR                  S5      nUR                  S5        UR                  S5        [        R                  R                  " SU4SU R                  5       S.UD6u  pxn	UR                  5         U R                  5       mU R                  5       m[        X5      n[        U4S jU 5       5      mS	 mUUUU4S
 jn
[        U
5      " U5      nU(       a  UR                  5         X4$ U4$ )Nr(   r&  rP  r  r|  welford_reduce)r+  r/  r   c              3  .   >#    U  H
  nTU   v   M     g 7fr   r   r/  s     r[   r   $var_mean_welford_.<locals>.<genexpr>3  s     1Dq47Dr   c                   [        U [        R                  5      (       aK  U R                  (       d:  [        R
                  " [        R                  " U [        R                  5      U5      $ [        R                  " X5      $ r   )
ra   r   r   	is_numberrI   r   rH  rp   rI  rJ  r.  s     r[   get_constant_or_index_expr5var_mean_welford_.<locals>.get_constant_or_index_expr5  sH    a$$Q[[<<q%++ >FF||A%%r]   c                   > T" TT5      nT" TT5      n[         R                  " ST5      nU [         R                  " X2U-
  5      -  $ r  )rI   rJ  rd  )rc   cNzeror<  r   rK  rnumels       r[   rF  #var_mean_welford_.<locals>.scale_fn:  sC    &z59&vu5||Au%ckk$A...r]   r   )r#  r  r+   WelfordReductionr   r   rk  r   r  rG   ra  )r   rm  r<  r4  r=  r   r  r2  m2r   rF  r  r   rK  rQ  r   s     `         @@@@r[   var_mean_welford_rU    s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KDa JJLKKME::<D#A,D1D11F&
/ / 
"2
&Cy6Mr]   c                  ^ U R                  5       m[        T5      n[        XSS9n [        U UUUUS9n[	        XUS9(       a  [        S0 UD6O
[        S0 UD6n[        U4S jU 5       5      nU(       d  US   $ U$ )NFr  )r   rm  r<  r4  r=  )rm  r4  c              3  :   >#    U  H  n[        UTS S9v   M     g7f)Fr  N)r   )r   r   rw  s     r[   r   #var_mean_helper_.<locals>.<genexpr>X  s     Fv!8Ayu5vr  r   r   )r   r   r   r   rD  rA  rU  r   )	r   rm  r<  r4  r=  compute_dtyper   r"  rw  s	           @r[   var_mean_helper_rZ  H  s    I))4M.A
F !w? 	(( 
 FvFFF'6!93V3r]   )r<  r4  c                   [        XX#SS9$ )NFrm  r<  r4  r=  rZ  r   rm  r<  r4  s       r[   var_r_  \  s    	% r]   c                   [        XX#SS9$ )NTr\  r]  r^  s       r[   var_meanra  c  s    	$ r]   c                (   US:  a"  [        [        R                  " U 5      U* U5      $ US:X  a  [        R                  " SU5      $ US:X  a  U $ [        XS-  U5      n[        R                  " X35      nUS-  S:X  a  [        R                  " X05      nU$ )Nr   r(   r&   )pow_recursiverI   rc  rJ  rS  )r   r#  r   r>  s       r[   rc  rc  j  s    1uS^^A.E::Av||Au%%Av11fe,FWWV$F	A!|#Mr]   c                .    [         R                  " X5      $ r   )rI   powr   r!  s     r[   
pow_nativerg  y  s    771=r]   )r   c                   ^ ^^ [        T[        5      (       a$  T[        T5      :X  a  [        T [        T5      5      $ [        T[        5      (       a  TS:X  a  [	        T 5      $ [        T[        5      (       a  TS:X  a  [        T 5      $ [        S T T4 5       5      n[        U5      n[        T[        5      =(       a*    STs=:  =(       a    S:  Os  =(       d    U=(       a    TS:  nU(       aY  T R                  5       mU UU4S jn[        R                  " T R                  5       T R                  5       UT R                  5       S9$ [        T [        5      (       aA  T S:X  a  [        TS5      $ T S	:X  a)  [!        TR                  5       5      (       a  [#        T5      $ U(       aN  [        T [        5      (       a  [%        T T5      $ [        T[        5      (       a  ['        T T5      $ [)        T T5      $ [+        T T5      $ )
NrD  r(   c              3     #    U  H6  n[        U[        R                  5      (       d  M$  UR                  5       v   M8     g 7fr   )ra   r+   r:   r   r  s     r[   r   pow.<locals>.<genexpr>  s'     N1*Q2Ms
   #A A i    r   c                F   > [        T" U 5      TTR                  5       5      $ r   )rc  r   )r  r   r!  r  s    r[   rs   pow.<locals>.fn  s     a??r]   rR  r&   )ra   r'  r   re  sqrtr  r3  r   rT  r7   r   rd   r   r   r   r  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorrg  )r   r!  r   is_integer_powembed_exponentrs   r  s   ``    @r[   re  re    s   !USV1c!f~	Au		!s(Aw	As		QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]337Na  &q!,,6""-a33-a33ar]   c                (   [        U [        5      (       a  U R                  nOU n[        U[        5      (       a  UR                  n[        U[        R                  5      (       dz  [
        R                  " U R                  5       U R                  5       UR                  5       U R                  5       S9R                  n[        U[        R                  5      (       d   e[        U[        R                  5      (       av  UR                  5       (       da  UR                  5       (       dL  [        UR                  [        R                  5      (       d#  UR                  5         UR                  Ul        U $ [        R                  R!                  XUS9  U $ )NrR  unsafe_alias)ra   r:   rc   r+   r  r7   r   rd   r   rT  r   is_input_bufferis_module_buffer	NopKernelrk  r  realize_into)changedr  rw  changed_datas       r[   r  r    s,   '9%%||#y!!hhc2==))%%'##%__&##%	

 $ 	 #r}}----,..$$&&((**l''66 	HH!!.. /  Nr]   c                ,    [        U [        X5      5      $ r   )r  r  )r   rl  s     r[   r{  r{    s    Q	!011r]   c                    XL a  U $ [        XR                  5       5      n[        XR                  5       5      n[	        XR                  5       5      n[        X5      $ r   r  rd   r   r   r  r   r  )r  r  r  s      r[   r  r    sH    
z

C)
*C
3
(C
lln
%CSr]   c                .    [         R                  " X5      $ r   )rI   floordivrf  s     r[   r  r        <<r]   c                .    [         R                  " X5      $ r   )rI   r  rf  s     r[   r  r    r  r]   c                t   [        U 5      =(       a    [        U5      n[        U 5      =(       a    [        U5      nUS:X  a4  U(       a   S5       eU(       a  [        X5      $ [        [	        X5      5      $ US:X  a4  U(       a   S5       eU(       a  [        X5      $ [        [	        X5      5      $ [	        X5      $ )Nr  z5floordiv operands can not be boolean at the same timer  z5truncdiv operands can not be boolean at the same time)r   r   r  r  r3  r  r  )r   r!  rounding_modeboth_integerboth_booleans        r[   div_moder    s    "1%</!*<L"1%</!*<L X!XX!-x~C5Q3CCX!XX!-x~C5Q3CCq9r]   c                    [        U 5      =(       a    [        U5      nU(       a  [        X5      $ [        [        R                  R
                  5      n[        U5      " X5      $ r   )r   logical_andr5   r  rS  r  ra  )r   r!  	both_boolrs   s       r[   rS  rS    sH    "9q'9I1  **+b!!''r]   c                   [        U [        R                  5      (       a  [        U R                  5      $ [        U [        R
                  5      (       a  [        U R                  5       5      $ [        U [        R                  5      (       a  U $ [        U [        R                  5      (       d  g[        R                  R                  R                  U R                  5       5      n[        R                  " U5         [         R"                  " [        R$                  SS5         U R&                  " U R)                  5       6 nSSS5        SSS5        [        W[        R                  R*                  R,                  5      (       d   e[        UR.                  [        R                  5      (       a  UR.                  $ g! , (       d  f       N= f! , (       d  f       N= f)z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)ra   r+   
MutableBoxget_constant_valuerc   r/  r0  r   Loopsrp   	_inductorops_handlerExtractConstantsHandlerrd   rJ   set_ops_handlerr   objectrl  rP  inner_fn_argsvirtualizedOpsValuer   )r   r$  rg   s      r[   r  r    s1   
 !R]]##!!&&))!R[[!!!!--/22!R[[!! a""oo))AA!,,.QG	'"R&&(8$?jj!//+, 	@ 	#
 c5??66??@@@@#))R[[))yy 	@? 	#"s$   ;'G"F; G;
G		G
Gc                :   [        S X4 5       5      nU(       a  [        X5      $ [        U5      =nbU  UR                  S:X  a+  [        R
                  " [        S5      UR                  5      nOSUR                  -  n[        X5      $ S n[        U5      " X5      $ )Nc              3  \   #    U  H"  n[        U5      =(       d    [        U5      v   M$     g 7fr   )r   r   r  s     r[   r   div_prim.<locals>.<genexpr>0  s"     O1oa(>OA,>>s   *,r   infr  c                 (    [         R                  " U 6 $ r   )rI   rY  r  s    r[   rs   div_prim.<locals>.fn=  s    {{D!!r]   )	r  r  r  r   mathcopysignr'  rS  ra  )r   r!  is_integralr  rc  rs   s         r[   div_primr  .  s    OOOK~%a((5==AuU|W]]CJw}},J1!!" "a##r]   c                N    [        X4[        R                  S9u  p[        X5      $ rI  )r8  r   INT_TO_FLOATr  rf  s     r[   r3  r3  C  s*     	
$C$P$PDA A>r]   c                z    [        U 5      =(       d    [        U 5      nU(       a  S nOS n[        U5      " X5      $ )Nc                .    [         R                  " X5      $ r   )rI   modrf  s     r[   rs   fmod.<locals>.fnU  s    771= r]   c                .    [         R                  " X5      $ r   )rI   fmodrf  s     r[   rs   r  Z  s    88A>!r]   )r   r   ra  )r   r!  r  rs   s       r[   r  r  O  s5    !!$:(:K	!
	" "a##r]   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        SUS9nU" XX#S9$ )Nr$  r  r   r   r   r   rp   rI  r(  r   rm  r  r   rs   s        r[   r2  r2  `  sM     	''+;AKKM+J+J
-	U	;Bax--r]   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        U R                  5       5      S:X  a+  US;   d   eU=(       d    U R                  5       n[        XSS9$ S n[        XUS9n[        R                  R                  " S	0 UDSU0D6u  nUc
  [        XUS9$ U$ )
Nr   r   rA  Tr  c                @    U u  nUu  n[         R                  " X#5      4$ r   )rI   r   a_tupleb_tupler   r!  s       r[   
combine_fncumsum.<locals>.combine_fn~  !    r]   rm  r   r  r   r   r   )r   r   r   rp   rI  r   r   r   r,  r+   Scanr   fallback_cumsumr   rm  r   r  r   r>  s         r[   cumsumr  r  s     	''+;AKKM+J+J
-
1::<Aw&t,, 
 a%8F??J?IV~q%88Mr]   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        U R                  5       5      S:X  a+  US;   d   eU=(       d    U R                  5       n[        XSS9$ S n[        XUS9n[        R                  R                  " S	0 UDSU0D6u  nUc
  [        XUS9$ U$ )
Nr   r  Tr  c                @    U u  nUu  n[         R                  " X#5      4$ r   )rI   rS  r  s       r[   r  cumprod.<locals>.combine_fn  r  r]   r  r  r  r   )r   r   r   rp   rI  r   r   r   r,  r+   r  r   fallback_cumprodr  s         r[   cumprodr    s     	''+;AKKM+J+J
-
1::<Aw&t,, 
 a%8F??J?IV~599Mr]   c                   S nU R                  5       n[        U R                  5       5      S:X  a  US;   d   e[        U 5      $ [	        XUS9n[
        R                  R                  " S0 UDSU0D6u  nUc	  [        XS9$ U$ )Nc           	     .   U u  nUu  n[         R                  " X#5      n[         R                  " X#5      nXE:g  [         R                  " U5      ) -  n[         R                  " U[         R
                  " [         R                  " XE-
  5      5      U-   U5      4$ r   )rI   re  rd  r  r  log1pexp)r  r  r   r!  min_vmax_vrT  s          r[   log_add_exp_helper(logcumsumexp.<locals>.log_add_exp_helper  ss    A!A!CIIe$4#45		$		#''%-*@ AE I1MOOr]   r   r  r  r  r  r   )	r   r   r   r  r,  r+   r  r   fallback_logcumsumexp)r   r   r  r   r   r>  s         r[   logcumsumexpr    s{    P KKME
1::<Ag~~Qxa7FGG4FGIV~$Q00Mr]   c                  ^ [        U R                  5       5      S:X  a+  TS;   d   e[        U 5      [        U [        R
                  S94$ U R                  5       n[        R                  " SUSS9n[        U TUS9nU[        R
                  4US'   U R                  5       U4S	 j4US
'   [        R                  R                  " S0 UDSU0D6u  pVUc
  [        U TS9$ XV4$ )Nr   r  r   r  Fr   arg_break_ties_leftr  r*  c                T   > [         R                  " U T   [        R                  5      $ r   rI   rH  rp   rI  r  rm  s    r[   rE  cummax.<locals>.<lambda>      CNN3t9ekk:r]   r+  r  r  r   )r   r   r  r  rp   rI  r   r+   get_reduction_combine_fnr,  rT  r  r   fallback_cummaxr   rm  r   r  r   r   r  s    `     r[   cummaxr        
1::<AwQxAU[[999KKME,,5J ad%8Fu{{+F8	:F; ggnnEvE*EOF~qd++?r]   c                  ^ [        U R                  5       5      S:X  a+  TS;   d   e[        U 5      [        U [        R
                  S94$ U R                  5       n[        R                  " SUSS9n[        U TUS9nU[        R
                  4US'   U R                  5       U4S	 j4US
'   [        R                  R                  " S0 UDSU0D6u  pVUc
  [        U TS9$ XV4$ )Nr   r  r   argminFr  r  r*  c                T   > [         R                  " U T   [        R                  5      $ r   r  r  s    r[   rE  cummin.<locals>.<lambda>  r  r]   r+  r  r  r   )r   r   r  r  rp   rI  r   r+   r  r,  rT  r  r   fallback_cumminr  s    `     r[   cumminr    r  r]   c                   [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        SUS9nU" XX#S9$ )Nr%  r  r   r  r  s        r[   r%  r%    sM     	''+;AKKM+J+J
-	e	<Bax--r]   c                V    [        U [        R                  5      n [        S5      " XUS9$ )Nr	  rm  r  )r   rp   r   r(  r   r   r4  s      r[   
reduce_anyr    s$    EJJA% w??r]   c                F    Ub  [        XUS9[        XUS94$ [        U S US9$ Nr  )reduce_amaxreduce_argmaxr  s      r[   
reduce_maxr    6    
g6!8
 	

 qtg66r]   c                F    Ub  [        XUS9[        XUS94$ [        U S US9$ r  )reduce_aminreduce_argminr  s      r[   
reduce_minr    r  r]   xor_sumr  r  r  r  r  
logical_or)r\  rN  stabler   
descendingc          
        Uc  SnU R                  5       nU R                  5       n[        [        U5      U5      n[        U5      S:X  a'  [	        U 5      [        SU[        R                  U5      4$ [        U5      (       a  XB   OSn[        R                  R                  R                  U[        R                  " [        R                  5      R                  5      (       d
  [        XX#S9$ [!        USS[        R                  USS9nS/[        U5      -  n[        U5      (       a  XhU'   [#        Xx5      n[%        Xt5      n[&        R(                  R+                  UU R,                  UR,                  4U R/                  5       UR/                  5       4UUUUS9u  pU	c
  [        XX#S9$ Uc   eU	[1        U[        R                  5      4$ )NFr   r(   r  )r(  r*  r   rl   r)  )rl   r*  r+  r   rm  r  r  )r   rd   r   r   r  rm  rp   rI  rJ   rS   r  statically_known_ltr  int16r  sort_fallbackr*  r  r  r+   Sortr   r   rT  r   )
r   r  r   r  r	  rl   r  r  
view_shaper   s
             r[   sort_stabler    s   ~JJLE\\^F
3u:s
+C
5zQQxq&%++u=== ZZuzQH77//%++ekk:R:V:VWWQ3NNVSXG s5z!J
5zz"37'GW$Gggnn'==?G$7$7$9: % OF ~Q3NN8GU[[111r]   c                    [        U SXS9$ )NFr  )r  )r   r   r  s      r[   sortr  F  s    qCGGr]   c                6    [        U U[        R                  US9$ )Nr   r   r^  )r  r   r  )rV   r   r^  s      r[   register_pointwise_numericr  K  s!    
;HH'	 r]   c                \    [        U R                  5        [        U [        R                  S9$ rI  )rF   r  r  r   r  r  s    r[    register_pointwise_numeric_ldf64r  T  s'    '4
;HH r]   r  logical_not)rN  )r   r   r6  identity)ri  pointwise_overrides_datac              #  V  ^#    [         U   m[        U TR                  S 5      nUc  g U4S jn[        U[        R
                  R                  5      (       a:  UR                  5        H%  n[        X$5      nUTR                  U" U5      4v   M'     g UTR                  U" U5      4v   g 7f)Nc                6   > TR                   c  [        U 5      $ g r   )tritonr  )rV   rc   s    r[   make_triton_fallback6_get_pointwise_overrides.<locals>.make_triton_fallback  s    ;;#B'' r]   )	r  r   r   ra   rp   rq   r   r   r   )nsr   rV   r  olnamer[  rc   s         @r[   _get_pointwise_overridesr    s     #D)D	TYY	%B	z( "ejj1122llnF$Bd..0DR0HHH % $**,@,DDDs   B&B)r  c                d   ^ U [         U'   [        R                  U 5        U4S jn[        X5        g )Nc            	     ~   > T" U 0 UD6n/ n[        U S   U5       H  u  pEUR                  [        XESS95        M      U$ )Nr   Trv  )r   re   r  )rj   r   resultsmut_resultsr   r>  outplace_ops         r[   rs   $register_foreach_inplace.<locals>.fn  sK    t.v.tAw0KCy4HI 1 r]   )rP   rg  r   r   )aten_opoutplace_aten_opr  rs   s     ` r[   register_foreach_inplacer  	  s,    07,-G$ w+r]   c                .   ^ [        U S S9U4S j5       nU$ )Nr  c                 j   > T" U 0 UD6n[        X S   R                  5       5      n[        U S   U5      $ r  )r   r   r  )rj   r   r>  r  s      r[   rs   register_inplace.<locals>.fn-  s:    d-f-&q'"3"3"56a&))r]   )r  )r  r  rs   s    ` r[   register_inplacer  ,  s#    wD9* :*
 Ir]   c                    g r   r   r  s      r[   sym_constrain_ranger  V      r]   c                    [         R                  R                  R                  S   n[	        U[
        R                  5      (       d   eUR                  R                  $ r  	rJ   rS   rT   rX  ra   rp   r  rY   r  r   r   r  s      r[   sym_sizer   [  sC    
''


#
#E
*C c5<<((((88==r]   c                    [         R                  R                  R                  S   n[	        U[
        R                  5      (       d   eUR                  R                  $ r  r  r  s      r[   
sym_strider"  l  sA    
''


#
#E
*Cc5<<((((88==r]   c                "    U R                  5       $ r   )r  )r   s    r[   	sym_numelr$  t  s    ;;=r]   c                (    [         R                  " U 6 $ r   )r   Addr  s    r[   sym_sumr'  }  s    99dr]   c                    [        S5      e)NzHelpful for debuggingr   )r  rj   r   s      r[   foobarr)    s    
5
66r]   c                8    U R                  5         [        U 5      $ r   )rk  r  r   s    r[   _realizer+    s    IIK8Or]   c                R    U R                  5         [        R                  " X5        U $ r   )rk  r+   ResizeStorageBytes)variabler  s     r[   resize_storage_bytes_r/    s!    (-Or]   c                    U R                  5         UR                  5         [        R                  " [        R                  " X5      5      $ r   )rk  r:   r   r+   SetSourceTensorKernel)r  source_tensors     r[   set__source_tensorr3    s3    LLNB44TIJJr]   r  c                    XL a  U $ [        XR                  5       5      n[        XR                  5       5      n[	        XR                  5       5      n[        X5      $ r   r  )r  r  s     r[   
fsdp_copy_r5    sH    :J^^-.sMMO,S,,.)""r]   c                 ^	^
^^ [        U [        5      (       d   e[        U[        [        45      (       d   eUc  [        R
                  nU[        R                  :X  a  [        SU 35      eU[        R                  :X  a  [        U5      S:X  d   eU[        R                  :X  a  [        U5      S:X  d   eU R                  5       m
U R                  5       nU R                  5       n[        U R                  [        R                   5      (       a  U R                  R#                  5       U l        [        R$                  " 5       (       ay  [        R&                  R(                  R*                  (       aP  [-        U5      (       a  [/        S5      mO6[1        U5      (       a!  [        R2                  " U5      R4                  mOSmOSm[6        R8                  R:                  R=                  T
S5      (       a  [?        UTX4S9$ [A        U T
/S	/5      nURC                  5       m	[        RD                  RG                  X5      n[        RH                  " XCX5      RK                  5       mU	U
UU4S
 jn[L        RN                  " XCU[        U5      S9nU$ )Nzunsupported memory format: r   r   nanTrG  r   r  r(   c                  >^ T" U 5      m[         R                  " T[        R                  5      n[         R                  " T[        R                  5      n[         R                  " X5      n[         R
                  " UUU4S jT5      $ )Nc                    > T" T /5      $ r   r   )
flat_indexflat_loaders   r[   rE  *resize.<locals>.inner_fn.<locals>.<lambda>  s    ZL(Ar]   )rI   rH  rp   rI  rL  rN  )	r  flat_index_exprlimitrT  r:  r;  	old_numelout_indexeruninitalized_vals	       @r[   rP  resize.<locals>.inner_fn  sX     %
..U[[Ay%++6vvo-zz$ ACSTTr]   rR  )(ra   r:   r_   r   rp   contiguous_formatpreserve_formatrY  channels_lastr   channels_last_3dr  r   r4  rc   r+   r/  r0  r  utilsdeterministicfill_uninitialized_memoryr   r'  r   r  r  rJ   rS   r  r  r  r:  rT  rl   stride_ordered_for_memory_formatr3  rn  r7   r   )r   r   r  r   rl   x_flat
out_striderP  rg   r;  r?  r@  rA  s            @@@@r[   resizerM    s
   a####dT5M****//---8HII+++4yA~~...4yA~~IKKME""$F!&&"++&&##% 	2244KK%%??%  $U|e$$${{5155# ww//	1==D*%GG		
 	
F $$&K""CCDXJ..ANNPKU U 

Xd4jC Jr]   )auto_functionalizedc                    SSK Jn  UR                  U5      n[        R                  " U UU0 UEUES9  UR                  5        VVs0 s H  u  px[        U[        5      (       d  M  Xx_M      snn$ s  snnf )Nr   )kernel_side_table)
kernel_idxgridtma_descriptor_metadatakernel_args)*torch._higher_order_ops.triton_kernel_wraprP  get_constant_argsr+   UserDefinedTritonKernelr   ra   r:   )	rQ  constant_args_idxrR  rS  r   rP  constant_argsr  r  s	            r[   triton_kernel_wrap_rZ    sl     M%778IJM 7/v//	 &,\\^R^z#y7QHCH^RRRs   A/%A/c                n   [        S U /UQ 5       5      (       aY  Sn[        R                  R                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        [        R                  R                  XX#5      n[        [        [        R                  U5      5      $ )Nc              3  f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fr   r>  r  s     r[   r   cond.<locals>.<genexpr>  s%     
M;La:a 1Yq\1;Lr@  z"control flow operator: torch.cond.stack_trace Found from : 
 )r	  rJ   rS   rT   rX  rY  disable_cudagraphs_reasonr+   Conditionalr   r_   mapr:   )predtrue_fnfalse_fnoperandsr   r^  r>  s          r[   r   r     s    

MD;L8;L
MMM2''..3377tLL;LE*;-8C,/)^^""4(EFI$$f-..r]   c                X   [        S X#-    5       5      (       aY  Sn[        R                  R                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        SS jn[        R                  R                  XX#5      n[        [        Xg5      5      $ )Nc              3  f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fr   r>  r  s     r[   r   while_loop.<locals>.<genexpr>  s*      3A 	1f.)A,.3r@  z(control flow operator: torch.while_loop.r^  r_  c                   [        U [        5      (       a  U $ [        U [        R                  5      (       a  [        U 5      $ [        U [        R                  5      (       a  [        R
                  " U 5      $ [        S[        U 5       35      e)NzNYI unsupported output type: )ra   r:   r+   r  MultiOutputr   rY  rZ  )rg   s    r[   _map_outputwhile_loop.<locals>._map_output  sh    c9%%JR]]++S>!R^^,,##C((!>tCykJKKr]   )rg   r   )r	  rJ   rS   rT   rX  rY  r`  r+   	WhileLoopr   r_   rb  )cond_fnbody_fncarried_inputsadditional_inputsr   r^  rl  r>  s           r[   
while_looprs    s    
 3   9''..3377tLL;LE*;-8C,/)L \\  >UFK())r]   c                    [         R                  R                  " U /UQ76 n[        [	        [
        R                  U5      5      $ r   )r+   InvokeSubgraphr   r_   rb  r:   )subgraph_fn
identifierrf  r>  s       r[   invoke_subgraphrx  )  s5    %%k=H=FI$$f-..r]   )schemec               
   S n[         R                  R                  R                  R	                  SS 5      nUc   e[        U R                  R                  R                  5       GH  u  pVUR                  S:X  a!  X%   [         R                  R                  U'   M7  UR                  S:X  Ga  [         R                  R                  U5      u  px[        R                  " XxR                  5       5       H  n	U	R                  5         UR                  (       a7  [         R                  R                   R#                  U	R%                  5       5        [         R                  R&                  R#                  U	R%                  5       5        M     [(        R*                  R,                  R/                  [         R                  XgU5      nGM_  [         R                  R1                  U5      [         R                  R                  U'   GM     U$ )Nquant_optionsr6  r"  )rJ   rS   rT   rX  rY  r`   r  nodesrV   envfetch_args_kwargs_from_envr   r   r   rk  codegen_low_precisionlow_precision_codegen_opsr   rl  invoke_quant_opsrp   r8  Interpreterr"  run_node)
rv  ry  rf  r"  r{  ri   rY   rj   r   r   s
             r[   invoke_quant_tracerr  /  sW   FGG((--11/4HM$$$[55;;AAB77m# (AGGKKWW 77==dCLD__T==?;		 66GG5599!:N:N:PQ((,,Q-A-A-CD < XX))00$fMF ! 0 0 6AGGKK% C( Mr]   c                  ^
 SSK JnJn  [        U5      S:  a  [	        S5      e[
        R                  " X5       Vs/ s H&  nU" UR                  5       UR                  5       S9PM(     nnU" X5      m
U
4S jn[        US   SS S9n[        S U 5       5      US	'   [        S
 U 5       5      US'   [        R                  R                  " SUSS.UD6n	U	S   c  [	        S5      eU	$ s  snf )Nr(   )InputDescriptorlower_pointwise_subgraphr   zSUnable to generate code for associative_scan op, because there are lifted argumentsr  c                h   > T" / [         R                  " U 5      Q[         R                  " U5      Q76 $ r   )r   r:  )lhsrhslowered_combine_fns     r[   wrapped_combine_fn,associative_scan.<locals>.wrapped_combine_fn]  s6    ! 
$
$
 	
r]   r  c              3  @   #    U  H  oR                  5       v   M     g 7fr   )r   r  s     r[   r   #associative_scan.<locals>.<genexpr>d  s     7Bq[[]]B   r*  c              3  @   #    U  H  oR                  5       v   M     g 7fr   r,  r  s     r[   r   r  e  s     <Ar  r+  F)r  can_fallback_to_atenz/Unable to generate code for associative_scan opr   )r  r  r  r   rY  r   r   r   rd   r,  r   r+   r  r   )r  xsrr  r  r  r   subgraph_inputsr  r   r>  r  s             @r[   associative_scanr  L  s     M
!a
 	
 ((A 	akkmALLNC(   2*N
 be!48F7B77F8<<<F;WW^^ %" F
 ayLMMM-s   -C'c                    g r   r   )tokenss    r[   _sink_tokensr  p  r  r]   c                \   [         R                  R                  " U/UQ70 UD6nSSKJn  U" XU5      nUc   e[
        R                  R                  U   nUc  U4$ [        R                  " [         R                  [        R                  U5      n[        U[        5      (       d  Xt4$ U/UQ7$ )Nr   )get_effect_key)r+   EffectfulKernelr   torch._higher_order_ops.effectsr  rJ   rS   effectful_opsr   r  rk  r:   ra   r   )tokenrV   rj   r   r>  r  effect_typeeffectful_kernels           r[   with_effectsr  u  s    &&r;D;F;F> 62K"""ww,,[9~ ""!!"..)2B2BFKF fe$$ )) *6**r]   )register_comm_loweringsc                d   [        XSSSS9nUS   n[        R                  R                  R	                  [        U5      5      n[        R                  R                  " S0 UDSUS.D6u  pVUS:X  aZ  [        R                  R                  R                  U5      [        R                  :  a  [        R                  " SU SUS	.UD6u  pxXx4$ [        R                  " [         R"                  " S
5      5        [%        XSS9n	[&        [(        R*                     " [-        X	5      5      n
[/        XSS9nX4$ )zf
Lowering inductor_prims.prepare_softmax_online to compute max/sum in one pass if no split is needed.
TNr&  r  online_softmax_reduce)r/  rC  r(   r&   )r  
num_outputreduction_hintz
            Online softmax is disabled on the fly since Inductor decides to
            split the reduction. Cut an issue to PyTorch if this is an
            important use case and you want to speed it up with online
            softmax.
            )r  r   )r#  rJ   rS   r  simplifyrG   r+   r8   
num_splitsr  r)   r  r4   r   r(  r)  textwrapdedentr  rN   r  r  r|  r2  )r   r   r   r  rQ  hint	num_split
max_tensor
sum_tensorr3  r  xsums               r[   prepare_softmax_onliner    s$   
 #	d$dF 01WW&&}5E'FGFll-- 
.OD 	QGG&&v.&2T2TT!7!>!> "
Qt"
?E"

 %% 	OO		
 1D1!#a,/Ct,zr]   )r#  )quantized_lowerings)mkldnn_lowerings)jagged_loweringsc              #  v  #    [        U [        R                  R                  5      (       d   S5       e[        R                  U 5      n [        U 5      " [        U 5      5        Sv   U(       a
  U[        U '   g[        R                  U 5        g! U(       a
  U[        U '   f [        R                  U 5        f = f7f)zR
A context manager to force fallback an op. Used in unit test
for FallbackKernel.
z+Only OpOverload to make the clean up easierN)	ra   rp   rq   rr   rN   rY  r  r  r  )rV   old_handlers     r[   force_fallbackr    s      b%**//00 50 --#K".r23'IbMMM" 'IbMMM"s   AB9	B ''B9(B66B9)rf   z%Iterable[Union[tuple[Any, Any], Any]])rs   zCallable[..., Any]r  zOptional[Callable[..., Any]])r   r   )r   r   )rj   z	list[Any]r   zdict[str, Any]r   r   r   )Optional[ELEMENTWISE_TYPE_PROMOTION_KIND]r   r   r  z tuple[list[Any], dict[str, Any]])r   r  )r   r  r  z.Callable[[Callable[_P, _T]], Callable[_P, _T]])NN)NNNFN)F)r   r:   r   torch.dtype)r   r:   rl   torch.devicer   )r   r   l            r(   Tr  )rp  r:   rq  r:   rs  r:   rm  r   rp  r   ro  r   r   r  r  r:   )rp  r:   rq  r:   rs  r:   rm  r   rp  r   ro  r   r   r  rw  Optional[torch.dtype]r  r:   )rp  r:   rg  r'  rh  r   rp  r   ro  r   r   r  r  r:   )rp  r:   rg  r'  rh  r   rp  r   ro  r   r   r  rw  r  r  r:   )rp  r:   rg  r:   rh  r:   rp  r   ro  r   r   r  r  r:   )rp  r:   rg  r:   rh  r:   rp  r   ro  r   r   r  rw  r  r  r:   )r   r   r(   )r  r   r  r   r  r   r  )T)rm   ztorch.Tensor)rY   ztorch.fx.Node)NTF)rl   r  )r   	list[int]ro  r:   r  r  r  r   )
r  r   r  r   r   r  ro  r:   r  r   )r  r:   r  z.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr])r  r:   r  ztuple[str, sympy.Expr])r  r:   r  r:   r  r   r  r   r  Optional[str]r  Optional[TensorBox]r  r:   )rp  r:   r  r:   r  r   r  r   )r   r   r-  r   )r   NNr(   )NNN)rA  FF)r   r   )rJ  r  r   r   r  r  r  r   )r   r   r  r  )r   r   r  r   )r&   F)rN  ztuple[Optional[float], ...]rK  r   rE  r   )rq  Optional[float])r]  r  r^  r  )rd  r  r]  r  r^  r  )r   r:   r|  zSequence[int]rl  r'  r  r  )ri   
sympy.Exprr  zUnion[sympy.Expr, int])ri   r  r  r  )ri   r  r  r  r  r  )Nr  N)
r  r:   r  "Sequence[Union[int, torch.SymInt]]r  r  r  zxCallable[[Sequence[Union[int, torch.SymInt]], Sequence[Union[int, torch.SymInt]]], torch._inductor.virtualized.OpsValue]r  r:   )Nr   r(   F)rG  )NNNN)r   r   FTN)r/  rK   r   )r   r  r  zOptional[ir.Constant])rA  F)rV   ztorch._ops.OpOverloadPacket)rv  ir.Subgraphrw  r  )rv  r  )r  r  rr  ztuple[torch.Tensor]r  (  
__future__r   
contextlibr  r   r   loggingr  r  rN  r  r(  collectionsr   collections.abcr   r   typingr   r   r	   r
   r   r   r   typing_extensionsr   unittest.mockr   r   rp   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreerG  _pytreer   torch._dynamo.utilsr   (torch._higher_order_ops.associative_scanr   rU  r   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r   rV  r    torch.utils._ordered_setr!   torch.utils._sympy.functionsr"   r#   r$   r%   _dynamo.utilsr'    r)   r*   r+   r,   decompositionr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   r  rI   rJ   r  rK   rL   rM   FALLBACK_ALLOW_LIST	getLoggerr  rV  rN   __annotations__rO   rq   rr   r  r  tr_c10dr  r   _higher_order_opsr  rX   rg  rP   quantized_decomposedr\   rn   rt   r   r   r   r   r:  r?  r]  r  bmmconvolutionconvolution_backwardr  r  r  rq  r_  ra  _int_mmr  r  r  r  rI  rW  rb  r{  	complex32	complex64r   rV  r   r   r   r   r   r   r   r   r   r  r2  r  r$  r8  ra  ry  r   r  r  r  r  r   r  r  
device_putr  r  r  r  r  r   aliasdetachdetach_liftview_ofr  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  _unsafe_viewreshaper"  slicer+  r<  rY  quantize_per_channelrv  dequantize_per_channelr  quantize_per_tensorr  dequantize_per_tensorr  r_  r  r  r  r  r  r  r  r   r  r  r  r  r  r&  r  r  cacher*  r/  r4  rE  r\  ra  rngprimsrr  rt  ry  	bernoulliru  r}  r  	lru_cacher  r  rd  r  r  r  r  r  r  randintforce_stride_orderr  ro  r  r  r  lookup_seedr  randomr  r  r  r  r  rz  r  	NO_OPMATHr  r  r{   r  r  ry   r}   r  _adaptive_avg_pool3dadaptive_max_pool3d*_scaled_dot_product_attention_math_for_mpsuniformexponential_pdist_forwardsoft_margin_loss_backward_fused_rms_normxpuis_availableembedding_dense_backward_cdist_forward_cdist_backward
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation_grouped_mm
_cudnn_rnn_cudnn_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backwardr  r  kthvaluetopkr  median	nanmedianr  resize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_stategraphsafe_run_with_rng_statemasked_scattermasked_scatter_backwardrA  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   rs  r  r  #_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward0_scaled_dot_product_fused_attention_overrideable9_scaled_dot_product_fused_attention_overrideable_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducer  r  r#  rB  r*  r0  r:  r<  scalar_tensorrP  
LongTensorrR  r`  rb  rd  rm  r  rq  r  rp  r  r  r_  
zeros_liker  r  rz  r  r  r  r  r  r  r  r  r  r-  r  r  r  r  r  r  r  r  r  fallback__unsafe_masked_indexr  ,fallback__unsafe_masked_index_put_accumulater%  r  r  r   r  r,  r*  r0  r'  rR  rW  rZ  re  rg  ra  rn  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r-  r7  rF  rM  rd  r^  rx  rv  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r(  r,  r2  rA  rD  rU  rZ  r  r_  ra  rc  rg  re  Tensor_Tensorrr  Scalarrp  Tensor_Scalarrq  r  r{  r  r  r  r3  r  rS  r  r  true_divider  r  r$  r2  r  r  r  r  r  r  r  r  r  r  r%  r	  r  r  r  r  r  r  r3  r  r4  r  r  r  r  r  r   r  r  r  r  rsqrtr  ro  expm1relur  rn  r;  r|  cossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfr  tantanhr  r  r  logical_xorrd  re  	clamp_min	clamp_maxnegrc  	remaindersignsignbit	_neg_viewlerL  rK  rI  r-  necoshsinhacosacoshasinasinhatan2atanatanhr  erfcerfinvhypotlog10log2	nextaftercodegen.commonri  r  r  r   rV   r   r^  _foreach_addListforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_rsqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyr  _foreach_add__foreach_mul__foreach_div_r  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r  r   r   r"  r$  r   methodfuncr'  r)  _inductor_testrk  r+  ru  r/  set_source_Tensorr3  fsdpr5  rM  *torch._higher_order_ops.auto_functionalizerN  rZ  higher_orderr   rs  rx  invoke_quantr  r  r  r  comm_loweringr  r  r#  r  register_quantized_opsregister_woq_mm_opsr  register_onednn_fusion_opsr  register_jagged_opscontextmanagerr  r   r]   r[   <module>r	     s5   "        	   # . O O O '    +  $ $ ( H U    M G / U U , 8 8 =          * T]t_ !   !FH	C H     uzz,,-/	yy~~
))

		"5::#8#89; ../
))*
 !!6!679 NP K Pyy55 ,	BF

3   !!$$$$--&&, {{zz{{{{{{}}}}}}

."# E
EE E C	E
  E &EP24 C	4r  	(// 4,#40j  $Sl7tE E 5**77TR) S)X 5--4H) I) @E 5 5( 499??=& >& ;@e K K 5##>F ?F 
7?? $*Z>    4::DI J0 4))UPTU  V F DJJT\\499emmTU V 4doo&s+ 4<<T:B ;B2 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::%%&% '% 4::! ! 4;;D9> :>4 5))tD
 E
 4>>t<# =# 4;;1  1h 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;> 4::48V 9V 4??=J >J$ 4##> ? 4''TB C
BJ '<<RVW/// / 	/
 / / / / X/d //T (,/// / 	/
 / / / %/ //d ,,44$!!! ! 	!
 ! ! !!H ..66D (,!!! ! 	!
 ! ! %! !!H ,,33*** * 	*
 * * **Z ..554 (,,,, , 	,
 , , %, ,,^ 488y: y:x 4==d;3E <3El 4%%4@6 A6 4((dC D 4;;D96 :6
 4::48 98 4((dC  D  4;;D9 : 4;;D9D :D8 4>>t< = 4??= > 488 $   >
3 
3%6 %6P/Nd	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  499J J 4::J J >44$OL PL
 >&&DAK BK >''TBJ CJ
 >--4H	 I	 >((dCRS  D6 >))tDLM	(09FI E8. 4$$++F
 "&^^
^ 	^
 ^ ^  ^ ^ G^B NN(G(Q(Q 777 	7
 77t"$CN d'' ( d&& ' d== > dll ' d&&U 3 d!! " d,,5 9 d"" /99%%E d!! " d"" # doo  d!!)) * d++33 4 djj  dnn## $ d))11 2 d,,44 5 dkk  d$$5 1 d . d'')@ A doo} - d&&(: ; d!!#5 6 d..0B C d** + d== > d== > d22 3 d== > d33 4 d00- @ d00 1 d// 0 d// 0 d11 2 d11 2 d-- . d-- . d-- . d..0B C d00 1 d++] ; d"" # dii  dii  dmm  dii  dii  dkk  dnn  dmm  dll  doo  d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  dnn  djj  d'' ( d// 0 d   d## $ d%% & d## $ d!! " djj  dmm  dll"" # dhhU + d((- 8 ell$$;; < ell$$77 8 ell$$AA B
 d!! " d** + d""$6 7 djj  d'' ( d;; < dnn  doo  djj  0088	
 99AA	
 ,,44	
 55==	
 ,,44	
 55==	
 44<<	
 ==EE	
 99AA	
 BBJJ	
 d++33_ E d,,44o F d//77 I d0088/ J d  
 499$7
 8
 4::"  ( 4"##d**+E2 5:: ( 4&&DA B: 4%%4@@ A@F ELL$"4"456tDU 5 75p 5??#4 $4 5##$+ %+ 4++,+" -+"\ 4&&' ( 4//0%)59$ 1< 4>>t<K =K8 EKK,- 
 .$.> t/0B50IJ
1!45	 21 56
 4>>" $T$4  # 4%%&Td '> 4))*"4 +" 5%%--.A /A EJJ		*+: ,:
 4;;D9! :!H 4>>t< =B#F FR"1J 4::48
 9
 4%%4@/ A/ 4>>t< = 4))* +;
 4??= > >44$O P{| !1%%5!  0@,,44%0 ,
 4,,$G H, 4;;QUV	P W	P 1 1 4**E F 4<<T:9 ;9 !& 
  B 4==d;DH : <:" 4##>3 ?3 4$$$?6 @6 4&&DAP BP 4''TBPT m Cmh , *, 	,
 ,^ 4**223> 4> 41199:J ;J 4**223RVI-I@OI 4I 41199:RVU-U@OU ;U 4**223 !% $ $S S 	S
 S 4S 41199: !% $ $	 	 		
 	 ;	7 599$$% &*\\(\6;\\~ 4''TB4 C4n :>< MQ < JN@:HV 5::PTU 5 V5D3 3	 > 	11t$< 4//TJ 
 K
 4//TJ 
 K
 -=))11- ) 488dSL TL^8B&R2j  0%%5  
 4,,-B .BJ  0$$%  
 4++,N -Nb. 4--.V /V 4--.V /V?D 433;;<BF* =*Z 'OO  'OO 
 4??=  >* 4??=  >*wt  0$$%  
 4++F b GbJ  0$$%  
 4++F D GDN$.b ( 499:t : : 4 (V4( DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,, -,^ F 4::2 2 4::48 9     488t, - DHH:.( /(@ EII;$/$ 0$( 	txx'7DD

 DIIuzz*d;$ <$  DHHeii(). . *. #4;;#6#67#DLL$8$89 ():):)B)BC "4;;#6#67"4;;#6#67 4;;  . 4<<  !. 4$$% &* 4;;D9 :, 4;;D9 :, 499. . . 488@ @
 48867 77 48867 77 %--  	!: ;		*>%+@A		*>%+@A!$++.85;;? "$++.85;;? HH$L !!1!1uM 499##>!re $2 ?$2N 499$$$?H @H 	#4::.&txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4   *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* $.. !' * $.. !' *""'8
 4>> "$))L 499  4<<uzz B $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 499 % 4>> * D DE$ %D4Ld50 	 3+		
5 5Mt50 	 3+		
5 %, .CT  0ct  4,,33Sd K-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4,,113 7 4,,33S 9 4,,44c : 4,,44c : 4,,33S 9 4,,113 7 4,,<<c B-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4--t 4 4.. 6 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4--t 4, T..335E t00779K T..335E t00779K T..335E t00779K
 C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  ( $,,  , $// "#5 6 $++ z * $// "#6 7 $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% &  4??&&' ( 4>>" # "'')LFD(01$7 * 5==! " 4<< 7 !7 599++334 5
 599%%;;< = 599>>&&445K 6K 599>>7##uyy~~++334# 5# 599>>(()%) = *=@ K J ! " 12S 3S( 599))..DI/ J/ 599))44$O* P*0 599))99tT/ U/
 5**77TRDH   S8 &DA  4G  B F 599??//778 9 599))66DQ+ R+* 3 2   >88dS2 T2l     ! !  * * ,  ' ' )    + + -    $ $ &  r]   