
    `iO                    g   U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZ d dlmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlZd dlZd dlZd dl Zd dl!m"c m#Z$ d dl%m&Z& d d	l'm(Z( d d
l)m*Z* d dl+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8 d dl9m:Z:m;Z; d dl<m=Z=m>Z>m?Z? d dl@mAZA d dlBmCZCmDZDmEZEmFZF ddlGmHZH ddlImJZJmKZKmLZLmMZM ddlNmOZOmPZP ddlLmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZa ddl"mbZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZm ddlnmoZompZp erddlqmrZr  ed          Zs ed          Zt eAddg          Zu ejv        ew          Zxi Zydezd<   i Z{d ezd!<    eAej|        j}                             Z~ejo        j        Zejo        j        Zejo        j        Z eAej|        j}                             Z eAej|        j}                 ej        j        g          Z eAej|        j}                             Zi Zd"ezd#<   ejo        j        Zd$ Zdd'Zdd,Zd- Zd. Zd/ Zd0 Z eej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        g           ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        ej        d1Zdd4Zd5 Zd6 Zdd9Zd: Zd; Zdd@ZddJZdK ZddLZdMe3j        dMeyfddOZdP ZddQZ	 	 	 	 	 ddRZddSZ	 dܐddWZ eej        j        dX          dY             Z eej        dX          ddZ            ZdMd[dd\Z eej        j        dX          dd]            ZdMdMd^dd_Z eej        dX          dܐdd`            Zddae3j        dMdddMdfdbZdc Z eƦ             	 dddZ eej        dMde          df             Z eej        dMde          dg             Z eej        ej        ej        ej        ej        g          dh             Z eedi          r  eej                  eϦ            eej        dX          ddj            Z eej        dX          ddk            Z eej        g          ddl            Z eej                  dm             Z eej                  dn             Z eej                  do             Z eej                  dp             Z eej        j                  dq             Z eej                  dr             Z eej        dX          ds             Z eej        dX          dt             Z eej        dX          du             Z eej                  dv             Z eej        dX           eej        dX           eej        dX          ddy                                    Z eej        dX          dz             Z eej        dX          dd|            Z eej        dX          dd}            Z eej        dX          dd~            Z eej        dX          dd            ZddZ eej        dX          dd            Zd Z eej        j                  d             Z eej        j                  d             Z eej        dX          dddd            Z eej        j        dX          dd            Z eej        j        dX          dddd            Z eej        j        dX          dd            Z eej        j        dX          dddd            Z eej                  dd            Z eej        dX          ddd            Z eej        dX          ddd            Z eej        dX          ddd            Z eej        dX          d             Z eej        dX          dd            Z eej        dX          dd            Z eej        dX          dd            Z eej        dX          d             Z eej         dX          d             Z  eej        dX          d             ZddZ eej                  dd            ZddZej        d             ZdddZdddZdddZ	ddZ
d Z eejo        j        j        dX          d             Z eej        dX          d             Z eej        dX          d             Z eej        j        dX          d             Z eej                  d             Z ej        d          d             Zd Z eej        j                  Z eej        j                  Z eej        j                  Z eej        j                  Z e
ej                    eej                  d             Z eej                  d             Z eeKj        dX          d             Z  eeKj!        dX          dd            Z" eeKj#        dX          d             Z$ eeKj%        dX          d             Z& eeKj'        dX          d dddń            Z( eeKj        dX          d dddȄ            Z)dd˄Z*dd̈́Z+ eej,        j-        dX          dMdMdddΜddׄ            Z, eej.        e3j/        X          dMdMd؜ddڄ            Z.dۄ Z0d܄ Z1d݄ Z2dބ Z3d߄ Z4d Z5d Z6d Z7 e
ej8                    e
ej9                    e
ej:                    e
ej;        dM            e
ej<        j        dM            e
ej=                    e
ej>        dM            e
ej?        dM           ej@        A                                r e
ejB        dM            e
ejC                    e
ejD                    e
ejE                    e
ejF        j                    e
ejG        j                    e
ejH                    e
ejI        jJ                    e
ejK        j                    e
ejL        j                    e
ejM                    e
ejN        dM            e
ejO        e0            e
ej        e6            e
ejP        e0            e
ejQ        e1            e
ejR        e1            e
ejS        e1            e
ejT                    e
ejU                    e
ejU                    e
ejV                    e
ejW                    e
ejX                    e
ejY        e0            e
ejZ                    e
ej[                    e
ej\                    e
ej]                    e
ej^                    e
ej_                    e
ej`                    e
eja                    e
ejb        e1            e
ejc                    e
ejd        e0            e
eje                    e
ejf                    e
ejf        jg                    e
ejh                    e
eji                    e
ejj                    e
ejk                    e
ejl                    e
ejm                    e
ejn                    e
ejo                    e
ejp                    e
ejq                    e
ejr                    e
ejs                    e
ejt                    e
eju                    e
ejv                    e
ejw                    e
ejx                    e
ejy                    e
ejz                    e
ej{                    e
ej|                    e
ej}                    e
ej~                    e
ej                    e
ej                    e
ej        j                    e
ej                    e
ej                    e
ej                    e
ej                    e
ej                    e
ej                    e
ej                    e
ej        j                    e
ej        j        dM            e
ej        e0            e
ej        j        j                    e
ej        j        j                    e
ej        j        j                    e
ej                    e
ej                    e
ej        e1            e
ej                    e
ej                    e
ej                    e
ej                    e
ej                    e
ej        j                    e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7dM            e
ej        j        e7            e
ej        j        e7            e
ej        j        e7            e
ej        j        e7            e
ej                    e
ej        j-        da            eej        dX          dd            Z eej                  ddd            Zd Z eed          r  eej                  e            eej                  d             Z eej        dX          dd            Z eej        dX          dd            Zd Z eej        ej        g          ddddMdd            Z eej                  dd            Z eej                  d             Z eej                  d             Z eej                  d             Z eej                  	 ddddd            Zd Zd Zd Z eej        ej        g          dddddddd            Zd Zd Z  eej                   ee                    Z e ed                    Z e ed                     Zd Z eej                  ddddd d            Z eej                  ddddd d            Z eej                  ddddd d            Z eej        j                  d             Z eej        ej        g          d             Z eej        dX          dܐd            Z eej        dX          dd            Zːd Z	 dd	Z͐d
 ZΐddZ eej        dX          d             Z eej        dX          d             Z eej        dX          dܐd            Z eej                  dܐd            ZӐd ZԐd Z eej        dX          dܐd            Z eeKj        dX          dܐd            ZאdܐdZ eej        j        dM          Z eej        j        dM          Z eej        dX          d             Z eej        dX          d             Zed             Z eej        dX          dd            Z eej        dX          dd            Zddaddd Z eej        dX          dd!dd"            Z eej        dX          dd#            Z eej        dX          dd$            Z eej        dX          dd%            Z eej        dX          dad&dd'            Z	 	 d dd,Z eej        j                  ddd.            Z eej        j                  ddd/            Z eej        j                  	 dڐdd2            Z eej        j                  	 dڐdd3            Z eej        j                  	 	 	 ddd5            Z eej        j                  	 	 	 ddd6            Zd7 Z eej        j                  d8             Zdd<Z eej        dX          dd=            ZddAZddBZddCZ	 d	dEZddFdGZdH ZddIdJZdK Z eej        dX          	 dܐdL            Zd
dSZ eej        dX          dT             ZdU Z eej        dX          	 	 	 	 ddV            Z eej        dX          	 	 	 	 ddW            Z eej        j        dM          Z eej        dX          dX             ZddZZd[ Zd\ Zd] Z eej         j        dM          Z eej                   d^             Z  eej        j        dM          Z eej                  d_             Zd` Z eej                  da             Z eej                  db             Zdc Z eej        j                  	 ddd            Z eej        j        dM          Z	 eej
        j        dM          Z eej        dX          	 	 	 	 	 ddf            Z eej
        dX          	 	 	 	 	 ddg            Z
dh Z eej        j        dM          Z eej        dX          	 ddi            Z eej        j        dM          Z eej        dX          	 ddj            Zdk Zdl ZdddoZdp Z eej                  dddqdr            Zds Zdt Zdu Zdv Z eej        ej        g          dddMdwdx            Z eej                  dddMdwdy            Zdz Zed{             Z eej        j        dM          Z  eej        j!        dM          Z" eej        j#        dM          Z$ eej        da|          d}             Zdܐd~Z% eej&                  d             Z& eej'        dX          dܐd            Z'ed             Z(ed             Z) eej*        da|          dd            Z+ eej,        gda|          d             Z,ddZ- eej*        gda|          d             Z. eej/        ej*        j-        gdae3j0        e          d             Z* eej1        ej1        gda|          d             Z1 eej2        ej2        g          dddqd            Z3 eej4        j                  Z5 eej6        j                  Z7 eej8        j                  Z9 eej:        j                  Z; eej<        j                  Z= eej4                  dڐd            Z4 eej6                  dڐd            Z6 eej8                  d             Z8 eej:        dX          dd            Z: eej<        dX          dd            Z< eej>                  dddqd            Z> eej?                  dd            Z@ eejA        dX          dd            ZB eejC        dX          dd            ZD  eejE                   ed                       eejF                   ed                    ZG  eejH                   ed                    ZI  eejJ                   edej                            ZK  eejL                   edej                            ZM eejN        dad          ZN eejf        jg        dM          ZO eejf        jg        dX          dddMdd            ZP eejf        j        dX          dd            ZfdڐdZQddZR eQejS                  ZS eRejT                  ZT eQejU                  ZU eQejV                  ZV eejW                  ZW eRejX                  ZX eRejY                  ZY eejZ                  ZZ eej[        da          Z[ eRej\                    eRej]                    eej^                  Z^ eej_                  Z_ eej`                  Z` eeja        d          Za eejb                  Zb eejc                  Zc eejd                  Zd eQeje                    eQejf                  Zf  eejg        e3j0        X          ef            eQejh                    eQeji                    eQejj                    eRejx                    eejk        ddaej                  Zk eejl        ddaej                  Zl eejm        ddaej                  Zm eejn        ddaej                  Zn eejo                  Zo eejp                  Zp  eejq                  eo             eejr                  ep            eejs                  Zs eej^                  Z^ eQejt                  Zt eeju                    eejv        d          Zv eej                    eejw        ej                     eejx                  es            eejy        ej                    eejz        ej                    eej{        ej                    eej|        ej                  Z| eej}        ej                    eej~        ej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                    eQej                   ddlmZmZ d ZeD ][Z eee          D ]\  ZZZ eŐeeee            eee          D ]\  ZZZ eŐeeee           \ eej        j        eNda          Z eej        j!        eNda          Z eej        j-        eNda            eej        j        e,          Z eej        j-        e,            eej        j!        e,          Z eej        j        e[            eej        j!        e[            eej        j        es            eej        j        e^            eej        j!        e            eej        j        e            eej        j        e            eej        j        e*          Z eej        j-        e*            eej        j!        e*          Z eej        eY            eej        eS            eej        j        eo            eej        j!        eo            eej        j        ep            eej        j!        ep            eej        j        eo            eej        j!        eo            eej        j        ep            eej        j!        ep            eej        et            eej        ev            eej        e          Zd Z eej        j        ej        j        e            eej        j!        ej        j!        e            eej        j        ej        j        e            eej        j!        ej        j!        e            eej        j        ej        j        e            eej        j!        ej        j!        e            eej        j        ej        j        e           d Z eej        eN            eej        e_            eej        e`            eej        ea            eej        eb            eej        ec            eej        ed            eej        e,            eej        j-        e*            eej        j        e+            eej        ek            eej        el            eej        em            eej        en            eej        e[            eej        eW            eej        eX             eej                  e_             eej                  e`             eej                  eb             eej                  ec             eej                  ed            eej        ej                    eej        ej                    eej        ej                    eej        ej                    eej        ej                    eej                  dڐd            Z eej        j                  d             Z eej        j                  d             Z eej                  d             Z e:j                    D ]&\  ZאZ  e e;eצ                    eئ           ' eej                  d             Z eej                  d             Z eejo        j        j                  d             Z eejo        j        j                  d             Z eejo        j        j        j                  d             Z eejo        j        d          r, eejo        j        j'        j                  d             Z eejo        j        j                  ddd            Zd dlmZ  e
e            ee*          d             Z eejo        j        j        dX          d             Z eejo        j        j        dX          dܐd            Z  eejo        j        j        dX           ej        eda                      eejo        j        j        dX          dd            Z eej        j        dX          ddÜddĄ            Z ee(dX          ddȄ            Z eejo        j        j        j                  dɄ             Z eejo        j        j        dX          dʄ             ZddːlmZ  e              eeKj        dX          d̄             ZddlImZ  eHe           ddlImZ  ej                      ej                     ddlImZ  ej                     ddlImZ  ej                     ej        ddф            Z dS (      )annotationsN)defaultdict)IterableSequence)AnyCallablecastOptionalTYPE_CHECKINGTypeVarUnion)	ParamSpec)patch)counters)associative_scan_op)triton_kernel_wrapper_mutation)get_layout_constraint_tag)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)free_unbacked_symbolshas_free_unbacked_symbolsresolve_unbacked_bindings)
OrderedSet)CeilDivFloorDivIdentityModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)BaseView	DtypeView
ExpandViewIndexingConstantIRNode	is_triton
MutableBoxOnlineSoftmaxReductionops_wrapperPermuteView	Pointwise	ReductionShapeAsConstantBufferSqueezeView	TensorBoxvalidate_irView)ceildivdecode_device
is_dynamicis_gpuis_pointwise_useis_view,needs_fallback_due_to_atomic_add_limitationspad_listlike#register_op_dtype_propagation_rules#register_op_requires_libdevice_fp64sympy_productuse_scatter_fallback)opsV)ReductionType_T_Pztorchvision::roi_alignzaten::index_addz8dict[Union[Callable[..., Any], str], Callable[..., Any]]	loweringsz9dict[torch._ops.OpOverload, Optional[Callable[..., Any]]]_maybe_layout_constraintsz2dict[torch._ops.OpOverload, torch._ops.OpOverload]inplaceable_foreach_opsc                     t           j        j        j        D ])} | j        D ]}|j        dk    r|j        t          v s  dS  *dS )Ncall_functionTF)rP   graphcurrent_nodeusersoptargetforeach_ops)nodeusers     l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/_inductor/lowering.pycur_node_has_non_foreach_usersrb      s^    $*  J 	 	DG..DK;4N4Nttt 5O	 5    	arg_pairs%Iterable[Union[tuple[Any, Any], Any]]c                   t          t                    }d}t          |           D ]\  }}t          |t                    sd}|f}t          |  pt          j        }d }|D ]2}t          |t                    r|j	        
                                } n3|
J d            |r|\  }|||f                             ||f           |S )NFTz.foreach op should have at least one tensor arg)r   list	enumerate
isinstancer   rE   r,   #combo_kernel_foreach_dynamic_shapesr@   data
get_deviceappend)rd   outunpack_argsiargsuse_foreachdevicets           ra   group_foreach_argsru      s    
d

CKY'' 5 54$)) 	K7DD!!OV%O 	  	 	A!Y'' **,, !!#S!!! 	GTV[!"))1d)4444Jrc   fnCallable[..., Any]returnOptional[Callable[..., Any]]c                    t          | t          j        j                  sdS t	          | d          x}rt          |          S | t          v rt          |          S dS )zHGet layout constraints. Returns None if there are no layout constraints.NF)with_default)ri   torch_ops
OpOverloadr   tag_to_layout_constraintrU   )rv   maybe_layout_tags     ra   maybe_layout_constraintsr      sd    b%*/00 t4ReLLLL :'(8999	&&&(,,4rc   c                $   | t           j        j        j        k    rt          S | t           j        j        j        k    rt          S | t           j        j        j        k    rt          S | t           j        j        j	        k    rd S t          d|            )NzUnknown layout constraint tag: )r|   _CTagneeds_exact_stridesconstrain_to_fake_tensorsneeds_contiguous_stridesrequire_contiguous_stridesneeds_fixed_stride_orderconstrain_to_fx_stridesflexible_layoutAssertionError)tags    ra   r   r      sx    
ehl...((
ehl333))
ehl333&&
ehl***t
@3@@
A
AArc   c                .    | st          d|           d S )Nzinductor does not support NotImplementedErrorcondmsgs     ra   
assert_nyir      s.     F!"Ds"D"DEEEF Frc   c                V    t           t          t          t          t          f          rd  D             S t
                                          t           t          j        j	                  r:t
          
                     fd                                 D                        d S d S )Nc                ,    g | ]}t          |          S  )add_needs_realized_inputs.0xs     ra   
<listcomp>z-add_needs_realized_inputs.<locals>.<listcomp>   s!    999)!,,999rc   c              3  8   K   | ]}t          |          V  d S N)getattr)r   overloadrv   s     ra   	<genexpr>z,add_needs_realized_inputs.<locals>.<genexpr>   s>       %
 %
&.GB!!%
 %
 %
 %
 %
 %
rc   )ri   rg   settupler$   needs_realized_inputsaddr|   r}   OpOverloadPacketupdate	overloads)rv   s   `ra   r   r      s    "tS%455 :99b9999b!!!"ej122 
$$ %
 %
 %
 %
24,,..%
 %
 %
 	
 	
 	
 	
 	

 
rc   c                    t          | t          j        j                  r1|                                 D ]}|t
          t          | |          <   d S |t
          | <   d S r   )ri   r|   r}   r   r   rU   r   )rv   
constraintr   s      ra   add_layout_constraintr      si    "ej122 3 	J 	JH?I%gb(&;&;<<	J 	J )3!"%%%rc   )r   r+   r)                     	   
         dtypeintc                |    t          | t                    s| S | t          v sJ d|  d            t          |          } | S )Nzid z missing from DTYPE_ID_LOOKUP)ri   r   DTYPE_ID_LOOKUPr   s    ra   decode_dtyper      sM    eS!! O###%O5%O%O%O###E"ELrc   c                    t          | t                    rBt          |                                           p t	          |                                           S t          | t
          j                  r	| j        du S t          | t                    S )NT)	ri   r@   r   	get_dtyper   sympyExpr
is_integerr   r   s    ra   is_integer_typer     sq    !Y "..Q2B1;;==2Q2QQ	Auz	"	" "|t##!S!!!rc   c                    t          | t                    r!t          |                                           S t          | t                    S r   )ri   r@   r   r   boolr   s    ra   is_boolean_typer     s:    !Y #...!T"""rc   type_promotion_kindr   c                H    d fd|D             }t          |d| i\  }}|S )Nc                    t          | t          t          j        f          r| S t	          |                                           }t          j        dg|z  |                                           S )Nr+   r   )	ri   r   r   Basiclenget_sizer|   zerosr   )inpdims     ra   construct_inputz+get_promoted_dtype.<locals>.construct_input  sY    cFEK011 	AJcllnn%%C;sSy@@@@rc   c                &    g | ]} |          S r   r   )r   argr   s     ra   r   z&get_promoted_dtype.<locals>.<listcomp>   s#    111SOOC  111rc   r   )r   )r   rq   inps_r   r   s        @ra   get_promoted_dtyper     sL    A A A 2111D111D!4Q=PQQHAuLrc   c                P   t          | t          t          f          s| g} nt          |           } t          |           D ]f}t          |t          j        j                  rE|                                D ]0}t          ||          }|t          vr| 	                    |           1g| S r   )
ri   rg   r   r|   r}   r   r   r   rT   rm   )aten_fnrv   r   other_fns       ra   get_overloadsr   %  s    ge}--  )w--7mm - -b%*566 	-LLNN - -"2x009,,NN8,,,Nrc   c                    t          | t          j        j                  r	|| j        v S t          | t          j        j                  r||                                 v S dS NF)ri   r|   r}   r   _qualified_op_namer~   name)r\   	namespaces     ra   in_namespacer   5  sV    "ej122 &B111	B
-	.	. &BGGII%%5rc   r   r@   rs   torch.devicec           	        t          | j        t          j                  r!t	          |                                           r| S d |                                 D             }|                                 }||j        dk    rx||k    rrt          |          dk    st          |          dk    rL|d         dk    r@t          t          j
        t          j                            | |d                              S | S )zB
    Copy cpu scalar if doesn't not match with given `device`
    c                V    g | ]&}t           j        j                            |          'S r   rP   rY   sizevarssize_hint_or_throwr   ss     ra   r   z)maybe_copy_cpu_scalar.<locals>.<listcomp>E  s+    IIIqAG//22IIIrc   Ncpur   r+   F)ri   rk   r.   ReinterpretViewr"   r   rl   typer   r@   
StorageBox
DeviceCopycreate)r   rs   size
cur_devices       ra   maybe_copy_cpu_scalarr   =  s     afb011 5N	

6 6  IIAJJLLIIIDJOu$$&  YY!^^D		Q47a<<r}';';Az5'Q'QRRSSSHrc   rq   	list[Any]kwargsdict[str, Any]	broadcastr   )Optional[ELEMENTWISE_TYPE_PROMOTION_KIND]convert_input_to_bool tuple[list[Any], dict[str, Any]]c           	     `    d t                     D             }d                                 D             }|s|s fS |s|r|rt          j        nId  D             }|                    d                                 D                        t          |d|i|r |d                  n|d                                                  |D ]}t           |                    |<   |D ]}	t          |	                   |	<   fdfd D              fd	                                D             |rht          t          t          j         fd
|D             fd|D                                  }
t          |
d                                                   }t          ||
dt          |                             D ]
\  }}| |<   t          ||
t          |          d                   D ]
\  }	}||	<   t!          t                               D ]@}t#           |         t$          j                  rt)          j         |         |           |<   AD ]@}	t#          |	         t$          j                  rt)          j        |	         |          |	<   A fS )zB
    Transforms arguments for broadcasting and type promotion
    c                B    g | ]\  }}t          |t                    |S r   ri   r@   r   rp   r   s      ra   r   z"transform_args.<locals>.<listcomp>\  s+    NNN$!QZ95M5MNANNNrc   c                B    g | ]\  }}t          |t                    |S r   r   )r   kvs      ra   r   z"transform_args.<locals>.<listcomp>]  s+    OOODAqjI6N6NOaOOOrc   c                t    g | ]5}t          |t          t          j        f          st	          |d           3|6S r   )ri   r   r   r   hasattrr   as     ra   r   z"transform_args.<locals>.<listcomp>g  sT       a&%+!677 <C1g;N;N  rc   c              3  :   K   | ]}t          |d           |V  dS )r   N)r   r   s     ra   r   z!transform_args.<locals>.<genexpr>m  s1      !T!T7@S@S!T!!T!T!T!T!T!Trc   r   r   c                    t          | t                    rt          |           S t          | t          j                  rt          j        | j                  S | S )Nvaluer   rs   )ri   r@   to_dtyper.   Constantr  )r   rs   r   s    ra   promoteztransform_args.<locals>.promote~  sW    #y)) U+++C-- {%OOOO
rc   c                &    g | ]} |          S r   r   )r   r  r  s     ra   r   z"transform_args.<locals>.<listcomp>  s!    )))q

)))rc   c                .    i | ]\  }}| |          S r   r   )r   r   r   r  s      ra   
<dictcomp>z"transform_args.<locals>.<dictcomp>  s'    ;;;DAq!WWQZZ;;;rc   c              3  (   K   | ]}|         V  d S r   r   r   rp   rq   s     ra   r   z!transform_args.<locals>.<genexpr>  s'      33T!W333333rc   c              3  (   K   | ]}|         V  d S r   r   )r   r   r   s     ra   r   z!transform_args.<locals>.<genexpr>  s'      771VAY777777rc   N)rh   itemsr|   r   extendvaluesr   rl   r   broadcast_tensorsrg   	itertoolschainr   zipr   rangeri   r.   r  r4   r   )rq   r   r   r   r   args_indiceskwargs_indicespromoting_argsrp   r   broadcastedr   r   rs   r   r  s   ``           @@@ra   transform_argsr  Q  sH    ON)D//NNNLOOFLLNNOOON  V| %<3 %<  	JEE   N !!!T!TV]]__!T!T!TTTT&$7 E &2PDa!!vnQ>O7P
*,, 	  	= 	=A+DGV<<DGG 	A 	AA-fQi@@F1II	 	 	 	 	 	 *)))D)));;;;FLLNN;;; ?'3333l3337777777  
 KN++--..k2EC4E4E2E&FGG 	 	DAqDGGC4E4E4G4G(HII 	 	DAqF1IIs4yy!! 	; 	;A$q'2;// ;$+DGT::Q 	? 	?A&)R[11 ?&-fQi>>q	<rc   c                    t          j                  fd            }t          |           }t                              |           t
                              t                              ||                     |S )a  
    Add a foreach lowering to lowerings dict.

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                 `    t          |           dk    sJ  | i |}t          |           |S )Nr)   )r   rA   )rq   r   rn   	decomp_fns      ra   wrappedz+_register_foreach_lowering.<locals>.wrapped  s=    4yyA~~~~i(((C
rc   )	functoolswrapsr   r^   r   rT   dictfromkeys)r   r  r  aten_fnss    `  ra   _register_foreach_loweringr%    s}     _Y      W%%Hx   T]]8W55666Nrc   c                     t          j                   fd            }t                      |                    t                               |                     |S )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                    t          |           } t          |          }d}t          |           dk    r9t          | d         t           t          f          rd}t          | d                   } t          d D                       s5t          d |                                D                       r
J d            t          | |          \  } }|r| g}  | i |}t          |           |S )NFr+   r   Tc              3  H   K   | ]}|t           v pt          |d           V  dS )_c10d_functionalN)	fallbacksr   )r   rv   s     ra   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  sG       
 
JLR9_DR1C D D
 
 
 
 
 
rc   c              3  "   K   | ]
}|d k    V  dS )rn   Nr   r   s     ra   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s&      ==!1:======rc   zout= ops aren't yet supported)
rg   r"  r   ri   r   allanykeysr  rA   )	rq   r   unpackedrn   r   r   r   r  r   s	       ra   r  z#_register_lowering.<locals>.wrapped  s   t**!%ft99>>ja4-@@>HQ==D 
 
PW
 
 
 
 
 	 ==v{{}}=====  / = &&)%8:O
 
f  	6Di(((C
rc   )r   r!  r   r   r"  r#  )r   r  r   r   r   lowering_dictr  s   `````  ra   _register_loweringr1    s    & _Y         : G$$Gw88999Nrc   F.Callable[[Callable[_P, _T]], Callable[_P, _T]]c                @    t          j        t          | ||||          S )z+
    Shim to support decorator syntax.
    )r   r   r   r0  )r   partialr1  )r   r   r   r   r0  s        ra   register_loweringr5    s0     /3#   rc   c                   g }t          j        t          |           t          |          t          j        j                  D ]\  }}t          j        j        	                    |          r|
                    |           @t          j        j        	                    |          r|
                    |           zt          j        j                            ||           t          t          j        |          j                  t          t          j        |          j                  k     r|
                    |           |
                    |           t          t          |                    S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    )	fillvalue)r  zip_longestreversedr   SOnerP   rY   r   is_size_one_or_falserm   check_equalsr   expandfree_symbolsr   )r  boutputr   ys        ra   broadcast_symbolic_shapesrC  	  s*    F%hqkk8A;;%'+VVV 
! 
!170033 		!MM!W22155 	!MM!G))!Q///5<??/003u|A7S3T3TTTa    a    &!!"""rc   c                   ||
J d            ||t           j        }t          d | D                       s| S t          d | D                       r!|pt	          | d|ifdfd| D             S t          d | D                       }g }| D ]L}t          |t          t          f          r|	                    t          j        t          j        ||                                |                                          t!          |                                                               t          |t$          j                  r}|	                    t          j        t)          ||                                |                                	          t!          |                                                               7|	                    |           N|S )
NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3  d   K   | ]+}t          |t          j        t          t          f          V  ,d S r   )ri   r   r   r   floatr   s     ra   r   z$promote_constants.<locals>.<genexpr>'  s5      HHAz!ek3677HHHHHHrc   c              3  d   K   | ]+}t          |t          t          t          j        f          V  ,d S r   )ri   r   rF  r   r   r   s     ra   r   z$promote_constants.<locals>.<genexpr>)  s5      
D
D:a#uek233
D
D
D
D
D
Drc   r   c                    t          | t          j                  r$t          j        | t          d                     S t          j        | t          d                     S )Nindexr   rs   r  )ri   r   r   r.   r5   rD   r  r   r   s    ra   
const_funcz%promote_constants.<locals>.const_func.  sb    !U[)) U*5t1D1D    {%d@S@STTTTrc   c                &    g | ]} |          S r   r   )r   r   rL  s     ra   r   z%promote_constants.<locals>.<listcomp>6  s!    ...!

1...rc   c              3  h   K   | ]-}t          |t          t          t          j        f          )|V  .d S r   )ri   r@   r4   r.   r  r   s     ra   r   z$promote_constants.<locals>.<genexpr>7  s:      WWA:a)Z1U#V#VWaWWWWWWrc   r  rI  )r   DEFAULTr-  r,  r   nextri   r   rF  rm   r4   r   r.   r  r   get_device_or_errorrg   r   r   r   r5   )inputsoverride_return_dtyper   exrn   r   rL  r   s         @@ra   promote_constantsrU    s(    (,?,G,GO -H,GG $)<)D=EHHHHHHH 

D
DV
D
D
DDD /% 
);*
)<*
 *
	U 	U 	U 	U 	U /...v....	WWWWW	W	WB
C  a#u&& 	JJ!Kr||~~b>T>T>V>V   ''	     5;'' 
	JJ!$r||~~b>T>T>V>V   ''	     JJqMMMMJrc   c                .     d dd fd}|S )NalpharR  r@   c           	       	
 "t          d D                       r	rJ   S t                    r1| .| dk    r(t                    t          d         |           d<   n| J d D             d                                         
pd                                         dd          D ]s}t          |t          j                  sWt          
          t          |                                          k    s%J d d
 d|                                             tt          j        t          j        f	t          j        d uobt          t          j        dd           d uoEt          j        j        j        d uo-t          j        j        j                            d	d
          o	v 	
fd}s_d }D ]>}t'          |                                j                  r|                                } n?|sd                                         }p|}t-          j        ||
          S )Nc              3  ^   K   | ](}t          |t                    ot          |          V  )d S r   ri   r6   r7   r   r   s     ra   r   z0make_pointwise.<locals>.inner.<locals>.<genexpr>[  sH       /
 /
;>JsF##6	#/
 /
 /
 /
 /
 /
rc   r+   c                6    g | ]}|                                 S r   make_loaderr   s     ra   r   z1make_pointwise.<locals>.inner.<locals>.<listcomp>h  s     333q1==??333rc   r   zndim mismatch  rZ   low_precision_pointwise_barrierFc                    t                     t                    k    sJ d  d             t          j        k    r  fdD              S g }t                    D ]q\  }} |           }
|                                         }r0|v r,t          j        ||d          }t          j        ||          }|                    |           r 	| }r,t          j        |d          }t          j        |          S |S )Nzwrong ndim ra  c                &    g | ]} |          S r   r   )r   loadrJ  s     ra   r   zCmake_pointwise.<locals>.inner.<locals>.inner_fn.<locals>.<listcomp>  s!    4U4U4UTTT%[[4U4U4Urc   F)use_compute_types)r   r|   r   rh   r   rO   r  rm   )rJ  inputs_loaded	inp_indexre  rn   	inp_dtypedowncastr   emulate_precision_castsrv   rR  loaders	low_pr_fpoverride_fn_when_input_boolrangess   `      ra   inner_fnz/make_pointwise.<locals>.inner.<locals>.inner_fn}  sH   u::V,,,.LE.L.LF.L.L,,,
""'B'N224U4U4U4UW4U4U4UVV "'0'9'9 . .OIt$u++C &y 1 ; ; = =I. @9	3I3I#&<YRW#X#X#X!l8Y??!((----b-(* 9  #|C%PPPH<%888
rc   rs   r   rp  ro  )r-  rU  rg   mulr   r   ri   r.   BaseConstantr   r|   bfloat16float16rP   rY   r   rZ   metagetrF   rl   r   r<   r   )rX  rR  otherrp  rs   rp   r   rk  rl  rm  ro  allow_alpharv   override_devicern  rS  triton_fallbacks    `    @@@@@ra   innerzmake_pointwise.<locals>.innerZ  s   &3 /
 /
BH/
 /
 /
 ,
 ,
& #""?"?F++"6+@AA 	! UaZZf U33r
===33F333##%%%>)<)<)>)>ABBZ 	A 	AEeR_55 AV  I I : : :@@@V@@enn.>.>@@: :  ^U]3	G4 #66dB#$)5# $)--.OQVWW# " 	 	 	 	 	 	 	 	 	 	 	 	 	,  	0F  !,,..-.. \\^^FE  0--// *F	
 
 
 	
rc   )rR  r@   r   )rv   rS  rz  rn  ry  r{  r|  s   `````` ra   make_pointwiser}  R  sc     )- I
 I
 I
 I
 I
 I
 I
 I
 I
 I
 I
 I
 I
V Lrc   c                     ddd fd}|S )Nr+   rW  rR  list[list[TensorBox]]c                   t          t          j        j        j                  dk    p*t          j        j        j        t          v pt                      }d }|D ]"}t          |t          t          f          r|} n#|
J d            g }|D ]Z}t          |t          t          f          s'|                    |gt          |          z             E|                    |           [t          t          |           }d gt          |          z  }|                                D ]\  \  }}	}
g }|
D ]\  }}r	 |d| i}n | }|||<   t          j                            |t           j                  r?|	r=|r;|                                 |                    |                                           |rt          j                            |           t+          d |D                       sJ |S )Nr   z1at least one input must be a list to a foreach oprX  c              3     K   | ]}|d uV  	d S r   r   r   s     ra   r   z8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>  s&      22Q1D=222222rc   )r   rP   rY   rZ   r[   r]   inplace_foreach_opsrb   ri   rg   r   rm   ru   r  r  has_featureBackendFeatureFOREACHrealizeget_operation_nameregister_operation_listr,  )rX  rR  realize_outputsa_list_inputinputbroadcast_inputsgroupsoutputsrs   rr   groupoperation_list
output_indrq   rA  ry  pw_fns                  ra   r|  z%make_foreach_pointwise.<locals>.inner  sO   $*++q0 0w#*.AA0-// 	  	 	E%$// $ ''? (''
  	/ 	/EedE]33 / ''#l2C2C(CDDDD ''....#C)9$:;;&3|,,,,2LLNN 	@ 	@(!V[5(*N G G  *"UD666FF"UD\F&,
# G''0FGGG#G (G
 NN$$$"))&*C*C*E*EFFF @//???22'22222222rc   )rR  r  r   )r  ry  r|  s   `` ra   make_foreach_pointwiser    s<    45 4 4 4 4 4 4 4 4 4l Lrc   'Union[TensorBox, ShapeAsConstantBuffer]torch.dtypecopyc                    |                                  k    r|rt          |           n| S fd} t          |          |           S )Nc                2    t          j        |           S )N)	src_dtype)rO   r  )r   r   r  s    ra   	_to_dtypezto_dtype.<locals>._to_dtype  s    |Au	::::rc   rS  )r   cloner}  )r   r   r  r  r  s    `  @ra   r  r    sq     IE&uQxxxQ&; ; ; ; ; ; B>)5AAA!DDDrc   r   c                   ddl m} |}| j        } ||t          j                  }t          j        |          5   |j        |  ddd           n# 1 swxY w Y   |j        }|sJ t          |          }dgt          |          z  }	|
                                D ]\  \  }
}}g }|D ]q\  }}||	|<   t          j                            |
t          j                  r=|r;|                                 |                    |                                           r|rt          j                            |           t%          d |	D                       sJ |	S )aI  
    This lowers an invocation of foreach_map
    The way this works is that an arbitrary N-arg func is provided by the user, looped over by the
    polyfill with the same semantics as a foreach op (a loop applying an n-ary function to n args)
    and then traced into a subgraph by dynamo.
    This code allows us to inline the subgraph into the main graph lowering using the PontwiseSubgraphLowering.
    The graph outputs represent the vertically fused sequence of ops, and then register_operation_list
    below registers the buffers as horizontally fuseable in the scheduler.
    r+   )PointwiseSubgraphLowering)root_graph_loweringNc              3     K   | ]}|d uV  	d S r   r   r   s     ra   r   z_foreach_map.<locals>.<genexpr>  s&      ..q}......rc   )subgraph_loweringr  graph_modulerP   rY   set_graph_handlerrungraph_outputsru   r   r  r  r  r  r  rm   r  r  r,  )subgraphrq   r   r  rR  gmpw_subgraphsub_outputsr  r  rs   rr   r  r  r  rA  s                   ra   _foreach_mapr    s    =<<<<<F		B++BAGLLLK	
	[	)	) ! !  ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! +K;,,Ffs;'''G(. < <$u$& 	C 	C 
"(GJw""6>+ABB C{ C   %%f&?&?&A&ABBB 	<G++N;;;..g........Ns   AAAc                H   |j         s|                                 j         rq|                                 r3t          | |          }t          j                            ||            |S  t          t          j	        j
        d          | |          S t          | |d          S )Nr   Fadd_to_fallback_setTr  )
is_complexr   r   
empty_liker.   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr  )r   r   dsts      ra   _convert_element_typer    s     
1;;==3 
::<< 		 Qe,,,C"))#q111J#*2     Au4((((rc   r  c               .   |                                  }||k    r|rt          |           n| S d } ||          } ||          }||k    r( t          t          j        j                  | |          S t          t          j        | |                    S )Nc                t    | j         rt          j        |           j        S t          j        |           j        S r   )is_floating_pointr|   finfobitsiinfor   s    ra   _get_primitive_bitwidthz1to_dtype_bitcast.<locals>._get_primitive_bitwidth1  s3    " 	+;u%%**;u%%**rc   )	r   r  r  atenviewr   r@   r3   r   )r   r   r  x_dtyper  src_bitsdst_bitss          ra   to_dtype_bitcastr  ,  s    kkmmG%&uQxxxQ&+ + + '&w//H&&u--H80	00E:::)!U33444rc   c                    |j         s|                                 j         rLt          j        t          j                            t          j        j        j	        j
        | |                    S t          | |          S r   )r  r   r@   r   r.   ComplexViewr|   rO   r  r  r   r  rK  s     ra   _view_dtyper  @  sg     
1;;==3 
N!!%)."5";QFF
 
 	
 Au%%%rc   r  non_blockingc                   t          |          }|                                 |k    r|rt          |           n| S t          j        t
          j                            | ||                    S r   )rD   rl   r  r@   r   r.   r   )r   rs   r  r  s       ra   	to_devicer  I  s\    6""F||~~&uQxxxQ&BM00FLIIJJJrc   c                (    t          | |d|          S )NTr  )r  )r   rs   r  s      ra   _device_putr  P  s    QTEEEErc   Tc	                `   |p| j         }t          |          }	t          |||           |t          |          }t          |	||||          }	 t	          | |||          |	          }	t          t          |          r. t	          t          t          |          d|          |	           |	S )z3A pointwise function that maps ops.{name} to inputsN)rS  rn  ry  r{  )r   r   r   )r   r   )__name__r:   rK   r}  r5  r   r  r   )
r   r   r   r   r   rS  rn  ry  r{  rv   s
             ra   register_pointwiser  U  s     #7#D	T		B'!#8   #.&12M&N&N#	
3$?'
 
 
B
	/3	
 
 

 	

 

B ud 	
E4   $"7	
 	
 	
 		 	 	
 Irc   c                 f   d} t          d          fd}fd}t          |          t          |t          j                  gfd} t	          t
          j                  |          }t          t          |           r- t	          t          t          |           d          |           |S )z2A pointwise function that maps ops.frexp to inputsfrexpc                       | i |d         S Nr   r   rq   r   r  s     ra   frexp0zregister_frexp.<locals>.frexp0      ud%f%%a((rc   c                       | i |d         S Nr+   r   r  s     ra   frexp1zregister_frexp.<locals>.frexp1  r  rc   r  c                 <     d         | i | d         | i |fS Nr   r+   r   )rq   r   pw_fnss     ra   rv   zregister_frexp.<locals>.fn  s4    vay$)&))96!9d+Ef+E+EEErc   Nr  )
r:   r}  r|   int32r5  r  r  r   r  r   )r   r  r  rv   r  r  s       @@ra   register_frexpr    s   D  E) ) ) ) )) ) ) ) ) 	vvU[AAAF
F F F F F
	

 


 

B ud 	
E4   $	
 	
 	
 	 	 	 Irc   c                H    t          ||          }t          | |          }|S )Nry  )r  r%  )r   pointwise_lowering_fnry  rv   s       ra   register_foreach_pointwiser    s+    
 
  5;	O	O	OB	#GR	0	0BIrc   )r   r   c           	     f   d }t          |t          t          f          r t          |          |          }t          |t          t          f          r t          |          |          }| ||gt	          d         d         t
          j                  }d t                    D             }t          |t          fd|D                        D ]
\  }}||<   t          t                              D ]k}t          |         t          j                  rIt          j        |         t!          |d                                                                      |<   l t%          ||          d         t'          d         |          t'          d         |                    S )	Nc                     t          j        |  S r   )rO   whererq   s    ra   rv   zwhere.<locals>.fn  s    y$rc   r+   r)   r  c                B    g | ]\  }}t          |t                    |S r   r   r   s      ra   r   zwhere.<locals>.<listcomp>  s+    IIITQ
1i0H0HIqIIIrc   c                     g | ]
}|         S r   r   r  s     ra   r   zwhere.<locals>.<listcomp>  s    0J0J0JQa0J0J0Jrc   r   r  )ri   rF  r   constant_liker   r   rO  rh   r  r  r  r   r.   r  r4   r   rg   r   r}  r  )	r   r  r@  rv   r   indicesrp   r   rq   s	           @ra   r  r    s         !eS\""  M!Q!eS\""  M!Q!Q<DQa.M.U  E JIYt__IIIGG.0J0J0J0J'0J0J0JKLL  1Q3t99 T Td1gr{++ 	T 'Qd71:6F6O6O6Q6Q1R1RSSDG:>"E:::Q$q'5))8DGU+C+C  rc   c                    t          |           dk    r1t          | d         t          t          f          rt	          | d          S t          j        t          d | D             g           }g }| D ]}|                                }t          |          t          |          k    s't          d t          ||          D                       rt          ||          }|                    |           |S )Nr+   r   c                6    g | ]}|                                 S r   )r   r   s     ra   r   z%broadcast_tensors.<locals>.<listcomp>  s     #A#A#AQAJJLL#A#A#Arc   c              3     K   | ]Q\  }}t           j        j                            |          t           j        j                            |          k    V  Rd S r   )rP   rY   r   r<  r   r  r@  s      ra   r   z$broadcast_tensors.<locals>.<genexpr>  sj       ,
 ,
 1 G11!44w44Q778,
 ,
 ,
 ,
 ,
 ,
rc   )r   ri   rg   r   r  r   reducerC  r   r-  r  r>  rm   )rR  r]   r  r   sizess        ra   r  r    s    
6{{aJvay4-@@ &),,(/!#A#A&#A#A#A2   F G 	 	

u::V$$ ,
 ,
 E6**,
 ,
 ,
 )
 )
$
 q&!!AqNrc   c                    | S r   r   r   s    ra   nopr    s    Hrc   
lift_freshc                .   t          | t                    sJ |&t          t          j        | j                            S t          |t
          t          j        f          r$t          j	        j
                            |          nt          d |D                       }t          t          |                                           |          }t!          t          |t                    s|fn|          }g }t#          |                                           D ]U\  }}||v r7t          j	        j
                            t          j        |d                    s|                    |           V||                                 k    rt+          | |          n| S )Nc              3  ^   K   | ](}t           j        j                            |          V  )d S r   rP   rY   r   	guard_intr   ds     ra   r   zsqueeze.<locals>.<genexpr>  s5      >>Q17#--a00>>>>>>rc   r+   )ri   r@   r?   r   rk   r   r   r   rP   rY   r   r  r   r   r   r   r$   rh   guard_or_falseEqrm   r  )r   r   dims	new_shaper  r   s         ra   squeezer     s]   a#####
{+AF33444 cC,--	?""3'''>>#>>>>> 
 C

--s
3
3CJsE$:$:CsffDDDI!**,,''    1T		ag.==ehq!nnMM	Q "+ajjll!:!:49Arc   c                <    t          t          | |                    S r   )r  r   )r   r   s     ra   squeeze_copyr    s    C!!!rc   c                    t          | |          }t          | t                    sJ t          |t                    sJ |j        | _        | S r   )r   ri   r@   rk   r   r   vals      ra   squeeze_r    sI    
!S//Ca#####c9%%%%%XAFHrc   c                    t          |           rt          | dt          j                  S t	          d          } t          |t          j                  |           S )NFr   isinfr  r   	full_liker|   r   r:   r}  r   rv   s     ra   r  r  
  U    q 5E4444	W		B?>"EJ???BBBrc   c                    t          |           rt          | dt          j                  S t	          d          } t          |t          j                  |           S )NFr   isnanr  r	  r  s     ra   r  r    r  rc   c                    t          |           rt          |           S t          d          } t          |          |           S )Nceilr   r  r:   r}  r  s     ra   r  r    sB    q Qxx	V		B>"a   rc   c                    t          |           rt          |           S t          d          } t          |          |           S )Nfloorr  r  s     ra   r  r  "  B    q Qxx	W		B>"a   rc   c                    t          |           rt          |           S t          d          } t          |          |           S )Nroundr  r  s     ra   r  r  *  sB    q %Qxx!!!~b!!!$$$rc   c                    t          |           rt          |           S t          d          } t          |          |           S )Ntruncr  r  s     ra   r  r  3  r  rc   c                X   t          | g          \  } t          | t          j                  r"t	          j        | t          |                    S t          | t                    sJ t          |t          t          f          sJ t          | 	                                          t          |          k    r| S t          | 	                                          st          j        j                            t          | 	                                                    }|dk    rVt          |          sG|                     t          j        j                            t          |                    |z             t          t	          j        | j        t          |                              S r  )rU  ri   r.   rs  r4   r   r   r@   rg   r   r!   rP   rY   r   r   rM   
mark_reuserk   )r   r  x_size_products      ra   r>  r>  ;  sW   aS!!DQ!R_%% 2 E%LL111a#####edE]+++++QZZ\\eEll** .. )<<!**,,''
 
 A&;E&B&BLL 33M%4H4HII!"   Z&qvuU||<<===rc   c                    t          |          }|D ]}d||<   | }t          |          D ]\  }}|dk    rt          ||          }t          ||          S Nr]  )rg   rh   	unsqueezer>  )r  shapebroadcast_dimensionsr   broadcast_dimensionr   idxr   s           ra   broadcast_in_dimr#  V  sr    UA3 $ $!#
	AA,, " "Q77!S!!A!Urc   c                F    t          | |                                          S r   )r>  r   )r   rB  s     ra   	expand_asr%  d  s    !QZZ\\"""rc   c                V   t          |                                           t                    t                    k    rRt          j        j        gt                    t                    z
  z  z   t          | t                              } t                    t          |                                           k    sJ t          |                                           }d}t          t                              D ]$}|         dk    rd}||         |         z  ||<   %|r6t          || 	                                | 
                                          S t          d t                    D                       rt          t          | |                    S fd}t                    st          |          s~t           j        j                            t)                              }|dk    rG|                     t           j        j                            t)          |                    |z             |                                 t/          j        | 
                                | 	                                |t          |                    S )NFr   Tr   rs   c              3  4   K   | ]\  }}|d k    p|d k    V  dS r+   Nr   r  s      ra   r   zrepeat.<locals>.<genexpr>{  s3      
A
A$!QAFa1f
A
A
A
A
A
Arc   c                T   t          |           t                    k    sJ t          |           } t          t                              D ]O}|         dk    rA|         dk    rt          j        j        | |<   /t          | |         d|                   | |<   P |           S r  )r   rg   r  r   r:  Zeror(   )rJ  rp   old_sizerepeatsx_loaders     ra   rp  zrepeat.<locals>.inner_fn  s    5zzS\\))))Us7||$$ 	I 	IAqzQA;!##$w|E!HH.uQxHQKHHE!Hxrc   rq  )rg   r   r   r   r:  r;  r  r  emptyr   rl   r,  r  r  r>  r!   rP   rY   r   r   rM   r  r`  r<   r   )	r   r-  new_sizezero_tensorrp   rp  old_size_productr,  r.  s	    `     @@ra   repeatr3  i  s]   AJJLL!!H
7||c(mm##GK=CLL3x==$@AHLDNN##w<<3qzz||,,,,,,AJJLL!!HK3w<<   / /1:??KqkGAJ. KXQ[[]]1<<>>JJJJ

A
A#gx*@*@
A
A
AAA *VAx(()))	 	 	 	 	 	 	 !** 3H3R3R 7+>>}X?V?VWWa LL 33M(4K4KLL#$  
 }}H||~~kkmmH~~	   rc   r  Sequence[sympy.Expr]c                P    t          t          j        | j        |                    S r   )r@   rB   r   rk   )r   r  s     ra   r  r    s      T[//000rc   c                    t          | t                    sJ t          |t          t          f          sJ t          t	          j        | j        t          |                              S r   )ri   r@   rg   r   r;   r   rk   )r   r  s     ra   permuter7    sU    a#####dT5M*****['d<<===rc               c           
         t          | t                    sJ t          | |d          }t          t          j                            | j        |||||                    S )Nr   clamp)ri   r@   _validate_dimr.   	SliceViewr   rk   )r   r   startendstepr;  s         ra   slice_rA    sV    a#####
3
"
"CR\((eS$e(TTUUUrc   c           	        t          | t                    r8t          | j        t          j                  r| j                                        } |                                  t          j        |           st          d|  d          t          j	        |           \  }}t          j
        |j        |j        d |D             d |D             t          j        |pd                    }t          t          j        ||                    S )Nzunrealized as_strided(z, ...)c                6    g | ]}t          j        |          S r   r   r>  r   s     ra   r   zas_strided.<locals>.<listcomp>  s     '''Qa'''rc   c                6    g | ]}t          j        |          S r   rD  r   s     ra   r   zas_strided.<locals>.<listcomp>       )))Qa)))rc   r   rk   layout)ri   r@   rk   r.   r2   unwrap_viewr  is_storage_and_layoutr   as_storage_and_layoutFixedLayoutrs   r   r   r>  r   )r   r   stridestorage_offsetstorage
old_layout
new_layouts          ra   
as_stridedrR    s    !Y !Jqvr{$C$C !F  IIKKK#A&& F!"D1"D"D"DEEE2155GZ''$'''))&)))^(q)) J R'WZHHHIIIrc   c                l    t          | t                    sJ t          | |||          j        | _        | S r   )ri   r@   rR  rk   )r   r   rM  rN  s       ra   as_strided_rT    s5    a#####488=AFHrc   c                D    t          | |||          }t          |          S r   )rR  r  )r   r   rM  rN  results        ra   as_strided_copyrW    s!    488F==rc   c                    g d} D ]B}                     |||                                         z   f           d         d         }Cd  D              fd}t           d                                                   }d         d         |<   t          j         d                                          d                                         ||          S )Nr   r]  c                6    g | ]}|                                 S r   r_  r\  s     ra   r   z!pointwise_cat.<locals>.<listcomp>  s"    :::Ccoo'':::rc   c           	       
 t          j        |          t          j                  }g }g }t	          t                              D ]X

dk    rt          j        dt          j                  n*t          j        
         d         t          j                  }t          j        
         d         t          j                  }t          j        ||          }t          j        ||          }
dk    r|}n.
t                    dz
  k    r|}nt          j	        ||          }|
                    |           t          |           t                   
         d         z
            <   |
                    t          j        |
fdd                     Z|d         }	t	          t                    dz
  dd          D ]$
t          j        |
         |
         |	          }	%|	S )Nr   r+   c                 &                         S r   r   )rp   idx_loadinputs_loaderss   ra   <lambda>z1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>  s    -N1-h77 rc           r]  r)   )rO   
index_exprr|   int64r  r   constantgeltand_rm   rg   r'   maskedr  )r"  idx_dimmasksmasked_loadsr>  r?  
start_condend_condmasknext_valrp   r\  r   rR  r]  inputs_rangess             @@ra   rp  zpointwise_cat.<locals>.inner_fn  s   .S5;77s6{{##  	  	A 66 Q,,,^M!$4Q$7EE 
 .q!1!!4ekBBC//Jvgs++HAvvc&kkAo%%!x
H55LLCyyH %Xc]]15Ea5H%HIIHSM
777777      #Fq("b11 	 	AyaQ HH
 rc   rq  )rm   r   rg   r<   r   rl   r   )rR  r   prev_endr   rp  r0  r]  rn  s   ``    @@ra   pointwise_catrp    s   9;MH ) )h3<<>>#3F(FGHHH $R(::6:::N. . . . . . . .` F1I&&(())H!"%b)HSMay##%%Qi!!##	   rc   r  scaleszero_pointsaxis	quant_min	quant_maxc           	     v  	
 t                                                    dk    s
J d            t                                                    dk    s
J d            |                                 t          j        k    rt          | t          j                  } |                                 t          j        k    sJ d|                                              t          |                                           k     s,J dt          |                                                        |                                                                 	                                
	
f	d}t          j	        | 
                                ||                                           S )Nr+   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                <  	 | 
         f} |           } |          } |          }t          t          j                  \  }}j        t          j        k    rt	          j        |t          j                  }j        t          j        k    rt	          j        |t          j                  }t	          j        |          }t	          j        ||z            |z   }t	          j	        |t	          j
        ||                    }	t	          j        |	          S Nr   )_create_constantsr|   float32r   rO   r  r  
reciprocalr  maximumminimum)r"  channel_idxr  scale
zero_pointqminqmax	inv_scaler  clampedrs  r   input_loaderru  rt  rq  scales_loaderrr  zero_points_loaders             ra   rp  z;quantized_decomposed_quantize_per_channel.<locals>.inner_fn2  s    4ylS!!k**''44
&y)5=QQQ
d<5=((L66E++j%+>>JN5))	i	)**Z7+dCKc$:$:;;|GU+++rc   rq  )r   r   r   r|   rt  r  r~  r`  r<   r   rl   )r  rq  rr  rs  rt  ru  r   rp  r  r  r  s    `````` @@@ra   )quantized_decomposed_quantize_per_channelr    s    v  !!Q&&&(=&&&{##%%&&!+++-G+++EN**..??---ZuGXGXZZ .-- #enn&&'''''9#enn&6&6"7"799 ('' $$&&L&&((M$0022, , , , , , , , , , , , ," !!~~	   rc   c           	     P                                       t           t          j                    fd}t	          j                                                                          |t           	                                                    }|                                  |S )Nc                    t           j                                        5  t          j                                         |                     cd d d            S # 1 swxY w Y   d S r   )r.   ComputedBufferforce_realizerO   device_assert_asyncr`  )rJ  r   r   s    ra   rp  z_assert_async.<locals>.inner_fnO  s    ,,.. 	K 	K*+=4+;+;+=+=e+D+DcJJ	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	Ks   0AA!$A!rq  )
r  r  r|   r   r<   r   rl   r   rg   r   )r   r   rp  assertion_ops   ``  ra   _assert_asyncr  K  s    LLNNND%*%%DK K K K K K #  nnDMMOO$$	  L rc   c                "    t          | |          S r   r  r   s     ra   lower_assert_asyncr  ]      s###rc   c                "    t          | |          S r   r  r   s     ra   lower_assert_functional_asyncr  b  r  rc   )	out_dtyper  Optional[torch.dtype]c                 	
 t                                                    dk    s
J d            t                                                    dk    s
J d            |                                 |k    s"J d| d|                                              t          |                                           k     s,J dt          |                                                        t          j        |                                 	                                
                                	
fd}t          j        |                                 ||                                           S )	Nr+   rw  rx  Expecting input to have dtype , but got dtype: rz  c                   |          f} |           } 
|          } |          }	j         t          j        k    rt          j        |t          j                  }j         t          j        k    rt          j        |t          j                  }t          j        t          j        |t          j                  |          |z  }t          j        |          }|S r   )r   r|   r~  rO   r  sub)r"  r  r  r  r  r  rs  r  r  rq  r  rr  r  s         ra   rp  z=quantized_decomposed_dequantize_per_channel.<locals>.inner_fn  s    4ylS!!k**''44
<5=((L66E--j%-@@Jgcl5%-88*EEMl3	**
rc   rq  	r   r   r   r|   r~  r`  r<   r   rl   )r  rq  rr  rs  rt  ru  r   r  rp  r  r  r  s    ```   ` @@@ra   +quantized_decomposed_dequantize_per_channelr  g  s    v  !!Q&&&(=&&&{##%%&&!+++-G+++??%%%TTTARARTT &%% #enn&&'''''9#enn&6&6"7"799 ('' M	$$&&L&&((M$0022           !!~~	   rc   r  rF  r  c                   |                                  t          j        k    rt          | t          j                  } |                                  t          j        k    sJ d|                                               |                                 fd}t          j        |                                 t          j
        |t          |          t          |                    |                                           S )Nry  c                J    
|           }t          d|z  |t          j                  \  }}t          j        ||z            |z   }t          t          j                  \  }}t          j        t          j        ||          |          }t          j        |	          S )N      ?r   )r}  r|   r~  rO   r  r  r  r  )r"  r  r  r  r  r  r  r  r  r   r  ru  rt  s            ra   rp  zBquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn  s    S!! 1%K5=!
 !
 !
	: i	)**Z7&y)5=QQQ
d+ck#t44d;;|GU+++rc   r  r  rq  )r   r|   rt  r  r~  r`  r<   r   rl   r   r4  rF  r   r   )r  r  r  rt  ru  r   rp  r  s      ``` @ra   0quantized_decomposed_quantize_per_tensor_defaultr    s
    EN**..??---ZuGXGXZZ .-- $$&&L, , , , , , , , !!"E%LLS__
 
 
 ~~   rc   c                  |                                  |k    s"J d| d|                                               t          j        |                                 fd}t	          j        |                                 t          j        |t          |          t          |                    |                                           S )Nr  r  c                     |           }t          ||t          j                  \  }}t          j        t          j        |t          j                  |          |z  }t          j        |          }|S r|  )r}  r|   r~  rO   r  r  )r"  r  r  r  r  r  r  s        ra   rp  zDquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fn  sg    S!!-eZu}UUUzgcl5%-88*EEMl3	**
rc   r  rq  )r   r|   r~  r`  r<   r   rl   r   r4  rF  r   r   )	r  r  r  rt  ru  r   r  rp  r  s	         ` @ra   2quantized_decomposed_dequantize_per_tensor_defaultr    s     ??%%%TTTARARTT &%% M	$$&&L      !!"E%LLS__
 
 
 ~~   rc   c                  	 |                                  t          j        k    rt          | t          j                  } |                                  t          j        k    sJ d|                                               t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            |                                                                                                 		fd}t          j	        | 
                                ||                                           S )Nry  r   r+   expect scale as scalar tensor"expect zero_point as scalar tensorc                    	|           } t                                                    dk    rdnd          } t                                                    dk    rdnd          }j        t          j        k    rt          j        |t          j                  }j        t          j        k    rt          j        |t          j                  }t          j        |t          j        |          z            |z   }t          
t          j                  \  }}t          j
        t          j        ||          |          }t          j        |          S )Nr+   r   r   r   )r   r   r   r|   r~  rO   r  r  r  r}  r  r  )r"  r  _scale_zero_pointr  r  r  r  r   r  ru  rt  r  scale_loaderr  zero_point_loaders           ra   rp  zAquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn  s#   S!!c%..*:*:&;&;q&@&@ddbII''ENN4D4D0E0E0J0JPRSS;%-''\&%-88Fu},,,{EMBBKiv 6 6677+E&y)5=QQQ
d+ck#t44d;;|GU+++rc   rq  )r   r|   rt  r  r~  r   r   r`  r<   r   rl   )
r  r  r  rt  ru  r   rp  r  r  r  s
    ````` @@@ra   /quantized_decomposed_quantize_per_tensor_tensorr    s    EN**..??---ZuGXGXZZ .-- u~~  A%%ENN""u~~'7'7':a'?'?'?& (@'? z""$$%%**J!!""a''J,?,?,A,A!,D,I,I,I+ -J,I $$&&L$$&&L"..00, , , , , , , , , , , , !!~~	   rc   c               j  	
 t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            t                                                    dk    sMt                                                    dk    r                                d         dk    s
J d            |                                 |k    s"J d| d|                                              t          j        |                                                                 	                                
	
fd}t          j        |                                 ||                                           S )	Nr   r+   r  r  r  r  c                J    |           } t                                                    dk    rdnd          } 
t                                                    dk    rdnd          }j        t          j        k    rt          j        |t          j                  }	j        t          j        k    rt          j        |t          j                  }t          j        t          j        |t          j                  |          |z  }t          j        |          }|S )Nr+   r  r   )r   r   r   r|   r~  rO   r  r  )r"  r  r  r  r  r  r  r  r  r  r  s        ra   rp  zCquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fn8  s    S!!c%..*:*:&;&;q&@&@ddbII''ENN4D4D0E0E0J0JPRSS;%-''\&%-88Fu},,,{EMBBKgcl5%-88+FFOl3	**
rc   rq  r  )r  r  r  rt  ru  r   r  rp  r  r  r  s    ``   ` @@@ra   1quantized_decomposed_dequantize_per_tensor_tensorr    s    u~~  A%%ENN""u~~'7'7':a'?'?'?& (@'? z""$$%%**J!!""a''J,?,?,A,A!,D,I,I,I+ -J,I ??%%%TTTARARTT &%% M	$$&&L$$&&L"..00
 
 
 
 
 
 
 
 
 
 !!~~	   rc   c                h  
 | d                                          j        dk    }|rt          d | D                       rt| D ]}|                                 t          d | D                       rt	          t
          j        g| R  \  } } t          t
          j        j                  | |          S t          |           dk    rt          | d                   S t          | d         |d          }t          | dt          j        ifd| D             } ddd fdt          fd| D                       }dfdt           j        rt%          | |          S |r-t'          t(          j                            | |                    S fdd}d
ddt          |           |k    s9t          |           t           j        k    rt          
fd| D                       rt          fdt0          j        j        D                       }t          fd| D                       o|}t          fd| D                       ot          fd| D                        }	|s|	r|st%          | |          S t'          t(          j                            | |                    S ) Nr   r   c              3  n   K   | ]0}|                                 t          j        t          j        fv V  1d S r   )r   r|   int8uint8r   r  s     ra   r   zcat.<locals>.<genexpr>O  sK        ;@ej%+66     rc   c              3  `   K   | ])}t          |                                          d k    V  *dS )r   N)r   r   r  s     ra   r   zcat.<locals>.<genexpr>V  s9      >>es5>>##$$)>>>>>>rc   r+   r   c                0    g | ]}t          |          S r   r  )r   r   r   s     ra   r   zcat.<locals>.<listcomp>a  s#    555shsE""555rc   r   Union[TensorBox, ir.StorageBox]rx   	ir.IRNodec                    t          | t                    r?t          | j        t          j                  r| j                                        S | j        S t          | t          j                  r| j        S | S r   )ri   r@   rk   r.   r2   rI  r   r   s    ra   unwrap_tensorzcat.<locals>.unwrap_tensorc  se    a## 	!&"+.. v))+++va'' 	6Mrc   c                t    t          | t          j                  ot          | j        t          j                  S r   )ri   r.   r  rk   r=   )rt   s    ra   is_reductionzcat.<locals>.is_reductiono  s(    !R.//TJqvr|4T4TTrc   c                   t          | t          t          j        f          r  |                     S  |           pFt          | t          j                  o,t          fd|                                 D                       S )Nc              3  h   K   | ],} t           j                            |                    V  -d S r   )rP   rY   
get_buffer)r   readcan_fuse_reductions     ra   r   z2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>x  sR         #"17#5#5d#;#;<<     rc   )ri   r@   r.   r   r<   r-  get_read_names)rt   r  r  r  s    ra   r  zcat.<locals>.can_fuse_reductionr  s    a)R]344 	8%%mmA&6&6777LOO !R\**     ,,..    	
rc   c              3  .   K   | ]} |          V  d S r   r   r   rt   r  s     ra   r   zcat.<locals>.<genexpr>  s/      BBa..q11BBBBBBrc   r   c                D   t          j        |           r9t          j        | d          \  }}t           j                            |           S t          | t          t           j        f          r  |                     S t          | t           j                  rdS dS )NF)freezeT)	r.   rJ  rK  ConcatKernelcan_realize_into_without_copyri   r@   r   r<   )r   rO  r   should_lower_cat_inputr  s      ra   r  z#cat.<locals>.should_lower_cat_input  s     #A&& 	N1!EBBBJGQDDWMMMMa)R]344 	<))--*:*:;;;a&& 	4urc   c                `   t          | t          t          j        f          r  |                     S t          | t          j                  sdS |                                 j        }|                                 D ]-}| t          j	        
                    |                    z  }.|S r  )ri   r@   r.   r   r<   inner_fn_opcountnum_opsr  rP   rY   r  )r   countr  op_countr  s      ra   r  zcat.<locals>.op_count  s    a)R]344 	.8MM!,,--- !R\** 	1""$$,$$&& 	8 	8DXXag0066777EErc   r   r)   r\   torch._ops.OpOverloadc                J    | t           j        j        t           j        j        fv S r   )r  catr  constant_pad_ndr\   s    ra   additional_pointwise_opsz%cat.<locals>.additional_pointwise_ops  s    dh&(<(DEEErc   c              3  6   K   | ]} |          k    V  d S r   r   )r   rt   MAX_SIMPLE_OP_COUNTr  s     ra   r   zcat.<locals>.<genexpr>  s1      CCq22CCCCCCrc   c              3  8   K   | ]}t          |          V  d S r   )rG   )r   user  s     ra   r   zcat.<locals>.<genexpr>  sB       
 
 S":;;
 
 
 
 
 
rc   c              3  .   K   | ]} |          V  d S r   r   r   r   r  s     ra   r   zcat.<locals>.<genexpr>  s/      >>&&s++>>>>>>rc   c              3  .   K   | ]} |          V  d S r   r   r  s     ra   r   zcat.<locals>.<genexpr>  s@       "
 "
,/""3''"
 "
 "
 "
 "
 "
rc   c              3  .   K   | ]} |          V  d S r   r   r  s     ra   r   zcat.<locals>.<genexpr>  s/      <<((++<<<<<<rc   )r   r  rx   r  )rx   r   r\   r  )rl   r   r,  r  require_channels_lastr  r  r  r  r   r  r<  r   r   rO  r-  r,   force_pointwise_catrp  r@   r.   r  r   max_pointwise_cat_inputsrP   rZ   r[   )rR  r   
cpu_devicer  r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_usehorizontal_fuse_catr  r  r  r   r  r  r  r  s             @@@@@@@@ra   r  r  L  s   %%'',5J 	?c  DJ     	?
  	 	EMMOOOO>>v>>>>> 	A-dh@@@@IFA1 011&#>>>
6{{aVAY
q	3
*
*C	%D%L E 6555f555F
 
 
 
U U U

 

 

 

 

 

 

 BBBB6BBBBB       ! *VS)))  >//<<===     $ !"F F F F 6{{///	V7	7	7CCCCCFCCCCC 
8  
 
 
 
~+
 
 
 
 
 >>>>v>>>>>Q> 	 " "
 "
 "
 "
39"
 "
 "
 
 
 =<<<<V<<<<<< 	  	."5 	.>O 	. ---R_++FC88999rc   offsetdim1dim2c                  	
 |                                  
t          
          	t          	          t          	          t          k    fd           t          j        j                            t          j	        |d                    }|rXt          j        j        
                    t          j        j                            
         |z   
                   d          }nWt          j        j        
                    t          j        j                            
         
         |z
            d          }d|r| dfnd|ffdt          
          D             }|                    |           	
fd}t          t          j                            | ||                    S )N)r"  rankc                     d  d S )Nz(diagonal dimensions cannot be identical z, r   )r  r  s   ra   r^  zdiagonal.<locals>.<lambda>  s    WWWQUWW rc   r   )r   r   c                &    g | ]\  }}|fv|S r   r   )r   rp   r   r  r  s      ra   r   zdiagonal.<locals>.<listcomp>  s+    NNN41a$8M8MQ8M8M8Mrc   c                
   | d         }dgt          	          z  }d}t                    D ]<}|k    r|d         z   ||<   |k    r|d         z   ||<   ,| |         ||<   |dz  }=|t          	          dz
  k    sJ |S )Nr]  r   r+   r)   )r   r  )
r"  diag_idxoriginal_idxcur_dimr  base_idxr  r  num_dimsoriginal_shapes
        ra   	reindexerzdiagonal.<locals>.reindexer  s    r7sS000x 	 	ADyy"*Xa["8Qd"*Xa["8Q"%g,Q1#n--11111rc   )r   r   r   r   rP   rY   r   evaluate_exprr   Ltevaluate_maxevaluate_minrh   rm   r@   r.   GenericViewr   )r  r  r  r  offset_negative	diag_sizer  r  r  r  r  s     ``    @@@ra   diagonalr    s   ^^%%N>""H8444D8444D	WWWWW   g&44UXfa5H5HIIO 
G$11G))t$v-~d/C  	
 
		 G$11G))t$nT&:V&C  	
 
	 H GQ<v;NNNNN9^44NNNE	LL          R^**5%CCDDDrc   c                @    t          t          | |||                    S r   )r  r  )r  r  r  r  s       ra   diagonal_copyr     s    %t44555rc   c                h    t          |           }t          ||||          }t          ||           |S r   )r  r  	mutate_to)r  srcr  r  r  rA  r]   s          ra   diagonal_scatterr    s5    5\\FffdD11FfcMrc   c                l   t          j        |          }t          j        |                                 |                   }d }t          j        j                            t          j        |d                    r||z   }n9t          j        j                            t          j        |d                    r|}|t          |          re|                                 }| 
                                }|                                 j        ||         |z  z   }||= ||= t          | |||          S t          | |||dz             }t          ||          S t!          t          j        j        j        t          j        j        j        d                   }	|	J t)          |	          dk    s
J |	            t+          t-          |	                                                    \  }
}|                                 }| 
                                }|
}t1          j        |
||                                 j        ||         |                                 |                   }t          j                            |          |_        t          j                            |           ||= ||= t          | |||          S )Nr   r+   unbacked_bindings)r   r>  r   rP   rY   r   r  r   Ger"   
get_stride
get_layoutr  rR  rA  r   r#   	shape_envrZ   rv  r   rP  iterr  r.   DynamicSelectStorageOffsetregister_bufferr   register_operation)r   r   r"  r   actual_indexr0  
new_stridenew_storage_offsetslice_resultr  unbacked_offset_symr   buffers                ra   selectr    sa   
,s

C<

S)**DLw&&uxQ'7'788 Tz	
		(	(#q)9)9	:	: $S)) 	. zz||HJ!"!6C<9W!W3a:7IJJJ!!S,q8HIIL<--- 2	"AG$8$=>Q$R  ((( !!Q&&&(9&&&!$'8'>'>'@'@"A"ABBzz||HJ,*	3	

S F '))&11FKGv&&&3a:/ABBBrc   c           
        t          | |d          }|}t          |t          t          f          sf|                                 |         }t
          j        j                            t          ||z   dz
  |                    }|g|z  }||dz
  |z  z
  |d<   g }d}|D ]0}||z   }	|
                    t          | |||	d                     |	}1|S )Nr   r+   r]  Fr:  )r<  ri   rg   r   r   rP   rY   r   r  r&   rm   rA  )
r   r  r   sizes_x_sizechunksrV  r>  r   r?  s
             ra   splitr"  G  s    
3
"
"CF edE]++ 3c"!++HVe^a5G,O,OPP6!vzU22r
 FE  dl 	fQUCu===>>>Mrc   c                $    t          | ||          S r   )r"  )r   r  r   s      ra   split_with_sizesr$  b  s    E3rc   c                     t           d          t          j        j                                                                                } fdt          |          D             }|S )Nr   c                2    g | ]}t          |          S r   )r  )r   rp   r   r   s     ra   r   zunbind.<locals>.<listcomp>k  s%    777AfQQ777rc   )r<  rP   rY   r   r  r   r  )r   r   r   rV  s   ``  ra   unbindr'  g  sa    
3
"
"CW''

S(9::F77777v777FMrc   c                   |                                  }t          |          }t          ||          |dk    rt          t	          | d          |          S |         }t
          j        j        }|                    ||           |	                    d           t          ||z
            dz   }|                    |          dk    r9|                     |                    t          ||z  |                               g |d          ||dz   d          |}	fd}
t          t          j                            | |	|
                    S )Nr   )r?  r+   c                d    | d         |          z  z   }g | d          || dz   d         R S )Nr]  r+   r   )r"  dim_idxr   r@  s     ra   r  zunfold.<locals>.reindexer  sF    b'CHtO+8TcT8G8c#'B,&7888rc   )r   r   r   rA  r  rP   rY   r   	check_leqcheck_ltr&   r   r  r%   r@   r.   r  r   )r   	dimensionr   r@  r  ndimdim_sizer   new_dim_sizeout_sizer  r   s      `       @ra   unfoldr2  o  sh   JJLLEu::D
4
+
+Cqyyi1oo40000SzHwHtX&&&aHtOT22Q6L""8,,q00	''t0CX(N(NOO	
 	
 	
 EttDlDU3799-=DtDH9 9 9 9 9 9 R^**1h	BBCCCrc   c                    t          | |d          }t          |                                           }|                    |t          j        j                   t          | |          S r  )r<  rg   r   insertr   r:  r;  r  )r   r   r  s      ra   r  r    sR    
3
"
"CQZZ\\""IS%'+&&&9rc   c                    t          | |          }t          | t                    sJ t          |t                    sJ |j        | _        | S r   )r  ri   r@   rk   r  s      ra   
unsqueeze_r6    sK    
As

Ca#####c9%%%%%XAFHrc   c                   t           j        j        j                            t          j        |                    }t          |                                           }|dk     r|||z   z  }d|cxk    r	||z   k     sn J |S r  )	rP   rY   r   r  r  r   sympifyr   r   )r   r   r  r.  s       ra   r<  r<    s    
'

$
2
25=3E3E
F
FCqzz||D
Qwwtf}####dVm######Jrc   r]  c                *   t          | |d          }t          j        j                            |                                 |                   dz  }t          | |d|          }t          | |||dz            }t          |t          |                    S )Nr   r)   )	r<  rP   rY   r   r  r   rA  rr  sigmoid)r   r   new_lenr  r@  s        ra   glur<    s}    
3
"
"Cg((c):;;q@Gq#q'""Aq#w!,,Aq'!**rc   c                X     |rt                                            fd}d|_        |S )Nc                 d    d }t          j        |t          j        j        g| R i |          S )Nc                b    t          | t          j                  rt          j        |           n| S r   )ri   r.   r6   r@   r   r   s    ra   wrap_tensorsz7fallback_handler.<locals>.handler.<locals>.wrap_tensors  s)    *4Q	*B*BI9#A&&&Irc   )pytreetree_mapr.   FallbackKernelr   )rq   r   r@  kernels      ra   handlerz!fallback_handler.<locals>.handler  sN    	J 	J 	J "+26KDKKKFKK
 
 	
rc   T)r*  r   _is_fallback_handler)rD  r  rE  s   `  ra   r  r    sE     f
 
 
 
 
 $(G Nrc   c                 .    t          j        d           d S )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   rc   ra   _warn_complex_not_supportedrJ    s#    Mt    rc   rt   torch.Tensorc                   |                                  rt                       dS | j        rdS | j        t          j        k    r|sdS t          |j        t          j        j	                  rI|j        t          j        j        t          j        j        t          j        j        t          j        j        fv p7t          |j        t          j        j	                  ot!          |j                   S dS )z0Do not support reading or writing to this tensorTF)r  rJ  is_metar   r|   float8_e8m0fnuri   r]   r}   r~   r  r  r  r  r  
_scaled_mmrH   )rt   r_   s     ra   unsupported_input_tensorrP    s    ||~~ #%%%ty tw%&&& 	4
 t{EJ$9:: 	 
"'	Y 4;
(=>>W74;CWCW

 
	
 5rc   c                    t           j        j        t          j        j        j        j        f}||j        |v r| 	                                rdS t          | |          rdS | j        ot          j        S )z2Do not support writing tensor but can read from itNFT)r  r  r   r|   rO   r  r  r  r]   r  rP  is_cpur,   disable_cpp_codegen)rt   r_   supported_complex_viewss      ra   unsupported_output_tensorrU    sn     			,4 DK+BBBq||~~Bu4(( t8222rc   r_   torch.fx.Nodec                     j         t          j        j        u rdS  j        dk    rdS  j         t          j        j        u rdS  fd}t          j         j        i  j	        D ]} ||d          r dS  | d          S )NFplaceholderc                6   t          | t          j        j                  sdS d| j        vrdS t          j        | j        d                   D ]K}t          |t          j        j                  s"|rt          |          r dS 8t          |          r dS LdS )NFr  T)ri   r|   fxNoderv  rA  tree_leaves_subclasses
FakeTensorrU  rP  )inp_out_node	is_outputrv  r_   s      ra   check_skip_conditionzCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition  s    ,66 	5)))5&|'8'?@@ 		  		 DdE$5$@AA   ,T488  44  ,D$77  44  urc   )r`  T)
r]   r  view_as_complexr  r\   lift_fresh_copyrA  arg_tree_leavesrq   r   )r_   allow_cpu_inputsra  r   s   `   ra   %fallback_node_due_to_unsupported_typerf    s    {d*222uw-u {d*222u    * %ty@DK@@  u555 	44	  5555rc   c                   | t           vs|sJ d|              |rt          t          j        d                    rt	          | g          rt
          j        r| t          j        j	        j
        v s[|sYt          j        j        j        r0dt          j        j        _        t                              d           t          d|  d          fd}t!          | t          j        j                  r4|                                 D ]}t)          | |          } ||           d S t!          | t          j        j        t          j        j        f          r ||            d S t/          d|  d	t1          |                      )
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                    t          |            t          |             t          | d           t          |                     S Nr  )r   r   r5  r  )op_overloadlayout_constraints    ra   register_fallbackz(make_fallback.<locals>.register_fallback?  sU    !+...(!+/@AAAG $GGG[))
 
 	
rc   zUnsupported fallback z with type )r0   r   osgetenvr1   r,   fallback_randomr|   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningr   ri   r}   r   r   r   r~   HigherOrderOperatorRuntimeErrorr   )r\   rl  rI  override_decomprm  olrk  s    `     ra   make_fallbackr|    s   ^####9R99 $#6 	
4!!
 t$$
 "
 em:OOO P =/ 	38EM 0KKH   fR f f f
 
 	

 
 
 
 
 "ej122 N,,.. 	+ 	+B!"b//Kk****	+ 	+ 
B.
0NO	P	P N"L2LL$r((LLMMMrc   c                P    d}| D ]}||z  }t          |t          j                  S )z
    TorchInductor offset calculation differs from PyTorch eager offset
    calculation for random ops (tl.rand vs torch.rand). In future, we should
    strive for same impl for tl.rand and torch.rand.
    r+   r   tensorr|   ra  )r  numelr   s      ra   philox_rand_offsetr  Q  s8     E  	%u{++++rc   c                f  	
 t          j        || t           j                            |                                                     
|                                |                                		
fd}t          j        ||t          |                     }t          |           }||fS )Nc                n   t          j         g           t          j                  }t          j         g           t          j                  }t          j        t          j         |           t          j                  |          }t          j        ||          }t          j        |          S r   )rO   r  r|   r  r   r`  rand)	rJ  seed_index_exproffset_index_exprrand_index_exprrV  r   offset_loader
random_posseed_loaders	        ra   rp  zphilox_rand.<locals>.inner_fnj  s     ,{{2DDLr):):EKHH'N::e,,ek::<M
 
 
 
 |FE***rc   rq  )
r.   rL  FlexibleLayoutcontiguous_stridesmake_indexerr`  r<   r   rg   r  )r   seedr  rM  rs   r   rp  random_values_nodeoffset_noder  r  r  s        `   @@@ra   philox_randr  ]  s     
,,T22	 
 lnn  ""$$K&&((M+ + + + + + + + #)Dzz	   %T**K{**rc   c           	         t           j        rNt          j        t          j        t          j                            t          j	        j
        | ||                    S t          d          )Nz&should be handled in replace_random.py)r,   rp  rA  rB  r@   r   r.   rC  r  native_dropoutr  r   )r   ptrains      ra   r  r    sY     G$$T%8%@!QNN
 
 	

 EFFFrc   c                t   t           j        s4|                                 t          j        d          k    s
J d            |                                  t          |          dk    st          |d         t                    rt          j
        j        nt          j
        j        }t          j        || g|R   | S )Nr   Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r,   rp  rl   r|   rs   r  r   ri   rF  r  
bernoulli_Tensorr.   InplaceBernoulliFallback)r   rq   rk  s      ra   r  r    s    ! Q\\^^u|E7J7J%J%J%J^ &K%JJ IIKKK t99>>ZQ77> 	_# 
 Q66666Hrc   c                    t           j        s4|                                 t          j        d          k    s
J d            t          t          |           g|R  S )Nr   r  )r,   rp  rl   r|   rs   r  r  )r   rq   s     ra   bernoulli_pr    sY    ! Q\\^^u|E7J7J%J%J%J^ &K%JJ eAhh&&&&&rc   c                    t           r   r   )r   s    ra   _foobarr    s    
rc   c                :    t                               d           d S )Nz1using triton random, expect difference from eager)rv  info)salts    ra   _warn_triton_randomr    s    HH@AAAAArc   c                 B    t          t          j        j                   d S r   )r  rP   rY   creation_timer   rc   ra   warn_triton_randomr    s    -.....rc   c                     |                     dd           t          | i |S t          j        r"|                    dd            t          | i |S t          d          N	generatorz-should have been handled in replace_random.py)rw  fallback_rand_generatorr,   rp  popfallback_rand_defaultr   rq   r   s     ra   r  r    sj    zz+t$$0&7777		 6

;%%%$d5f555
H
I
IIrc   c                     |                     dd           t          | i |S t          j        r"|                    dd            t          | i |S t          d          r  )rw  fallback_randn_generatorr,   rp  r  fallback_randn_defaultr   r  s     ra   randnr    sj    zz+t$$0'8888		 7

;%%%%t6v666
H
I
IIrc   c                j    t          j        |          }t           j                            | |          S r   )r.   get_stride_orderExternKernelrequire_stride_order)input_tensorrM  stride_orders      ra   inductor_force_stride_orderr    s*    &v..L?//lKKKrc   c                     t          d          )Nz.should be handled in fuse_seed_creation_pass()r  )rs   s    ra   inductor_seedr    s    
I
J
JJrc   c                    t                       t          j        t          j        | t          |                              S r   )r  r@   r   r.   RandomSeedsrD   )r  rs   s     ra   inductor_seedsr    s3    BN5-2G2GHHIIIrc   c                      fd}t          j                                                                          |g           S )Nc                R    t          j                                                  S r   )rO   	load_seedget_name)r   rJ  seedss    ra   rp  z&inductor_lookup_seed.<locals>.inner_fn  s    }U^^--u555rc   rq  )r<   r   rl   r   )r  rJ  rp  s   `` ra   inductor_lookup_seedr    s`    6 6 6 6 6 6 !!oo	   rc   r  r   	list[int]r  modestrc                 	 t           j        rJ dv sJ g | } t          j        }|                                }t          j        ||| t
          j                            |           |          	                                |
                                		fd}t          j        |||g |           }|                                 |S )N)r  r  r  c                     t          t                     g           t          j         |           t          j                            S r   )r   rO   r`  r|   r  )rJ  r  r  r  s    ra   rp  z!inductor_random.<locals>.inner_fn  sG    !wsD!!KOON::e,,ek::
 
 	
rc   rq  )r,   rp  r|   r~  rQ  r.   rL  r  r  r  r`  r<   r   r  )
r   r  r  r  r   rs   rp  rV  r  r  s
     `     @@ra   inductor_randomr    s    %%%%$$$$$T7DME%%''FtR.AA$GGPV  lnn  ""$$K
 
 
 
 
 
 
 w	  F NNMrc   lowhighc               j   	 t           j        rJ g |}t          j        }|                                }t          j        |||t
          j                            |          |          	                                |
                                	 	fd}t          j        |||g |          S )Nr  c           	        t          j         g           t          j         |           t          j                  t          j        t          j                  t          j        t          j                            S r   )rO   	randint64r`  r|   r  ra  )rJ  r  r  r  r  s    ra   rp  z"inductor_randint.<locals>.inner_fn	  s_    }KOON::e,,ek::N3,,N4--	
 
 	
rc   rq  )r,   rp  r|   ra  rQ  r.   rL  r  r  r  r`  r<   r   )
r  r  r   r  r  r   rs   rp  r  r  s
   ``      @@ra   inductor_randintr  	  s     %%%%T7DKE%%''FtR.AA$GGPV  lnn  ""$$K
 
 
 
 
 
 
 
 w	   rc   tb.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]c                    |                                  |                                 d         |                                 d         |                                 d         z  |                                 d         fS Nr]  r   )r  r   r  r  s    ra   _boundaries_helperr  (	  sY    

b
a2==??1--
	 rc   tuple[str, sympy.Expr]c                ^    |                                  |                                 d         fS r  )r  r  r  s    ra   _sorter_helperr  1	  s"    ;;=="--//"---rc   	out_int32rightsidesortersorted_sequenceselfr  r  r  Optional[str]r  Optional[TensorBox]c               x   
 d } |           r ||          r: |          s/ t          t          j        j        d           |||          S ||dk    rd|rt          j        nt          j        
|                                                                                                   t           
                                          dk    r

 fd}n	
 fd	}|                                }t          j        |
||j        
          }	|	                                 |	S )Nc                V    t           j                            | t          j                  S r   )rP   rY   r  r  	BUCKETIZEr  s    ra   r^  zsearchsorted.<locals>.<lambda>?	  s     AG$7$7
N$% % rc   Fr  r  r  Tr+   c           
          |           }t          j        |t                    dd nt                    d nd          S )Nr   r  sorter_indicesrO   	bucketizer  r  )r"  r  index_dtyper  r  r  values_loaders     ra   rp  zsearchsorted.<locals>.inner_fnd	  s`    -$$C="?33%~tt>&3I3I'-~tt1   rc   c                                 }d fd}t          j        |t                     |          d nt                    d n
 |                    S )Nr  r@   c                    |                                  }t          j        t          j        t
          j        d t          |d d         d d                   D                                 S )Nc              3  &   K   | ]\  }}||z  V  d S r   r   )r   r   rp   s      ra   r   zNsearchsorted.<locals>.inner_fn.<locals>.get_flattened_index.<locals>.<genexpr>{	  s*      &U&UAq1u&U&U&U&U&U&Urc   r]  )r  rO   r`  r   r  operatorr   r  )r  stridesr"  r  s     ra   get_flattened_indexz;searchsorted.<locals>.inner_fn.<locals>.get_flattened_indexw	  sj    --//~$ &U&UWSbS\3sPRs89T9T&U&U&U   	  rc   r  )r  r@   r  )r"  r  r  r  r  r  r  r  s   `  ra   rp  zsearchsorted.<locals>.inner_fnr	  s    -$$C       ="?33##O44%~tt>&3I3I'-~tt;N;Nv;V;V   rc   rq  )r  r  searchsortedr  r|   r  ra  r`  r  r   r   rl   r<   r   r  )r  r  r  r  r  r  validate_bucketizerp  rs   rV  r  r  s   `  ` `    @@ra   r  r  5	  s     //
!!$''
 '9'9&'A'AT 1 8eTTT
 
 
 	
 DGOO!*;%++K$$&&M 
?##%%&&!++
	 
	 
	 
	 
	 
	 
	 
	 
	 
		 	 	 	 	 	 	 	 	0 __Fz	  F NNMrc   r  r  
boundariesc                  t                                                    dk    sJ t          j                            | t
          j                  r*t          j                            t
          j                  s- t          t          j	        j
        d          | |          S                                  |                                 }|                                 |rt          j        nt          j        fd}t#          j        |||                                           }|                                 |S )Nr+   Fr  r  c                h     |           }t          j        |t                    d          }|S r  )rO   r  r  )rJ  r  r  r  r  r  r  s      ra   rp  zbucketize.<locals>.inner_fn	  sA    l5!!-z**
 
 rc   rq  )r   r   rP   rY   r  r  r  r  r  r  r  r  rl   r`  r|   r  ra  r<   r   )	r  r  r  r  rs   rp  rV  r  r  s	    ` `   @@ra   r  r  	  sW    z""$$%%**** 	
E>#;<<
G
N,DEE
 R 55QQQ:%
 
 
 	
 F$$&&L!*;%++K
 
 
 
 
 
 
 
 ~~	  F NNMrc   c                r    t          j        t          j        t          j        j        ||f          \  }}||fS r   )rA  tree_map_onlyr.   r6   r  require_stride1r   rq   r   s      ra   require_denser  	  s6    '
	2?2T6N LD& <rc   c                r    t          j        t          j        t          j        j        ||f          \  }}||fS r   )rA  r  r.   r6   r  require_contiguousr  s      ra   r  r  	  s6    '
	2?5f~ LD& <rc   c                r    t          j        t          j        t          j        j        ||f          \  }}||fS r   )rA  r  r.   r6   r  r   r  s      ra   r   r   	  s8     '
	2?=f~ LD& <rc   c                r    t          j        t          j        t          j        j        ||f          \  }}||fS r   )rA  r  r.   r6   r  r  r  s      ra   r  r  	  s6    '
	2?84. LD& <rc   c                    t           t          j                  r>d                                 D             }t          j                             |          S t           t                    r! fd                                 D             S t           t          t          f          r0 t                     d t                     D                       S  S )Nc                ^    g | ]*}t          |t          j                  r|j        j        n|+S r   ri   r|   SymIntr_   exprr   s     ra   r   z,constrain_to_fake_tensor.<locals>.<listcomp>	  >     
 
 
BC:a66=AFKKA
 
 
rc   c                J    i | ]}|t          |         |                    S r   constrain_to_fake_tensor)r   keyr   fake_args     ra   r  z,constrain_to_fake_tensor.<locals>.<dictcomp>	  s;     
 
 
GJC)#c(HSMBB
 
 
rc   c              3  <   K   | ]\  }}t          ||          V  d S r   r	  )r   r  f_as      ra   r   z+constrain_to_fake_tensor.<locals>.<genexpr>	  sB       
 
19!S$Q,,
 
 
 
 
 
rc   )ri   r.   r6   rM  r  require_exact_stridesr"  r.  r   rg   r   r  )r   r  meta_stride_exprs   `` ra   r
  r
  	  s   #ry!! L
 
GOGXGX
 
 
 44S:JKKK#t 

 
 
 
 
NQhhjj
 
 
 	
 
C%	'	' 
tCyy 
 
=@h=O=O
 
 
 
 
 	
 Jrc   c                    t          d t          | |          D                       } fd|                                D             }| |fS )Nc              3  <   K   | ]\  }}t          ||          V  d S r   r	  )r   r   r  s      ra   r   z,constrain_to_fake_tensors.<locals>.<genexpr>
  sD        C 	!h//     rc   c                D    i | ]\  }}|t          ||                   S r   r	  )r   r   r   fake_kwargss      ra   r  z-constrain_to_fake_tensors.<locals>.<dictcomp>
  s.    XXXAa)![^<<XXXrc   )r   r  r  )rq   r   	fake_argsr  s      `ra   r   r   
  sf       y11    D YXXXXXXF<rc   c                     fdt          fdt          | j                  D                       } fd|                                D             }||fS )Nc                z    t           t          j                  rft          j        j        d                                         t          j        j        j	                  }t          j
                             |          S t           t                    r" fd                                 D             S  S )Nr  c                B    i | ]}| |         |                   S r   r   )r   r  apply_constraintr   fx_args     ra   r  zEconstrain_to_fx_strides.<locals>.apply_constraint.<locals>.<dictcomp>
  s1    WWWSC))#c(F3K@@WWWrc   )ri   r.   r6   r  rv  rM  rP   rY   r   r  r  r  r"  r.  )r   r  r  r  s   `` ra   r  z1constrain_to_fx_strides.<locals>.apply_constraint
  s    c29%% 	K.E"))++QW-=-G L ?77\JJJc4   	XWWWWWWCHHJJWWWW
rc   c              3  6   K   | ]\  }} ||          V  d S r   r   )r   r   r  r  s      ra   r   z*constrain_to_fx_strides.<locals>.<genexpr>
  sF        *5#vf%%     rc   c                F    i | ]\  }}| |j         |                   S r   r   r   r   r   r  fx_nodes      ra   r  z+constrain_to_fx_strides.<locals>.<dictcomp>
  s4    SSSDAqa!!!W^A%677SSSrc   )r   r  rq   r  r  rq   r   r  s   `  @ra   r   r   
  s             9<T7<9P9P    D TSSSSFLLNNSSSF<rc   c           	           fdt          fdt          t          | j                            D                       } fd|                                D             }||fS )Nc           
     `
   t          |t          j                  s|S |j        d         }d |                                D             }t          j        |                                          }|rT|d         dk    rHt          t          t          t          |
                                                                        }j        t          j        j        k    r| dv rt          |          dk    sJ d}|j        s t          j                            ||          S dj        t$          j        j        j        j        k    o| d	k    }t          |t*                    sJ t          |
                                          d
vr|S t          j        |          }|r2t          j        t          j                            |          |          S t          |t                    rH|                                4|r2t          j        t          j                            |          |          S |rt          |
                                          }g }	|                                }
t          t          |
                                          dz
            D ]o}t4          j        j                            ||         d          s-|
@t4          j        j                            |
|         d          r|	                    |           pdgt          |          z  }d|d<   d}t          t          |          dz
  dd          D ]}||dz            dk    r|||dz            z  }||	v r7t4          j        j                            ||dz            z  d          rd||<   Zt4          j        j                            |z  d          st?          |          z  }|||<   t          j                             ||          S |r2t          j        t          j                            |          |          S t          |t                    rH|                                4|r2t          j        t          j                            |          |          S fd}t          |j!        t          j"                  rZ ||          sO ||#                                          r2t          j        t          j                            |          |          S t          j                            ||          S )Nr  c                ^    g | ]*}t          |t          j                  r|j        j        n|+S r   r  r   s     ra   r   z=sdpa_constraint.<locals>.apply_constraint.<locals>.<listcomp>&
  r  rc   r]  r   )r   r   r   )r   r+   r)   r   r   r   r   r   r+   r)   c                    t           j        j                            |                                 d                   z  dk    S r  )rP   rY   r   	size_hintr   )r   	ALIGNMENTs    ra   
is_alignedz=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned
  s4    G$..qzz||B/?@@9LQRRRrc   )$ri   r.   r6   rv  rM  r  rg   r9  r  r   r   r]   r  0_scaled_dot_product_efficient_attention_backwardr  is_cudar  r  r|   rO   '_scaled_dot_product_efficient_attentionr@   is_aligned_realized_tensor_hinttry_match_insignificant_stridesrealize_inputmaybe_get_striderP   rY   r   statically_known_equalsrm   rC   r  rk   r2   rI  )r"  r   r  meta_valr  r  effn_attn_fwd_biasis_aligned_tensorr1  expanded_dimsmaybe_striderp   out_stridesrM  r(  r'  r  s                  @ra   r  z)sdpa_constraint.<locals>.apply_constraint!
  s7   #ry)) 	J;u%
 
GOGXGX
 
 
 *8??+<+<== 	FL,11s3<<>>/B/B)C)C D DEEL NDLM Mv|$$))))
 (L 	K?77\JJJ 	 Ny~EMN q 	 #y)))))s||~~f,,J>sINN 	5--c224D  
 sF##	$$&&2! 3 5--c224D    $	KCLLNN++HM //11L3s||~~..233 , ,7#;;<LQ<OQRSS , ,(@@aRSTT - "((+++ $X.KKOF3x==1,b"55 ( (q1u%**#hq1uo5F %%w'??#AE*Y6  ! *+A w'??@RTUVV D$VY77)CF!'A?88kJJJ 	5--c224D  
 sF##	$$&&2! 3 5--c224D  	S 	S 	S 	S 	S ch,, 	:c?? :coo//00 =55c::<L   33CFFFrc   c              3  >   K   | ]\  }\  }} |||          V  d S r   r   )r   r"  r   r  r  s       ra   r   z"sdpa_constraint.<locals>.<genexpr>
  sN        C#v 	c6**     rc   c           	     H    i | ]\  }}| d |j         |                   S r]  r  r  s      ra   r  z#sdpa_constraint.<locals>.<dictcomp>
  s6    WWW1a!!"a):;;WWWrc   )r   rh   r  rq   r  r   s   `  @ra   sdpa_constraintr:  
  s    zG zG zG zG zGx     "+Cgl,C,C"D"D    D XWWWWWWWF<rc   )rI  )rz  c                
   |}|                                  |                                 k    r"t          ||                                            }|                                 |                                k    r"t          ||                                           }|                                 |                                k    r1t          ||                                           }t          |          S t          |          S r   )rl   r  r   r  r   r>  r  )r  r  r  r   rn   s        ra   r  r  z  s    ACNN,,,,a**++~~3==??**Q(())}}#,,..((Q((Szz88Orc   )memory_formatc          	         t          j        |                                 |                                 |                                 t          |                                                     S Nrq  )r<   r   rl   r   r`  rg   r   )r   r<  s     ra   r  r    sR     ||~~kkmmAJJLL!!	   rc   c                   g }t          | t                    rt          | j        t          j                  rx| j        } t          | t          j                  rH|                    |                                            | j        } t          | t          j                  Ht          |           } t          |           } |r:| j        } |d d d         D ]}t          j        | |          } t          |           } | S )Nr]  rG  )ri   r@   rk   r.   r   rm   r  r  )r   reinterpret_view_layoutsrH  s      ra   clone_preserve_reinterpret_viewrA    s    !!Y Jqvr7I$J$J FB.// 	$++ALLNN;;;A B.// 	 aLLaA F.ttt4 	: 	:F"&999AAaLLHrc   rc  c               b    fd}t          j        t          |          || g          S )Nc                H    t          j        | d         z  z             S )Nr   r   rO   r`  )rJ  r   r>  r@  s    ra   rv   ziota.<locals>.fn  s%    ~dU1Xo5UCCCCrc   rq  )r<   r   rD   )lengthr>  r@  r   rs   requires_gradrv   s    ```   ra   iotarG    sa    D D D D D D D V$$x	   rc   r   rJ  c           	     f   |                                  |                                 k    sJ |                                 t          | d          t          j        j                            t          j        d                    r| 	                                         z   t          j        j        
                    d           t          j        j                            | 	                                                    t          t          |          | 	                                          }|                                fd}t          j        |                                 |                                  |t#          | 	                                                    S )Nr   c           	         t          j        t          j        t          j        |          t          j                  t          j        t          j                             |            |                     S r   )rO   r  eqr`  r|   r  )r"  r   rJ  
src_loaderr.  s    ra   rp  z select_scatter.<locals>.inner_fn  sd    yFs3x55uek22  JsOOHSMM
 
 	
rc   rq  )r   r`  r<  rP   rY   r   r  r   r   r   r+  r,  r>  r  r<   r   rl   rg   )r   r  r   rJ  rp  rK  r.  s     `` @@ra   select_scatterrL    sf   ;;==CMMOO++++}}H
3
"
"Cw%%ehua&8&899 *

S))Gq%(((GeQZZ\\#%6777
3$$ajjll
3
3C""J
 
 
 
 
 
 
 
 ||~~kkmmAJJLL!!	   rc   c           	        	
 t          |                                           }                                 
t           d                                                    t
          j                                       \  t                                                     }t          z
  dz
  z             |<   t          ||          }|                                		 
fd}t          j                                                                          |t                                                               S )Nr   r+   c                2   	dk    rk    r
dk    r |           S t          j        |          t          j                  }t	          |           t          |          	z
  
          <   g }	dk    rW|                    t          j        |t          j        t          j	        	          t          j                                       k    rW|                    t          j
        |t          j        t          j	                  t          j                                       
dk    rz|                    t          j        t          j        t          |          	z
  d
          t          j                  t          j        dt          j                                       |sJ t          j        t           j        |          }t          j        |fdt%                    rdnd          }t          j        || |                     S )Nr   r+   c                                 S r   r   )src_idxrK  s   ra   r^  z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>  s    JJw'' rc   r_  )rO   r`  r|   ra  rg   r&   rm   rc  r   r>  rd  rJ  r(   rb  r   r  re  rf  r   r  )r"  rg  rl  src_valrP  r   r/  r?  rK  r>  r@  r   r.  s       @ra   rp  zslice_scatter.<locals>.inner_fn  s   A::#//daii:c??".S5;77s))C5 0$77A::KKN5<#6#6DD    (??KKN5<#4#4ekBB    199KKN'C5(8!TBBEK  LEK00	    t$//*''''' ##,AA
 

 yHSMM
 
 	
rc   rq  )r  r   r`  r<  r   r.   r=  normalize_start_endrg   r&   r>  r<   r   rl   )r   r  r   r>  r?  r@  src_sizerp  r/  rK  r.  s   ` ````  @@@ra   slice_scatterrT    sH   
3
&
&C}}H
3
"
"Czz||C H11!S%EEJE3AJJLL!!HS5[D1H5t<<HSM
h

C""J,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
 ,
\ ||~~kkmmAJJLL!!	   rc   c                    t          | t          t          f          r(t          |           dk    rt	          | d                   S | S r  )ri   rg   r   r   _unwrapr   s    ra   rV  rV  "  s;    !dE]## A

qt}}Hrc   r   rs   rH  
pin_memoryc               :    t          |d t          j        fv d|            t          | d           t          t	                     t
                    rpt          j        npt          j                    g }t           t          j	                  r fd}nt           t          t
          f          r fd}nt                     dk    s5t           d         t          t
          f          rNt                     dk    r;|                    t          j        t                                           fd}n4t          j                            t          j         |                    S t%          j        t)          |          ||	          S )
Nlayout=rX  c                .    t          j                  S r   rD  rJ  rk   r   s    ra   rp  ztensor.<locals>.inner_fn5  s    >$...rc   c                .    t          j                  S r   rO   rb  r\  s    ra   rp  ztensor.<locals>.inner_fn:  s    <e,,,rc   r   r   c                      fdt                    dk    rt          j        d          S  dt                              S )Nc           	     l   | |k     sJ || z
  dk    rt          j        |                    S || z
  dz  | z   }t          j        t          j        t          j        d         t
          j                  t          j        |t
          j                             | |           ||                    S )Nr+   r)   r   )rO   rb  r  rd  r`  r|   ra  )r>  r?  midbinary_searchrk   r   rJ  s      ra   rb  z/tensor.<locals>.inner_fn.<locals>.binary_searchB  s    s{{{{;!##<UU;;;U{q(50yFuQx==S%+66  "M%--!M#s++  rc   r   )r   rO   rb  )rJ  rb  rk   r   s   `@ra   rp  ztensor.<locals>.inner_fnA  sh            4yyA~~|Au--- =CII...rc   r'  rq  )r   r|   stridedri   rV  r   ra  get_default_dtyper   r   rF  r   rm   IntegerrP   rY   add_tensor_constantr  r<   r   rD   )rk   r   rs   rH  rX  ro  rp  s   ``     ra   r  r  (  s   v$..0B&0B0BCCC:~|,,,'$--%% 3$2022!F$$$ $
	/ 	/ 	/ 	/ 	/ 	/ 	/ 
D5#,	'	' 
	- 	- 	- 	- 	- 	- 	- 
Ta:d1gs|<<TaemCII..///	/ 	/ 	/ 	/ 	/ 	/ 	/( w**LU6:::
 
 	
 V$$	   rc   c                    t          | t                    r&|t          | |          } |t          | |          } | S t	          | ||          S )Nr'  )ri   r@   r  r  r  )rk   r   rs   s      ra   	as_tensorrh  a  sX    $	"" D%((DT6**D$eF3333rc   c                8    t          | t          j                  S r|  r~  )rk   s    ra   long_tensorrj  l  s    $ek****rc   c                   t          t          j        j        j        t          j        j        j        d                   }|J t          |          dk    s
J |            t          t          |
                                                    \  }}t          j        |||           }t          j                            |          |_        t          j                            |           t          j        j        j        d         }t!          |t"          j        t"          j        t"          j        f          r|j        j        S t/          j        |          S )Nr  r+   r  )r#   rP   rY   r   r  rZ   rv  r   rP  r  r  r.   DynamicScalarr  r   r  ri   r|   r  SymFloatSymBoolr_   r  r   r8  )rk   r  binding_symkeypathr  r  s         ra   _local_scalar_denserq  q  s    2	"AG$8$=>Q$R  ((( !!Q&&&(9&&&   %6%<%<%>%> ? ?@@Kk7D99F'))&11FKGv&&& '

#E
*C#enemDEE "x}}S!!!rc   c                    d S r   r   )rk   r   s     ra   _assert_scalarrs    s	     4rc   )rs   rH  c                   d S r   r   )r  r   rM  r   rs   rH  s         ra   _assert_tensor_metadataru    s	     4rc   c                   | t          | t          t          f          st          d          rj        t          t          t          f          rfd}nat          t
          j                  rfd}n@t                                                    dk    sJ 	                                fd}t          j        ||t          |                    S )Nr  c                .    t          j                  S r   r^  rJ  r   r  s    ra   rp  z_full.<locals>.inner_fn  s    <u---rc   c                .    t          j                  S r   rD  rx  s    ra   rp  z_full.<locals>.inner_fn  s    >%///rc   r   c                     g           S r   r   )rJ  value_loaders    ra   rp  z_full.<locals>.inner_fn  s    <###rc   rq  )ri   r   rF  r   r  r   r   r   r   r`  r<   r   rg   )
fill_valuers   r   r   rp  r  r{  s     `  @@ra   _fullr}    s'   Ej3,// GE74K4K %#u&& $	. 	. 	. 	. 	. 	. 	. 
E5;	'	' 
$	0 	0 	0 	0 	0 	0 	0 5>>##$$))))((**	$ 	$ 	$ 	$ 	$ Dzz	   rc   c                H     t          t          |                    | fi |S r   create_tensor_liketensor_constructor)r   r|  r   s      ra   r
  r
    s,    =0<<==aJJ6JJJrc   c                "     d d d d dd d fd
}|S )NF)namesr   rs   rH  rX  r<  c                   t          | d u d           t          |d t          j        fv d|            t          | d           t          |          }|pt          j                    }t          |          dk    rBt          |d         t          t          t          j	        f          rt          |d                   }|D ]}t          |t          j
                  rJ d |D             }t          |||          S )Nnamed tensorsrZ  rX  r+   r   c                6    g | ]}t          j        |          S r   rD  r   s     ra   r   z5tensor_constructor.<locals>.inner.<locals>.<listcomp>  s     ...AQ...rc   )r   r|   rc  rD   rd  r   ri   rg   r   Sizer  r}  )	r  r   rs   rH  rX  r<  r   r   r|  s	           ra   r|  z!tensor_constructor.<locals>.inner  s     	5D=/2226dEM224Ff4F4FGGGz><000v&&2022t99>>ja4
2KLL>a>>D  	3 	3A!!U\222222.....Z555rc   r   )r|  r|  s   ` ra   r  r    sA     6 6 6 6 6 6 6. Lrc   )r  r   rH  rs   rX  r<  c                   t          | d u d           t          |          }t          |          dk    rBt          |d         t          t
          t          j        f          rt          |d                   }t          |d ||||          S )Nr  r+   r   r   rH  rs   rX  )	r   rD   r   ri   rg   r   r|   r  empty_strided)r  r   rH  rs   rX  r<  r   s          ra   r/  r/    s     u}o...6""F
4yyA~~*T!WtUEJ.GHH~T!W~~d%v*   rc   c                      dddddd fd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    NF)r   rs   rH  rX  r<  c               B   t          | d           t          |d t          j        fv d|            ||                                 }nt	          |          }|p|                                 }t          |                                           } |||||          S )NrX  rZ  rW  )r   r|   rc  r   r   rl   rg   r   )r   r   rs   rH  rX  r<  r   creation_fns          ra   _constant_likez*create_tensor_like.<locals>._constant_like  s     	z><0006dEM224Ff4F4FGGG=KKMMEE ''E)1<<>>AJJLL!!{fV

 
 
 	
rc   r   )r  r  s   ` ra   r  r     s<     duTX
 
 
 
 
 
 
 rc   c                :    t          t          |                     S r   r  )r|  s    ra   r  r    s    0<<===rc   c                     d d d d d fd
}|S )Nr  c               l   t          |t          t          f          sJ t          | d           t          |d t          j        fv d|            t          |          p|                                 }|p|                                 }d |D             }t          t          |          ||          S )NrX  rZ  c                6    g | ]}t          j        |          S r   )r   re  r   s     ra   r   z7new_constant.<locals>._new_constant.<locals>.<listcomp>)  s"    ///Qa  ///rc   )ri   rg   r   r   r|   rc  r   r   rl   r}  rD   )r   r   r   rH  rs   rX  r|  s         ra   _new_constantz#new_constant.<locals>._new_constant!  s     $u.....z><0006dEM224Ff4F4FGGGU##4q{{}})1<<>>//$///Zv!6!6tDDDrc   r   )r|  r  s   ` ra   new_constantr     s>    tDT	E 	E 	E 	E 	E 	E 	E rc   r  c          	         ||                                  }||                                 }t          |d ||t          |          |          S Nr  r   rl   r  rD   )r   r   r   rH  rs   rX  s         ra   	new_emptyr  /  sW    }~V$$   rc   c               B   t          | t          t          f          sJ t          |t          t          t          d           f          sJ t	          | d           t	          |d t
          j        fv d|            t          |          pt          j                    }|pt          j	        d          j
        }t          |          }t          d|||           }|                                 |j        j        }t          j        |j        dgt#          |           z            |_        t          |t$          j                  sJ d | D             } |rd |D             nt$          j                            |           }t%          j        ||| |	          |_        |S )
NrX  rZ  r_  r   )r|  rs   r   r   )ro  c                6    g | ]}t          j        |          S r   rD  r   s     ra   r   z!empty_strided.<locals>.<listcomp>P  s     ***ELOO***rc   c                6    g | ]}t          j        |          S r   rD  r   s     ra   r   z!empty_strided.<locals>.<listcomp>R  rF  rc   )rs   r   r   rM  )ri   rg   r   r   r   r|   rc  r   rd  r  rs   rD   r}  r  rk   dataclassesreplacer   r.   r  r  r  rL  rH  )r   rM  r   rH  rs   rX  	pointwiser  s           ra   r  r  ?  s    dT5M*****ftUDJJ788888:~|,,,v$..0B&0B0BCCC<5#:#<#<E/u|C((/F6""F6TJJJI^ F%fk1#D		/JJJFKfb/00000**T***D 	8))&))))11$77 
 N	  FM rc   c          	         ||                                  }||                                 }t          ||||t          |          |          S r  r  )r   r   rM  r   rH  rs   rX  s          ra   new_empty_stridedr  _  sY     }~V$$   rc   c                    d |D             }t          t          t          |                    |j                  }t          j                            | |          S )Nc                V    g | ]&}t           j        j                            |          'S r   r   r   s     ra   r   z copy_strided.<locals>.<listcomp>s  s+    EEEag11!44EEErc   )r  )sortedr  r   __getitem__r.   r  r  )r   rM  r  s      ra   copy_stridedr  q  sP    EEfEEEF%F,,&2DEEEL?//<@@@rc   c                l    |                     d          
J d             t          |          | fi |S )Nr   z(dtype should be handled by decomposition)rw  r  )r   r|  r   s      ra   fullr  x  sD    ::g**,V***)j))$99&999rc   c                D   t          | t                    sJ |                                dk    r"t          | |                                          S |                                 t                    dk    }t          | |          |rt          | dg          } dg|                                 |                                fd}t          j
        |                                 |                                 ||                                          S )Nr   r+   c                    t          |           } t          j         |                              }t          |           dk    r|g} n|| <    |           S r  )rg   rO   indirect_indexingr   )r"  
gather_idxr   index_loaderr   r.  s     ra   rv   zgather.<locals>.fn  s]    3ii*<<+<+<d3iHH
s88q==,CC!CHx}}rc   rq  )ri   r@   	get_numelr  r   r   r<  r>  r`  r<   r   rl   r   )	r   r   rJ  sparse_gradr  rv   r  r   r.  s	    `    @@@ra   gatherr  ~  s     a#####AENN,,---::<<DYY!^F
3
'
'C 1qcNNs}}H$$&&L        ||~~kkmm~~	   rc   c                  	
 |r+ t          t          j        j                  | ||||          S |rJ t	          | t
                    sJ t	          |t
                    sJ dt          |                                          v sJ |                                 	|                                t          |
                                          | 
                                
g |
                                
dd          	
fd}t          j        |                                 |                                 |          S )Nr   r+   c                    t          |           t                    k    sJ |  d              | d                    }t          j        |d                   gg | d          z   } |          S )Nz != r   )r   rO   r  )r"  	var_index
weight_idxindices_loaderindices_ndimr0  weight_loaderweight_sizes      ra   rv   zembedding.<locals>.fn  s    3xx3x==(((S*@*@h*@*@((("N3}}#566	+I{1~FFG K
K
 

 }Z(((rc   rq  )r  r  	embeddingr  ri   r@   r  r   r`  r   r   r<   r   rl   )weightr  padding_idxscale_grad_by_freqsparserv   r  r  r0  r  r  s         @@@@@ra   r  r    su    
7 677G[*<f
 
 	
 :fi(((((gy)))))C))++,,,,,,&&((M((**Nw''))**L//##K6!!##6k!""o6H) ) ) ) ) ) ) ) )   ""  	   rc   c                    t          d  D                       sJ dd  D                          t          d  D                       rt          d          d t                     D             }t	          |          dk    s
J d            d gt	                     z  }t          |t           fd	|D                        D ]1\  }}|                                |k    rt          d
          |||<   2||fS )Nc              3     K   | ]H}||                                 t          j        t          j        t          j        t          j        fv V  Id S r   )r   r|   ra  r  r   r  r   rp   s     ra   r   z.check_and_broadcast_indices.<locals>.<genexpr>  sN        = 	
%+u{EJLL=== rc   z)indices must be int64, byte or bool. Got c                :    g | ]}||                                 S r   r   r  s     ra   r   z/check_and_broadcast_indices.<locals>.<listcomp>  s&    4e4e4eqWXWdQ[[]]WdWdWdrc   c              3  r   K   | ]2}||                                 t          j        t          j        fv V  3d S r   )r   r|   r   r  r  s     ra   r   z.check_and_broadcast_indices.<locals>.<genexpr>  sA        78q}%*ek22}}}} rc   zFallback for bool indicesc                B    g | ]\  }}t          |t                    |S r   r   r   s      ra   r   z/check_and_broadcast_indices.<locals>.<listcomp>  s+    OOO1jI6N6NO!OOOrc   r   z"requires at least 1 non-None indexc                     g | ]
}|         S r   r   )r   rp   r  s     ra   r   z/check_and_broadcast_indices.<locals>.<listcomp>  s    3S3S3S1GAJ3S3S3Src   z.Fallback when indices is on a different device)r,  r-  r   rh   r   r  r  rl   )r  rs   
valid_idxsnew_indicesrp   r   s   `     ra   check_and_broadcast_indicesr    sb           
 	h4e4eG4e4e4egg     <C     ? ""=>>>OO	' 2 2OOOJz??Q D&3w<<'KJ 13S3S3S3S
3S3S3S TUU  1 <<>>V##%&VWWWA
""rc   c	           
     n    dt          dd                    D ]\  }	}
|
|	z
  dk    rd fdt                    D             g  t                    t                    z   d          d         }rz   nd |         z   |d          z   f
d}|fS )NFr+   Tc                *    g | ]\  }}||         S r   r   )r   rp   r  r   s      ra   r   z2index_output_size_and_inner_fn.<locals>.<listcomp>  s"    MMMC6!9rc   r   c                v  
 t          |           t                    k    sJ t                    t          
          k    sJ t                    }g }d         }rdn|}d}t          d         dz             D ]}||k    r||z  }|         6|t          |           k     sJ |                    | |                    |dz  }K|         }|J 
|         }|                    t          j         || |||z                      |	                     g || |d          }|n
 |          S )Nr   r]  r+   r   wrap_neg)r   r  rm   rO   r  )r"  r  	new_indexfirst_tensor_indexstart_offsetnext_idxrp   loaderr   r   indexed_sizer  indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizer  r.  s            ra   rv   z*index_output_size_and_inner_fn.<locals>.fn  s   3xx3{++++++?##s<'8'88888;	+A.3Kqq9K~b)A-.. 	 	AL  D qz!#c((****  X///A(+)))#A  )s<,2E#EFGG#!)	     

^
	 %,yy((92E2EErc   )r  rh   r   )r   r  r  r  r  r  r.  r   r  previouscurrentr  rv   r  r  s   `````````    @@ra   index_output_size_and_inner_fnr    sU   $ $ 1CDD + +'X""&*#MMMM9W+=+=MMMKSKS&[)9)9C<O<O)O)Q)Q"RSK'* 
!K/ +++,,--./ 	 F  F  F  F  F  F  F  F  F  F  F  F  F  FD ?rc   c                    t          | ||          \  }}}t          j        |                                 |                                 ||          S r>  )index_impl_helperr<   r   rl   r   )r   r  r   r  rp  r   s         ra   
index_implr  (  sP    0GUCCK1||~~kkmm	   rc   c                  
 t          t          t          f          sJ |                                 t	          |                                           \  }t          |          dk    s
J d            d D             }t          |d                                                            }|                                 fdt          t                              D             }|rd|v rd|vrt          d          fdt          t                              D             }t          ||||d ||	  	        \  }

fd}	||	
fS )	Nr   z Must have at least one valid idxc                >    g | ]}||                                 nd S r   r_  r  s     ra   r   z%index_impl_helper.<locals>.<listcomp>9  (    SSSa!-q}}TSSSrc   c                0    g | ]}|         
|         S r   r   )r   rp   r  r   s     ra   r   z%index_impl_helper.<locals>.<listcomp>A  s&    UUU!gaj>TF1I>T>T>Trc   z0index is out of bounds for dimension with size 0c                     g | ]
}|         S r   r   r   rp   r   s     ra   r   z%index_impl_helper.<locals>.<listcomp>E      ;;;!F1I;;;rc   r  c                ,      |                     S r   r   )r"  index_inner_fnr.  s    ra   rp  z#index_impl_helper.<locals>.inner_fnR  s    xs++,,,rc   )ri   rg   r   r`  r  rl   r   r   r  
IndexErrorr  )r   r  r   r  r  r  r  r  r  rp  r  r.  r   s    `        @@@ra   r  r  3  s   ge}-----}}H9'1<<>>RRG^~"""$F"""SS7SSSO w~a01::<<==KZZ\\FUUUUUuS\\':':UUUL Ml""q';';KLLL;;;;uS\\':':;;;L"@
# 
# 
#K- - - - - - .00rc   c                    	 t          | |d          S # t          $ rA |                                   t          t          j        j        d          | |          cY S w xY w)NTr   Fr  )r  r   r  r  r  rJ  r  r   r  s     ra   rJ  rJ  X  sz    
!WD1111 
 
 
			M
 1uMMMw
 
 	
 	
 	

s    AAAc                &    t          | |d          S )NFr  )r  r  s     ra   _unsafe_indexr  d  s    a....rc   c                F    t          t          |           |||dd          S )NTFr   may_realizeindex_put_impl_r  r   r  r  
accumulates       ra   	index_putr  q  s,    a'6:Tu   rc   c                F    t          t          |           |||dd          S )NFr  r  r  s       ra   _unsafe_index_putr  x  s,    a'6:U   rc   c                   |                                 |                                  k    r"t          ||                                            }|rt          | |          }t          | t	          |d         ||                     S r  )rl   r  r   r
  r  )r  r  r  r  s       ra   index_put_as_masked_fillr    ss    T__....%!2!233 !D%  T5UD99:::rc   c                    t          t          j        j        j        t
          j        j                  sJ t          j	        t          j        j        j        | |||           | S r   )
ri   rP   rY   rZ   r]   r|   r}   r~   r.   IndexPutFallbackr  r  r  r  s       ra   index_put_fallbackr    sK    ag*15:3HIIIII,3T7FJWWWKrc   c                ,    t          | |||dd          S )NTr  r  r  s       ra   
index_put_r    s&    gvz4   rc   c                ,    t          | |||dd          S )NFTr  r  r  s       ra   _unsafe_index_put_r    s&    gvzD   rc   c           
     v   |rTd }d  ||           |                                 v r/t          fd|D                       s|                                 |                                dk    rt	          |          dk    r|d                                         t          j        t          j        fv r||d         }t          t	          |
                                          t	          | 
                                                    D ]}t          |d          }t          | |g||          S t          j                    rt          | |||          S |D ]D}	|	@|	                                t          j        t          j        fv rt          | |||          c S E| 
                                t	                    }
|rbt          |                                           rA|
dk    rt!          | dg          } t          | |||          } |
dk    rt!          | g           } | S t#          ||                                           }	 t%          ||                                           \  }}n"# t(          $ r t          | |||          cY S w xY wd |D             }t+          | t,                    sJ |                                  |
dk    rt!          | dg          } t/          ||d                  
                                          }fdt          t	          |                    D             }t1          |||||d |	          \  }}t3          ||          }|                                 }|J t5          j        ||                                 |                                |||rd
nd           }t5          j        d t5          j        |           |          }t>          j         !                    |          |_"        t>          j         #                    |           |
dk    rt!          | g           } | S )Nc                B   t          | t          j                  r| j        } t          | t          j                  r|                                 } t          | t          j                  r| j        } t          | t          j                  r|                                 nd S r   )	ri   r.   r@   rk   r2   rI  r   Bufferr  r   s    ra   try_get_namez%index_put_impl_.<locals>.try_get_name  s|    !R\** F!R[)) $MMOO!R]++ F#-a#;#;E1::<<<Erc   c                   t          | t                    rt          | j        t          j                  r| j                                        } t          | t          j                  obt          | j        t          j                  oCt          | j        dd           o-| j        j	        j
        t          j        j        j        j        k    S dS )Nr  F)ri   r@   rk   r.   r2   rI  r   r  r   r  r]   r|   rO   r  randpermr  )indices    ra   indice_slice_from_randpermz3index_put_impl_.<locals>.indice_slice_from_randperm  s     &),, FK1U1U 0022vr}55 V"6;@@VY==V +2ein6M6UU	 5rc   c              3  .   K   | ]} |          V  d S r   r   )r   r  r	  s     ra   r   z"index_put_impl_.<locals>.<genexpr>  sH       E
 E
39&&v..E
 E
 E
 E
 E
 E
rc   r+   r   r]  c                >    g | ]}||                                 nd S r   r_  r  s     ra   r   z#index_put_impl_.<locals>.<listcomp>  r  rc   c                     g | ]
}|         S r   r   r  s     ra   r   z#index_put_impl_.<locals>.<listcomp>  r  rc   r  
atomic_addrs   r   rp  ro  output_indexerscatter_moder   rH  rk   )$r  r,  r  r  r   r   r|   r   r  r  r   r  r  $are_deterministic_algorithms_enabledr  rI   r  r  r  rl   r   ri   r@   rg   r  r>  r.   Scatterr`  r  MutationLayoutSHOULDREMOVErP   rY   r  r   r  )r  r  r  r  r   r  r  rl  r   rJ  x_ndimr  r  r  r  expected_vals_sizerp  rs   scatterr  r	  r   s                       @@ra   r  r    s    !	F 	F 	F	 	 	 <!6!6!8!888 E
 E
 E
 E
=DE
 E
 E
 B
 B
8 NN 	aLLAAJ  ""uz5;&???qzs4==??++S-A-ABB 	' 	'AT2&&DD'tffjIII 133 E!$DDD  I I!2!2uz5;6O!O!O%dGVZHHHHH]]__F[[F B4>>CSCSTT Q;;qc??D!$DDQ;;b>>Dfdnn..//FE"=T__&&#
 #
  E E E!$DDDDDE TS7SSSOdI&&&&&LLNNN {{D1# w~a01::<<==K;;;;uS\\':':;;;L#A	$ 	$ 	$  F.//F __Fjnn##%%!%/9\\T  G ,T22  F
 '))&11FKGv&&&{{D"~~Ks   %I. .JJr  c                  	 t          | |dd          \  }}                                |                                 		fd}t          j        |                                 |                                 ||          S )NFr  c                     j         t          j        k    r)t          j                    t          j                  }n            }t          j        | fd          S )Nc                 ,                           S r   r   )_unsafe_index_fnr"  self_loaders   ra   r^  z8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>1  s    KK8H8H8M8M,N,N rc   )r   r|   r   rO   r  rf  )r"  mask_valr  fillrl  mask_loaderr  s   ` ra   rp  z&_unsafe_masked_index.<locals>.inner_fn,  sf    :##|KK$4$4ejAAHH"{3''Hz($N$N$N$N$N$NPTUUUrc   rq  )r  r`  r<   r   rl   r   )
r  rl  r  r  ro  r   rp  r  r  r  s
    ` `   @@@ra   _unsafe_masked_indexr   $  s    "3gUU# # #FA ""$$K""$$KV V V V V V V V V   nn	   rc   c                    t          ||d          }|                                 fdt          t                              D             }t	          | ||d          S )Nr   c                r    g | ]3}|         r't          |         |          |         d z
            nd4S r)  r:  )r   rp   r  r  s     ra   r   z7_unsafe_masked_index_put_accumulate.<locals>.<listcomp>?  sU        7>ajJgaj58)U1X\222d  rc   T)r  )r  r   r  r   r  )r   rl  r  r  masked_valueclamped_indicesr  s     `   @ra   #_unsafe_masked_index_put_accumulater%  ;  sv    vq))LJJLLE    s7||$$  O Q$OOOOrc   c                R    t          j        |t          j        ||                     S r   )rO   r  r  r  minmaxs      ra   r;  r;  H  s     ;sCKQ//000rc   c                h    t          |           }t          ||||          }t          ||           |S r   )r  rR  copy_)r  r  r   rM  rN  rA  output_views          ra   as_strided_scatterr-  M  s5    4[[FVT6>BBK	+sMrc   c                <    t          t          |           |||fi |S r   )scatter_r  )r   r   rJ  r  r   s        ra   r  r  U  s$    E!HHc5#88888rc   r  include_selfrk  r  r  r1  c               b   t          |t                    }t          | ||                                t	          t
          j        |r|                                nt          |                    |r|                                j        nd|          rt          j
        | ||||||           |S d S )Nznot implr0  )ri   r@   rN   r   r	   r|   r   r   rl   r.   ScatterFallback)rk  r  r   rJ  r  r  r1  src_is_tensors           ra   scatter_fallbackr5  Z  s     sI..MU[]I#--///S		JJ!.>J   	%	
 	
 	
 	
 4rc   r  c                   |dv sJ |Lt          t          j        t          j        j        j        j                  }t          || ||||          }||S |dk    rd}n|dk    rd}t          | ||||          S )N)Nr   multiplyr6  r   sumr8  prod)
r   r  r/  rP   rY   rZ   r]   _overloadnamer5  scatter_reduce_)r  r   rJ  r  r  rk  fallback_results          ra   r/  r/  {  s    .....~dmQW-A-H-VWW*sE3v
 
 
 &""	:		4eS&999rc   c                @    t          t          |           |||          S r   )scatter_add_r  r   r   rJ  r  s       ra   scatter_addrA    s    a#uc222rc   c                (    t          | |||d          S )Nr9  )r<  r@  s       ra   r?  r?    s    1c5#u555rc   c                >    t          t          |           ||||fi |S r   )r<  r  )r   r   rJ  r  reduction_typer   s         ra   scatter_reducerE    s&    588S%nOOOOOrc   )r1  c          
         |dv sJ t          t          j                                                  dk    r dt          j                                        v s
J d            t	          t
                    rt                     t          t          j        j         |||          }|r|S t	           t                    sJ dt          |                                          v sJ t                                                     }|dk    rt           dg           t	          t                    r6t                                                    dk    rt          dg          t	          |t                    r6t          |                                          dk    rt          |dg          }|                                dk    r S t                                                       |                                t	          t                    r                                nd  fd} fd	}	d
 }
                                 }|J |st'          j        |                                  fd|                                |d           }t'          j        d t'          j                   |          }t.          j                            |          |_        t.          j                            |           t'          j        |                                 |	|                                | |
|                    }t'          j        d t'          j                   |          }t.          j                            |          |_        t.          j                            |           |dk    rt           g             S )N)Nr9  r:  meanamaxaminr+   twozKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r0  r   r   c                                                     }t          |          }t          |           }t          j         |           |dk    rdn|         d          |<   |S )Nr   r+   F)r  )r   r   rg   rO   r  )r"  r  r.  indirect_idxr   r  r  s       ra   r  z'scatter_reduce_.<locals>.output_indexer  sk    5zzCyy1LDAIIqq5:
 
 
S rc   c                l    r |           S t          j                                                  S r   rO   rb  r   )r"  r  r  rK  s    ra   rv   zscatter_reduce_.<locals>.fn  s7     	7:c??" <T^^%5%5666rc   c                    | dk    rdS | J d S )Nr9  r  r   r6  s    ra   backend_reduce_strz+scatter_reduce_.<locals>.backend_reduce_str  s     U??< >>>4rc   c                R    t          j        d                                          S r  rN  )rJ  r  s    ra   r^  z!scatter_reduce_.<locals>.<lambda>  s    3<4>>3C3C#D#D rc   r  r  )r   r  r<  r   ri   r   r
  r5  rJ  r@   r  r   r   r  r  r<  r  r`  rl   r.   r  r  r  rP   rY   r  r   r  )r  r   rJ  r  r  r1  r=  r.  r  rv   rP  rs   zero_outr  r  r  rK  s   `` `           @@ra   r<  r<    s   BBBBBD **,,--22T)3355555T 65	6 #v #c""& !  O  dI&&&&&C))******t}}DqyyD1##y!! c#,,..&9&9Q&>&>3nn%## !ENN,<,<(=(=(B(BUQC  A
c
"
"CLLNNN$$&&L&0i&@&@J"""dJ      7 7 7 7 7 7 7   __F +:..""DDDD>>##)
 
 
 "066
 
 

 g--f55	""6***
 jnn~~%''//  G ,T22  F
 '))&11FKGv&&&qyyD"~~Krc   scales_xtuple[Optional[float], ...]nexactc                &  
 |                                   |                                 |                                  d          
|                                 d           }d 
D             
t          |          k    sJ |}d t	          
|          D             t          |          D ]\  }}|d|z  |<   fd
fd}	t          j        |                                 | 	                                |	g ||          S )Nc                V    g | ]&}t           j        j                            |          'S r   r  r  s     ra   r   z&upsample_nearestnd.<locals>.<listcomp>  s+    >>>qw))!,,>>>rc   c                    g | ]
\  }}||z  S r   r   )r   rp   os      ra   r   z&upsample_nearestnd.<locals>.<listcomp>   s     :::DAq!a%:::rc   r  c                z   t          j        | t          j                  } r2t          j        | t          j        dt          j                            } t          j        | t          j        |t          j                            } t          j        | t          j                  } t          j	        | |d          S )N      ?Fr  )
rO   r`  r|   r~  r   rb  rr  r  r  r  )r   r  r   rV  s      ra   scale_fnz$upsample_nearestnd.<locals>.scale_fn%  s     N1em,, 	=3<U];;<<AGAs|E5=99::LEK(($QE::::rc   c           	         |  d          }| d           } g |fdt          |          D                       S )Nc                2    g | ]\  }}} |||          S r   r   )r   rp   r   r   r]  s       ra   r   z2upsample_nearestnd.<locals>.fn.<locals>.<listcomp>4  s-    VVVJAq$88Aq$''VVVrc   )r  )r"  r   r@  i_sizes
inv_scalesrU  r]  r.  s      ra   rv   zupsample_nearestnd.<locals>.fn0  sc    H!HxWaWVVVV#aW:U:UVVVW
 
 	
rc   rq  )
realize_hintr`  r   r   r  rh   r<   r   rl   r   )r   r  rS  rU  rV  batcho_sizesrp   r  rv   r`  ra  r]  r.  s      ``     @@@@ra   upsample_nearestndre    sZ    NN}}HjjllA233GJJLL1"E>>g>>>Gx==AG::C$9$9:::Jh'' ( (5%KJqM	; 	; 	; 	; 	;
 
 
 
 
 
 
 
 
 ||~~kkmm!!!	   rc   Optional[float]c                *    t          | ||fd          S )Nr+   rU  re  r   r  rq  s      ra   upsample_nearest1drk  ?  s    avi1====rc   c                ,    t          | ||fdd          S )Nr+   TrU  rV  ri  rj  s      ra   _upsample_nearest_exact1drn  D  s    avi1DIIIIrc   scales_hscales_wc                ,    t          | |||fd          S )Nr)   rh  ri  r   r  ro  rp  s       ra   upsample_nearest2drs  I  s      ax.BaHHHHrc   c                .    t          | |||fdd          S )Nr)   Trm  ri  rr  s       ra   _upsample_nearest_exact2dru  P  s"     ax.BatTTTTrc   scales_dc                .    t          | ||||fd          S )Nr   rh  ri  r   r  rv  ro  rp  s        ra   upsample_nearest3dry  W  s#     ax8.LPQRRRRrc   c                0    t          | ||||fdd          S )Nr   Trm  ri  rx  s        ra   _upsample_nearest_exact3dr{  b  s-     	;8X6!4   rc   c                :     t           fd|D                       S )Nc              3  B   K   | ]}t          j        |          V  d S r   r^  )r   r  r   s     ra   r   z$_create_constants.<locals>.<genexpr>p  s/      66Aa''666666rc   )r   )r   rq   s   ` ra   r}  r}  o  s&    6666666666rc   c                    |                                  |                                 fd}t          j        |                                 |                                 |          S )Nc                    t          |           } t          |           t                    k    sJ D ]}|         dz
  | |         z
  | |<    |           S r  )rg   r   )r"  r   r  r  r.  s     ra   r  zrev.<locals>.loadery  sd    3ii3xx3u::%%%% 	3 	3Cc
Q#c(2CHHx}}rc   rq  )r`  r   r<   r   rl   r   )r   r  r  r  r.  s    ` @@ra   revr  s  s|     }}HJJLLE       ||~~kkmm	   rc   paddingSequence[int]r|  c                j   d } |            rdS t          |          dk    s%t          |                                           dk    rdS |                                  t          | t          j                  rt          | j        t          j                  ret          | j        j        t          j                  s0t          j
        r5t          | j        j        t          j                  r| j        j        j        sdS |                                  t	          j        |           \  }}|j        }|d         dk    rdS |d         dk    s|d         dk    s|d         dk    rdS |d         }|dk    rdS |d         }|j        d         }	||	|z   k     rdS | j        j        j        }
|j        d         |j        d         |z   g}|t"          j        j        |
<   t)          | ||j        |j                  }t-          |d|	|	|z             }t/          ||           t0          d	         d
xx         dz  cc<   |S )z
    This optimization changes the semantics of padding from 'clone'
    style to 'view' style.

    Thanks to functionalization, this change can still maintain numerical
    correctness.
    c                     t           j        j        } | dS t          | j                  }t          |          dk    r1|d         j        t          j        j	        t          j
        j	        fv rdS dS )a  
        Conservatively check if padding can be fused with downstream op.
        1. if the downstream op is a sum, then there is little benefit to
           do inplace padding
        2. if the downstream op is a matmul, doing inplace padding can
           save membw.
        NTr+   r   F)rP   rY   rZ   r   r[   r   r]   r  mmr  addmm)rZ   r[   s     ra   _padding_can_be_fusedz6inplace_constant_pad_nd.<locals>._padding_can_be_fused  sj     w+4l())u::??uQxGOJ3
  
  
 5trc   Nr   r)   r+   r   r   )r   r>  r?  inductorinplace_padding)r   r   r  ri   r.   r@   rk   r   r  r,   can_inplace_pad_graph_inputInputBufferr   freeze_layoutrK  rM  r   rP   rY   buffer_to_padded_sizerR  r  rA  fill_r   )r   r  r|  r  r   rH  r  npadstride0rowsizebufnamepadded_size	resized_xsliced_xs                 ra   inplace_constant_pad_ndr    s?     (  t 7||qC

--22t
 IIKKK
 q",''!&"-00 qv{B$566	 2 qv{BN;; v{ tOO(++IAvmGqzQtqzQ'!*//WQZ1__t1:DqyytajGk!nG4tfkG;q>6;q>D#89K-8AG!'*		 I iQg7T>JJJH	(JZ*+++q0+++rc   c                   t          |          dz  dk    sJ t          d |D                       rt          |           S t          j        rt          | |          }|r|S |                                 }t          t          t          t          |d d d         |dd d                                                 t          |          t                    z
  g D ]>\  }}
                    t          j        j                            |          |f           ?t          |d                    }g t          |d                    D ]J\  \  }}	}

                    |
           |
                    t          j        |
|z   |	z                        Kt          |          t          |          k    sJ  t#          |                                                     fdfd}|                                 t)          j        |                                 |                                 ||          S )Nr)   r   c              3  "   K   | ]
}|d k    V  dS r   Nr   r   r  s     ra   r   z"constant_pad_nd.<locals>.<genexpr>  s&      
#
#a16
#
#
#
#
#
#rc   r+   c                f    g }t           	d                    D ][\  }\  }}}|dk    r#|                    t          |d                     |dk    r#|                    t          ||                     \t	          j        t          j        |          }t          j        | 
fd          S )Nr   c                                 S r   r   )rJ  r.  s   ra   r^  z/constant_pad_nd.<locals>.mask.<locals>.<lambda>  s     rc   )	r  rm   range_mask_lowrange_mask_highr   r  rO   re  rf  )rJ  rl  r"  r  r  rE  boundsr|  
mask_sizesrU  r.  s   `     ra   rl  zconstant_pad_nd.<locals>.mask  s    (+E!""Ivz(J(J 	: 	:$C#tfaxxN322333qyyOC88999$//z$ 7 7 7 7 7DDDrc   c                    t          | d                    }t          | d                    D ] \  }\  }}|                    ||z
             !t          |          t          |           k    sJ  |          S r   )rg   r  rm   r   )rJ  r  r"  r  _highbounds_precomprl  rU  s        ra   	offset_fnz"constant_pad_nd.<locals>.offset_fn  s    rrOO	!$U122Y!?!? 	( 	(C#uS3Y''''9~~U++++tIrc   rq  )r   r,  r  r,   r  r  r   rg   r9  r  rm   rP   rY   r   lookup_precomputed_sizer   r>  r   r   r`  r<   r   rl   )r   r  r|  rn   r  lhr  r  r  r   r  r  r  rl  r  rU  r.  s     `         @@@@@@ra   r  r    s   LL1""""

#
#7
#
#
### Qxx %a*== 	J JJLLE(4GCCaCL'!$Q$- @ @AABBCCFE

S[[ A 68N P P1qw/GGJJANOOOOuRaRy//KJ qrr33 < <dT$5<s
T(9::;;;;{s5zz))))-q{{}}--j99JE E E E E E E E E       }}H||~~kkmm	   rc   rp   
sympy.ExprUnion[sympy.Expr, int]c                    t          j        t          j        | t          j                  t          j        t          j        |          t          j                            S r   )rO   rc  r`  r|   ra  r   re  )rp   r  s     ra   r  r     sB    6q%+&&u}S))5;77  rc   c                    t          j        t          j        | t          j                  t          j        |t          j                            S r   )rO   rd  r`  r|   ra  )rp   r  s     ra   r  r  '  s8    6q%+&&tU[))  rc   c                d    t          j        t          | |          t          | |                    S r   )rO   re  r  r  )rp   r  r  s      ra   
range_maskr  .  s0    8q#4    rc   r  c                                                        d                                           pdgz   fd}|S )Nr   c                   | d           |  d          t          j        t          j        	fdt	                    D                       }rt          j        |
fd          nt          j        |fd          S )Nc                j    g | ]/}t          |         |         |         z   |                    0S r   )r  )r   rp   r  ih	padding_hs     ra   r   z=constant_boundary_condition.<locals>.load.<locals>.<listcomp>B  s<    WWWqZ1qtil2Yq\MBBWWWrc   c                 B     t                     g           S )Nr   )constant_boundary_condition)r   r  pad_fill_valueprefixr   s   ra   r^  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>G  s/    O3A~3OOO"fNrN  rc   c                 "     g            S r   r   )r  r  r.  s   ra   r^  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>M  s    ((>V>b>*B*B rc   )r   r  rO   re  r  rf  )rJ  rl  r  r  r   r|  r  r  r  r  r   r.  s     @@ra   re  z)constant_boundary_condition.<locals>.load<  s    uuC455\HWWWWWWERUJJWWW
 
 PCJ           D"B"B"B"B"B"BJOO
	
rc   )r   r`  )	r   r|  r  r  r   re  r  r  r.  s	   ````` @@@ra   r  r  5  s     	


cTUUA}}H$A39I
 
 
 
 
 
 
 
 
 
 
 
( Krc   dilationc                  |dgt          |          z  }t          | d||         z  z   ||         ||         dz
  z  z
  ||         dz
  z   ||                   }|rt          | d||         z  z   ||         ||         dz
  z  z
  d||         dz
  z  z   ||                   }t          j        j                            |dz
  ||         z  | z
  ||         z
            dk    r?|dz  }t          j        j                            d|||         z  | z
  ||         z
             t          j        j                            ||z
            dk    r(t          j        j                            ||           d}n|}||fS )Nr+   r)   r   F)r   r&   rP   rY   r   r&  r+  r=  )	r   rp   kernel_sizerM  r  	ceil_moder  x_outx_alts	            ra   pooling_sizer  S  s   3W%	A
NXa[KNQ,>??6!9q=Qq	 E
  '!*nqk[^a/01 6!9q=!" 1I
 
 7%%uqyF1I&=&AGAJ&NOOSTTTQJEG&&q%&)*;a*?'!**LMMM7%%eem4499G))%777IIE)rc   c               l    t          | |          } t          j        t          j        |           }|dk    S )N   )rJ   r   r  r  rr  )r  n_dimwindow_sizes      ra   %should_fallback_max_pool_with_indicesr  q  s0    {E22K"8<==Krc   assert_fallbackc               .   |dk    rdg|z  }|dk    rdg|z  }|s|}t          ||          }t          ||          }t          ||          }t          ||          }t          | t                    sJ t          |          |k    sJ t          |          |k    sJ t          |          |k    sJ t          |          |k    sJ t          |                                           |dz   |dz   fv sJ t          ||          }|||k    sJ |||||fS )Nr   r+   r)   r  )rJ   ri   r@   r   r   r  )r   r  rM  r  r  r  r  use_fallbacks           ra   max_pool_checksr  w  sQ    !||#+1}}3; {E22K&%((F7E**GHe,,Ha#####{u$$$$v;;%w<<5    x==E!!!!qzz||EAI 666668ERRRL"....<??rc   c          
        |                                   | j        d           }| j         d          t          fdt                    D              \  }| j        }	|	t
          j        u rdn.|	j        rt          d          nt          j	        |	          j
        }
t          |          t          |          z   }t                    s(t                    st          d D                       rt          | |
          n|                                 fd}t          j        d| |                                 |	|	||          }t          j        d	| |                                 t
          j        |	||          }t'          |j        j        t                    r|                                 t'          |j        j        t                    r|                                 ||fS )
Nc                H    g | ]}t          |         |           S )r  r  )r   r  r  dhwr  r  r  rM  s     ra   r   z*_max_pool_with_offsets.<locals>.<listcomp>  sM     

 

 

  A;X  

 

 

rc   F-infc              3  "   K   | ]
}|d k    V  dS r)  r   r  s     ra   r   z)_max_pool_with_offsets.<locals>.<genexpr>  s&      ,E,EqQU,E,E,E,E,E,Erc   r  c                    | d           }|  d          fdt                    D             } 	g ||          S )Nc                h    g | ].}|         |         z  |         |         z  z   |         z
  /S r   r   r   rp   bhr  r  reduction_idxrM  s     ra   r   z<_max_pool_with_offsets.<locals>.fn_inner.<locals>.<listcomp>  sR     
 
 
 UVAY=#3hqk#ABWQZO
 
 
rc   r  )
r"  r  r  r  r  r  r  r  rM  r.  s
    `  @ra   fn_innerz(_max_pool_with_offsets.<locals>.fn_inner  s    WufW%\
 
 
 
 
 
 
 
5\\
 
 
 x&2'''rc   r)  rD  
input_noders   	dst_dtyper  rp  ro  reduction_rangesargmax)rb  r  r  r  r   r|   r   r  rF  r  r(  rg   r-  r  r`  r=   r   rl   ra  ri   rk   r  )r   r  rM  r  r  r  r  rc  dhw_outr   	min_valuer0  r  rV  offsetsr  r.  s    ``````        @@ra   _max_pool_with_offsetsr    sK    NNGGeVGE
'5&''
C

 

 

 

 

 

 

 

 

 5\\	

 

 

GY GE EJ 	$6ReFmmmEK<N<N<R  E{{T']]*H
7|| #s9~~ #,E,EH,E,E,E)E)E #.q)GGG==??( ( ( ( ( ( ( ( ( ||~~$	 	 	F ||~~+$	 	 	G &+"I.. ',#Y// 7?rc   c           
         t          |          }t          | |||||d          \  }}}}}t          j        d          5  t	          | ||||||          \  }}	|t          |	t          j                  fcd d d            S # 1 swxY w Y   d S )NFr  r  unroll_reductions_thresholdr  )r   r  r,   r   r  r  r|   r  )
r   r  rM  r  r  r  r  r   rV  r  s
             ra   !_low_memory_max_pool_with_offsetsr    s    E 1@	1 1 1-K(A 
"	5	5	5 
5 
50
 
 
 x444
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5 
5s   5BB
Br  r  "Sequence[Union[int, torch.SymInt]]
input_sizeincrements_to_indexxCallable[[Sequence[Union[int, torch.SymInt]], Sequence[Union[int, torch.SymInt]]], torch._inductor.virtualized.OpsValue]c                Z   t                    |                                 t          j        t	          j        t          j                            fd}t          j	        | 
                                t          j        ||                                           }|S )Nc                     	|           }t          j        |
          }t          j        |          } | |          }t          j        t          j        | d                    t          j                  S r   )rO   r  r-   _flattened_index_to_ndr`  _flatten_indexr|   ra  )r"  r  offset_sympyr  idhwr  r  r  r  offsets_loaderr  s        ra   offsets_to_indicesz4_pool_offsets_to_indices.<locals>.offsets_to_indices  s{    $$,V[AA&=lKXX""366~)$
E6770CDDek
 
 	
rc   rq  )r   r`  r   r8  r   r  r  rr  r<   r   rl   r|   ra  r   )	r  r  r  r  r  r  r  r  r  s	    ```  @@@ra   _pool_offsets_to_indicesr    s     E((**N-	 0{ K KLLK
 
 
 
 
 
 
 
 
 
 !!##k#!!	  G Nrc   c                \    t          |          fd}t          | |||          S )Nc                \    |  d          fdt                    D             S )Nc                h    g | ].}|         |         z  |         |         z  z   |         z
  /S r   r   r  s     ra   r   zX_low_memory_max_pool_offsets_to_indices.<locals>.increments_to_index.<locals>.<listcomp>,  sR     
 
 
 UVAY=#3hqk#ABWQZO
 
 
rc   r  )r"  r  r  r  r  r  rM  s    `@ra   r  zD_low_memory_max_pool_offsets_to_indices.<locals>.increments_to_index*  sY    %\
 
 
 
 
 
 
 
5\\
 
 
 	
rc   )r   r  )r  r  r  rM  r  r  r  r  s      ``` @ra   '_low_memory_max_pool_offsets_to_indicesr  !  s[     E
 
 
 
 
 
 
 
 $j*=  rc   c           	         t          | |||||          \  }}}}}t          | ||||||          \  }}	t          |	|| j        | d          |||          }
||
fS )Nr  )r  r  r  r  )r   r  rM  r  r  r  r  r   rn   r  r  s              ra   _max_pool_with_indicesr  6  s     1@	;1 1 1-K(A *	;9E  LC 6	 G <rc   c           	     .    t          | |||||d          S Nr)   r  r  r   r  rM  r  r  r  s         ra   max_pool2d_with_indicesr  T  *     "	;9A   rc   c           	     .    t          | |||||d          S Nr   r  r  r  s         ra   max_pool3d_with_indicesr  c  r  rc   c                    dk    rddg|dk    rddg}st          |t                    sJ t                    dk    sJ t                    dk    sJ t                    dk    sJ t          |          dk    sJ t          |                                          dv sJ |                                  |                                 }t          |t                    rt          |j        j        t                    r|j        j        }	|	                                }
|
J t          j
        d t          j        |
|	                                |	                                          |	          }|                                 |                                }n|                                }|d ur|d         dk    p|d uo|d         dk    }t          d |D                       rt!          | ||||          S |                                ^ }}|                                 ^ }|                                |                                 t%          |                                          }t'          fdt)          d         dz            D                       t'          fd	t)          d         dz            D                       z  }|d
k    rt!          | ||||          S |                                fd}t          j        |                                 |                                 ||          }|rt          j                            |          S |S )Nr   r+   r)   r$  )rs   r   r   r  c              3  "   K   | ]
}|d k    V  dS r)  r   r  s     ra   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>  s&      
$
$a16
$
$
$
$
$
$rc   c           
   3     K   | ]@}t          |d          z  t          d |d          z
  d          z            z
  d          V  AdS r   r+   Nr)  r   r  r  rM  s     ra   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>  j         	ANSQQ%7F1I$EFFFJJ     rc   c           
   3     K   | ]@}t          |d          z  t          d|d          z
  d          z            z
  d           V  AdS r+   r   Nr  r   wr  rM  s     ra   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>  r  rc   r  c                d   | ^ }}}t          j        |z  |z   t          j                  }|d         z   }|d         z   }t          j        t	          |d         z
  d         z   d                   t          j                  }t          j        t	          |d         z
  d         z   d                   t          j                  }t          j        t	          |d                   dz   t          j                  }t          j        t	          |d                   dz   t          j                  }t          j        |t          j        dt          j                            }t          j        |t          j        dt          j                            }t          j        |t          j        t          j                            }t          j        |t          j        t          j                            }d }	t                    D ]}
t                    D ]}t          j	        |t          j        |
t          j                            }t          j	        |t          j        |t          j                            }g |t          j
        t          j        |t          j        |t          j        dt          j                                      d         d          t          j
        t          j        |t          j        |t          j        dt          j                                      d         d          } |          } |          }t          j        ||          }|	5t          j        ||t          j        dt          j                            }	t          j        t          j        t          j        ||          t          j        ||                    |          }t          j        |t          j	        |	|          |	          }	|	J |	S )Nr   r+   Fr  r]  r_  )rO   r`  r|   r  r&   r  rb  r  r  r   r  r  rJ  r  r~  re  rd  )r"  r  r  r  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   rl  grad_loaderh_window_sizer  indices_sizer  r  pooled_heightpooled_widthrM  w_window_sizewidths                      ra   rv   z,max_pool2d_with_indices_backward.<locals>.fn  s   A^AIM5;??

N
N.QQ'&)3VAY??
 
 .QQ'&)3VAY??
 
 x6!95595;GGx6!95595;GG+gs|Au{'C'CDD+gs|Au{'C'CDDE3>-#M#MNNE3>,#L#LMM'' #	W #	WC]++ "W "WWWcl3&D&DEEWWcl3&D&DEE)Bs|Au{7S7S(T(TUU$R(#   )Bs|Au{7S7S(T(TUU$R(#  
  .~j99'K
33	|Z88#"yy#,sEM*J*J   HH 8F2u--F2u--   D  #yswx/K/KXVVHHE"WF ###rc   rq  )ri   r@   r   r   rb  r/  rk   r<   rl   r.   r  r  r   decide_layoutr  r-  )fallback_max_pool2d_with_indices_backwardr`  rg   r)  r  r   r  r  )grad_outputr   r  rM  r  r  r  r  	gO_striderk   rs   x_bufferx_strideis_channels_last_batch_heightr   r0  r  rv   rn   r  r  r  r  r  r   r!  r"  s     ```                @@@@@@@@ra    max_pool2d_with_indices_backwardr,  w  s>    !||a&1}}q6 a#####{q    v;;!w<<1x==Aqzz||&&&& ,,..I!Y (Jqv{I$F$F (v{""!!!$$nn&&]]__  
 
 
 
 	   &&((%%'' ,A!1A 3)A,!"3  
$
$8
$
$
$$$ 
8K(Iw
 
 	
  jjllVWe&1&:&:&<&<#Q|((**N))++KAJJLL!!H     {1~)**    M      {1~)**    M
  -/KR8K(Iw
 
 	
 ##%%L9 9 9 9 9 9 9 9 9 9 9 9 9 9 9v 
%%''##%%	  C  44S999
rc   r_  c                >    |                                  fd}|S )Nc           
        	
 |\  	|\  
|\  }}t          j        t          j        t          j        z   t          j                  t          j        |t          j                            t          j        t          j        
	z   t          j                  t          j        |t          j                                      }t          j        |	 
fd          S )Nc                 2     g  z   z             S r   r   )h_start_indexr  iwr  w_start_indexr.  s   ra   r^  z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s+    HHNvN}r'9N=2;MNOO rc   )rO   re  rd  r`  r|   ra  rf  )r  
incrementsstart_indicesend_indicesh_end_indexw_end_indexrl  r0  r  r1  r2  pad_valr.  s   `      @@@@ra   re  z!pad_adaptive_loader.<locals>.load  s    B'4$}#. [xF}r15;??{EK88  F}r15;??{EK88 	
 	
 zOOOOOOOOO
 
 	
rc   r_  )r   r8  re  r.  s    ` @ra   pad_adaptive_loaderr9  	  s6    }}H
 
 
 
 
 
, Krc   c                    t          j        | ||          }t          j        |||          }t          j        | ||          }t          j        |||          }	||||	fS )N)out_diminp_dim)r   r4  )
start_index	end_indexh_inw_inh_outw_outr0  r6  r2  r7  s
             ra    compute_indices_adaptive_poolingrC  %  sm    %k5$OOOM#IudKKKK%k5$OOOM#IudKKKK+}kAArc   c                l    |\  }}|\  }}	t          | |||||	          \  fd}
|
S )Nc                <   | ^ }}} |          } |          } |          } |          }d }	t          j        t          d                   t          d                             D ]*\  }
} |||
|g||g||g          }|	|}	 ||	          }	+|	S r  )r  productr  )r"  r  r  r  bwr0  r6  r2  r7  rV  r  r1  r  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                ra   rv   z _adaptive_pooling_fn.<locals>.fn>  s    R((,,$nR((((,,$nR(('l1o(>(>lSTo@V@VWW 
	1 
	1FB&R.k*	 C ~#C00rc   rC  )r=  r>  rJ  in_sizes	out_sizesrK  r?  r@  rA  rB  rv   rH  rI  rL  rM  s     `  `     @@@@ra   _adaptive_pooling_fnrQ  /  s     JD$LE5 	)YdE5	 	
         . Irc   c                p   
 |\  }|\  }}t          | ||||          \  

fd}	|	S )Nc                   | ^ }}} |          } |          } |          } |          }d }	d }
t          j        t          d                   t          d                             D ]\  }} ||||g||g||g          }t          j        ||z   z  |z   |z   t
          j                  }|
|}
n)t          j        t          j        ||	          ||
          }
|	|}	w ||	          }	|
S r  )	r  rF  r  rO   r`  r|   ra  r  gt)r"  r  r  r  rG  r0  r6  r2  r7  maxvalmaxindexr  r1  r  rJ  rH  rI  rJ  rK  rL  r@  rM  s                  ra   rv   z)_adaptive_pooling_fn_with_idx.<locals>.fng  s:   R((,,$nR((((,,$nR(('l1o(>(>lSTo@V@VWW 	1 	1FB&R.k*	 C N#t+m;b@%+ E  9SVC%8%8%JJ~#C00rc   rN  )r=  r>  rJ  rO  rP  rK  r?  rA  rB  rv   rH  rI  rL  r@  rM  s     `  `    @@@@@ra   _adaptive_pooling_fn_with_idxrW  X  s     JD$LE5 	)YdE5	 	
! ! ! ! ! ! ! ! ! ! !F Irc   c                                                      t          j        k    rt          d          t	           t
                    sJ t          |          dk    sJ                                                                    ^ }}}t          j
        j                            |          }t          j
        j                            |          }|\  }}||k    r||k    rt                     S |dk    s|dk    r>g |||}t          |                                                                             S ||z  dk    r#||z  dk    r||z  ||z  g}t!           |          S t#          ||z   dz
  |          }	t#          ||z   dz
  |          }
t%          |          ||gz   }                                  }|	|
z  }|dk    rt'           |          S d }d }t)          |||	|
g||g||gt*          j        	          t/          t1                                fd
}t3          j                                         |||          }|S )Nz0'adaptive_avg_pool2d' not implemented for 'Long'r)   r   r'  r+   r  c                (    t          | |z  |          S r   r&   rJ  r;  r<  s      ra   r=  z)_adaptive_avg_pool2d.<locals>.start_index      7333rc   c                :    t          | dz   |z  |z   dz
  |          S r  rZ  r[  s      ra   r>  z'_adaptive_avg_pool2d.<locals>.end_index  %    g-7!;WEEErc   r=  r>  rJ  rO  rP  rK  c                p    t          j         | t                               |                     S r   )rO   truedivr9  )r"  fn_sumones_loaderr   s    ra   rv   z _adaptive_avg_pool2d.<locals>.fn  s=    {F3+A..//[1I1I
 
 	
rc   rq  )r   r|   ra  ry  ri   r@   r   rb  r   rP   rY   r   r  r  r/  rl   
avg_pool2drC   rg   fallback_adaptive_avg_pool2drQ  rO   r   r9  	ones_liker<   r   )r   r  rc  r?  r@  rA  rB  o_sizer  h_kernel_maxw_kernel_maxr0  r   r  r=  r>  rv   rvrb  rc  s   `                 @@ra   _adaptive_avg_pool2drk    s   {{}}##MNNNa#####{q    NNUD$7%%d++D7%%d++DLE5 u}}QxxzzUaZZ'5'%''V1;;==HHHHe|qTE\Q..u}dem4![)))D5L1,u55LD5L1,u55LE{{eU^+HKKMME-KR+A{;;;4 4 4F F F ""L1%.7  F &ill33K
 
 
 
 
 
 

 
	||~~	
 
 
B Irc   c                0                                      t          j        k    rt          d          t	           t
                    sJ t          |          dk    sJ                                                                    ^ }}}t          j
        j                            |          }t          j
        j                            |          }|\  }}|dk    s|dk    rlg |||}t          |                                                                             t          |t          j                                                   fS ||z  dk    r||z  dk    rt          t!          ||z   dz
  |          }t!          ||z   dz
  |          }	t#          |          ||gz   }
                                  }||	z  }|dk    rt%           |          S d }d }t'          ||||	g||g||gt(          j        	          t-          ||||	g||g||gt(          j        	           fd
} fd}t/          j                                         |||
          }t/          j                                         t          j        ||
          }||fS )Nz,adaptive_max_pool2d not implemented for Longr)   r   r'  r+   r  c                (    t          | |z  |          S r   rZ  r[  s      ra   r=  z(adaptive_max_pool2d.<locals>.start_index  r\  rc   c                :    t          | dz   |z  |z   dz
  |          S r  rZ  r[  s      ra   r>  z&adaptive_max_pool2d.<locals>.end_index  r^  rc   r_  c           	     R     | t          t          d                              S Nr  r9  rF  )r"  inner_func_max_valr   s    ra   inner_fn_max_valz-adaptive_max_pool2d.<locals>.inner_fn_max_val  '    !!#':1eFmm'L'LMMMrc   c           	     R     | t          t          d                              S rp  rq  )r"  inner_func_max_idxr   s    ra   inner_fn_max_idxz-adaptive_max_pool2d.<locals>.inner_fn_max_idx  rt  rc   rq  )r   r|   ra  ry  ri   r@   r   rb  r   rP   rY   r   r  r/  rl   
ValueErrorrC   rg   fallback_adaptive_max_pool2drQ  rO   r  rW  r<   r   )r   r  rc  r?  r@  rA  rB  rg  rh  ri  r0  r   r  r=  r>  rs  rw  rj  rirv  rr  s   `                  @@ra   adaptive_max_pool2dr{    s   {{}}##IJJJa#####{q    NNUD$7%%d++D7%%d++DLE5zzUaZZ'5'%''V1;;==HHH%%+allnnK
 K
 K
 
 	
 e|qTE\Q..D5L1,u55LD5L1,u55LE{{eU^+HKKMME-KR+A{;;;4 4 4F F F ."L1%.;   7"L1%.;  N N N N N NN N N N N N 
	||~~!	
 
 
B 
	||~~k!	
 
 
B r6Mrc   c                                                                                   fd}|S )Nc                r                                    }t          |          dk    rwt          |           dk    r d| d         dz
  
z
  g          }n^t          |           dk    r! | d         | d         dz
  
z
  g          }n* dddz
  
z
  g          }n g | dz
  
z
            }t          j        |                                          }t          j        z
  t
          j                  }t          j        dz
  t
          j                  }t          j        t          j        |t
          j	                  t          j        |t
          j	                            }t          j
        t          j        |d          d|          }t          j        ||z   |z            t          j        ||z            z
  }t          j        |t
          j                  }t          j        ||          }	t          j        t          j
        |	||          t          j                            S )Nr   r+   r   r)   )r   r   rO   r`  r   r|   ra  ra  r  float64r  rJ  r  rd  r  r   r8  )r  rp   samples_shapesamplei_exprdiffout_sz_exprrX  seq_irl  r   in_sz	kernel_szndimsout_szsamplessamples_loaders             ra   re  z)_fractional_pooling_offsets.<locals>.load5  s     ((**}""6{{a (F1Iuqy3(GHHV!! (F1Iuqy3(OPP (Auqy3(?@@ $^$>f$>eai#o$>??F7#4#4#6#677~ei/==nVaZ==Lu}--s|K/W/W
 
 	#&a00!U;;	6F?e344sy%7P7PPUEK00vfk**$SYtUD%A%A5=QVCWCWXXXrc   r_  )r  r  r  r  r   r  re  r  s   `````` @ra   _fractional_pooling_offsetsr  /  s    C[F#JE#I((**N!Y !Y !Y !Y !Y !Y !Y !Y !Y !Y !YF Krc   c                *    t          | |||d          S r  _fractional_max_poolr   r  r  random_sampless       ra   fractional_max_pool2dr  [      ;^STUUUUrc   c                *    t          | |||d          S r   r  r  s       ra   fractional_max_pool3dr  `  r  rc   c                
   |                                   | j        d           | j         d          c}t          j        d          5  fdt	                    D             |                                 fd}fdt          |          t                    z   }|                                 }t          j	        d| | 
                                ||||          }	t          j	        d| | 
                                t          j        |||          }
t          |	t                    s
J |	            t          |	j        j        t                    r|	                                 t          |
t                    s
J |
            t          |
j        j        t                    r|
                                 t#          |
| j                  }|	|fcd d d            S # 1 swxY w Y   d S )	Nr  r  c           
     :    g | ]}t          |           S ))r  r  r  r  r  r   )r  )r   r  inp_dhwr  r  r  r  s     ra   r   z(_fractional_max_pool.<locals>.<listcomp>j  sL     

 

 

  (&"%  

 

 

rc   c                L    | d           } g | | |                    S r   r   )r"  r  r  r  r  r.  s      ra   r  z&_fractional_max_pool.<locals>.fn_innerx  s;    5&\F8OfO':':3'N'NOPPPrc   c                r    | d           |  d          fdt                    D             S )Nc                R    g | ]#} |         |                   |         z   $S r   r   )r   r  bdhwdhw_index_fnr  r  s     ra   r   zE_fractional_max_pool.<locals>.increments_to_index.<locals>.<listcomp>  sG         QQ00=3CC  rc   r  )r"  r  r  r  r  r  s    `@@ra   r  z1_fractional_max_pool.<locals>.increments_to_index|  sd    5&\Fvww<D      u   rc   r)  r  r  )rb  r  r,   r   r  r`  rg   r   r=   r   rl   r|   ra  ri   r@   rk   r  r  )r   r  r  r  r  rc  r  r0  r   rV  r  r  r  r  r  r.  s    ````       @@@@ra   r  r  e  s   NNWWufW%qwvww'7NE7	"	5	5	5 = =

 

 

 

 

 

 

 

 5\\

 

 

 ==??	Q 	Q 	Q 	Q 	Q 	Q 	Q	 	 	 	 	 	 ;;k!2!22! <<>>(	
 	
 	
 "#<<>>k(	
 	
 	
 &),,44f44,fk&	22 	NN'9--66w66-gl'33 	OO*[!'+>
 
 w{= = = = = = = = = = = = = = = = = =s   FG88G<?G<c                                                                                        ^ }}}t          j        j                            |          }t          j        j                            |          }|^ }}}	||z  dk    r#||	z  dk    rt           ||z  ||	z  gd          S t          ||          }
t          ||	          }d fd}t          ||
|g||g||	gt          j
                   fd}t          j                                                                          |t          |                    }|S )	Nr   r+   )divisor_overridec                L    t          | |z  t          j        |                    S r   )r%   r   r8  r[  s      ra   r=  z0upsample_nearest2d_backward.<locals>.start_index  s     uwg(>(>???rc   c                $     | dz   ||          S r  r   )rJ  r;  r<  r=  s      ra   r>  z.upsample_nearest2d_backward.<locals>.end_index  s    {EAI999rc   r_  c                6     | t                              S r   )r9  )r"  rb  r   s    ra   rv   z'upsample_nearest2d_backward.<locals>.fn  s    vc.q11222rc   rq  )rb  r   rP   rY   r   r  rd  rC   rQ  rO   r   r<   r   rl   r   rg   )r   r  r  ro  rp  r*  inp_hinp_wout_hout_wrh  ri  r>  rv   rj  rb  r=  s   `              @@ra   upsample_nearest2d_backwardr    s    NNJJLLVUEG&&u--EG&&u--E&VUEu}eemq00!euneun=PQRRRR5%((L5%((L@ @ @: : : : : ""L1%.7  F3 3 3 3 3 3 
	||~~kkmmJ	
 
 
B Irc   r   c           
     0    t          | ||||||d          S )Nr)   r  _avg_poolndr   r  rM  r  r  count_include_padr  s          ra   rd  rd    3     		 	 	 	rc   c           
     0    t          | ||||||d          S )Nr   r  r  r  s          ra   
avg_pool3dr    r  rc   c           	        ssdgz  t                    t                    t                    t          | t                    sJ t                    k    sJ t                    k    sJ t                    k    sJ t          |                                           dz   dz   fv sJ |                                  |                                 d           }|                                  d          t          fdt                    D              \  }	}
t                    st          |
          rt          | d          d}n| 
                                d}t          |          t          |	          z   }|                                 t          j        t          j                  }|d	k    r?dk    rt"          }n d
k    rt$          }nt'          d            || |          S fd|r|r$|r|n|j        rdz  fd}nfd}nf	d}t+          j        |                                 ||          }|S )Nr   r+   r)   c           
     D    g | ]}t          |         |          S r   r  )r   rp   r  r  r  r  rM  s     ra   r   z_avg_poolnd.<locals>.<listcomp>(  s?     

 

 

 1q+vw	JJ

 

 

rc   r_  r  TFr  r   zUnknown dim: c                   | d           }|  d          d }t          j        	fdt                    D              D ]I
fdt                    D             } |g ||          }||}4t          j        ||          }J|S )Nc                :    g | ]}t          |                   S r   r  )r   rp   r  s     ra   r   z/_avg_poolnd.<locals>.fn_sum.<locals>.<listcomp>P  s%    %P%P%PeKN&;&;%P%P%Prc   c                V    g | ]%}|         |         z  |         z   |         z
  &S r   r   )r   rp   r@  r  r  rM  s     ra   r   z/_avg_poolnd.<locals>.fn_sum.<locals>.<listcomp>Q  s8    MMMQ1Q4&)#be+gaj8MMMrc   )r  rF  r  rO   r   )r"  r  r  totalr   r  r@  r  r   r  r  rM  s         @@ra   rb  z_avg_poolnd.<locals>.fn_sumL  s    UsdUJ#%P%P%P%PU3ZZ%P%P%PQ 	, 	,BMMMMMMM%**MMMC&6C))C}U++rc   c                h    t          j         |           t          j                            S r   )rO   rr  rb  )r"  r   rb  r  r.  s    ra   rv   z_avg_poolnd.<locals>.fn^  s-    wvvc844cl5%6P6PQQQrc   c                h    t          j         |           t          j                            S r   )rO   truncdivrb  )r"  divisorr   rb  r.  s    ra   rv   z_avg_poolnd.<locals>.fnc  s.    |FF3$9$93<QV;W;WXXXrc   c                j  	 | 	 d          }g }t          	          D ]}||         |         z  |         z
  }t          j        ||         z   |         |         z             }s0t          j        |d          }t          j        ||                   }t	          j        ||z
  t          j                  }|                    |           t          j
        t          j        |          }
j        rt	          j         |           |          S t	          j         |           |          S r  )r  r   MinMaxrO   r`  r|   r  rm   r   r  rr  r  ra  r  )r"  r  divide_factorsrp   hstarthendfactordivide_factorr  r   r   rb  r  r  r  rM  r.  s           ra   rv   z_avg_poolnd.<locals>.fni  s&   cTUUBN3ZZ . .A*WQZ7y+a.!8!A$:KLL( 1"Yvq11F 9T1Q400Dvu{CC%%f----%,SWnEEM& I{66#x#8#8-HHH<sH 5 5}EEErc   rq  )rJ   ri   r@   r   r   rb  r  r  r-  r  r`  rg   r   r   r  r  rr  fallback_avg_pool2dfallback_avg_pool3drx  r  r<   r   rl   )r   r  rM  r  r  r  r  r   rc  rA  
ceil_modeshad_paddingr0  r  fallbackrv   rj  r  r   rb  r  r  r.  s    ````` `         @@@@@@ra   r  r    s      #){C00K&#&&F7C((Ga#####{s""""v;;#w<<3qzz||q#' 22222NNJJLL3$E	

cTUUA

 

 

 

 

 

 

 

3ZZ

 

 

E: 7|| s: .q#3???==??E{{T%[[(HKKMME"8<==KR!88*HHAXX*HH2S22333x
 
 	
          F*  F&6G""K" 
	YKER R R R R R R R R
Y Y Y Y Y Y Y Y Y	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F$ 
	||~~	
 
 
B Irc   c                   dk    s
J d            ssddgt          | t                    sJ t          |t                    sJ t                    dk    sJ t                    dk    sJ t                    dk    sJ t          |                                          dv sJ |                                  |                                ^ }t          d|          \  }	}
t          d|          \  }}|                                 d         pd         p|
p||                                 ^ }t          |                                          }|                                }t          fdt          d         dz            D                       t          fdt          d         dz            D                       z  }|dk    rt          | ||          S fd	fd
}t          j        |                                 |||          }|S )Nr   divisor must be not zeror)   r$  r+   c           
   3     K   | ]@}t          |d          z  t          d |d          z
  d          z            z
  d          V  AdS r  r  r  s     ra   r   z&avg_pool2d_backward.<locals>.<genexpr>  r  rc   c           
   3     K   | ]@}t          |d          z  t          d|d          z
  d          z            z
  d           V  AdS r
  r  r  s     ra   r   z&avg_pool2d_backward.<locals>.<genexpr>  r  rc   r  c           	        t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        d         t          j                  }t          j        t          j        | |          |          }t          j        t          j        ||          |          }	t          j        t          j        ||          t          j        t          j        t          j                  |                    }
t          j        t          j        |	|          t          j        t          j        t          j                  |                    }t          j	        |t          j        dt          j                            }t          j	        |	t          j        dt          j                            }	t          j        |
t          j        t          j                            }
t          j        |t          j        t          j                            }t          j        t          j        |
|          t          j        ||	                    }|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r+   )
rO   rb  r|   r  r  rr  r  r   r`  r  )r  r  stride_hstride_wpad_hpad_wkernel_hkernel_wr  wstartr  wendr  heightr  r  rM  r"  s                ra   !compute_pool_size_without_paddingz>avg_pool2d_backward.<locals>.compute_pool_size_without_padding  s   
 <q	5;77<q	5;77WQZ55WQZ55<A<<<A<<X..66X..66{GFH%%GCN65;77??
 
 {GFH%%GCN5%+66>>
 
 VS\!U[%A%ABBVS\!U[%A%ABB{4!D!DEE{4u{!C!CDDf 5 5swtV7L7LMMrc   c                   | ^ }}}|d         z   }|d         z   }t          j        t          |d         z
  d         z   d                   t          j                  }t          j        t          |d         z
  d         z   d                   t          j                  }t          j        t          |d                   dz   t          j                  }t          j        t          |d                   dz   t          j                  }t          j        |t          j        dt          j                            }t          j        |t          j        dt          j                            }t          j        |t          j        t          j                            }t          j        |t          j        t          j                            }d }t                    D ]}	t                    D ]}
t          j	        |t          j        |	t          j                            }t          j	        |t          j        |
t          j                            }}n"ssd         d         z  }n ||          }t          j
         g |t          j        t          j        |t          j        |t          j        dt          j                                      d          t          j        t          j        |t          j        |t          j        dt          j                                      d                    |          }t          j        t          j        ||          t          j        ||                    }|5t          j        ||t          j        dt          j                            }t          j        |t          j	        ||          |          }|J |S )Nr   r+   Fr  r_  )rO   r`  r&   r|   r  r  rb  r  r  r   ra  r  r  re  rd  r  r~  )r"  r  r  r  r  r  r  r  r  r  r  r  r  r  partrl  r  r  r  r  r  r  r  r  r  r   rM  r!  s                   ra   rv   zavg_pool2d_backward.<locals>.fn  s   A
N
N.QQ'&)3VAY??
 
 .QQ'&)3VAY??
 
 x6!95595;GGx6!95595;GG+gs|Au{'C'CDD+gs|Au{'C'CDDE3>-#M#MNNE3>,#L#LMM'' *	R *	RC]++ )R )RWWcl3&D&DEEWWcl3&D&DEE#/,EE& Fk F'N[^;EE==b"EEE{K#1 #$&s|Au{7S7S(T(T!" !" !.&+    1 #$&s|Au{7S7S(T(T!" !" !-&+   & ) . xF2u%%F2u%%  #"ytS\#u}5U5UVVHH"yswx/F/FQQHHS)RT ###rc   rq  )ri   r@   r   r   rb  r  r`  rg   r   r)  r  fallback_avg_pool2d_backwardr<   r   rl   )r%  r   r  rM  r  r  r  r  r   _h_out
ceil_mode1_w_out
ceil_mode2r0  r   r  rv   rj  r  r  r  r  r  r  r   r!  r"  s     ``` ``          @@@@@@@@@ra   avg_pool2d_backwardr    s?    #'71'<'<'<>X'<'<<  a&k9-----a#####{q    v;;!w<<1qzz||&&&&

Q%; FJ &eQVWiXXFJ))++K!*F
FjFJK&1&:&:&<&<#Q|AJJLL!!HKKMME     {1~)**    M      {1~)**    M
  -/KR+	
 	
 		
        8? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?B 
	%%''	
 
 
B Irc   c                j    dk    s
J d            ssg dt          | t                    sJ t          |t                    sJ t                    dk    sJ t                    dk    sJ t                    dk    sJ t          |                                          dv sJ |                                  |                                ^ } t          d|          \  }	}
t          d|          \  }}t           d|          \  }}|                                 t                    p|
p|p||                                 ^ }t          |                                          }|	                                }fdt          d          D             \  z  z  }|d	k    rt          | ||          S  fd
fd}t          j        |                                 |||          }|S )Nr   r  )r   r   r   r   )r   r   r+   r)   c              3     K   | ]7t          fd t                   dz            D                       V  8dS )c           
   3     K   | ]@}t          |         z  t          d |         z
           z            z
  d          V  AdS r  r  )r   r  rp   r  rM  s     ra   r   z0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>c  sj       
 
 VAYQ[^);q	(I!J!JJANN
 
 
 
 
 
rc   r)   N)r)  r  )r   rp   r  rM  s    @ra   r   z&avg_pool3d_backward.<locals>.<genexpr>b  s       3 3
 	 	 
 
 
 
 
 
;q>A-..
 
 
 	
 	
3 3 3 3 3 3rc   }   c           	     @   d D             \  }}}d D             \  }}}d D             \  }	}
}d t          | ||g|||g|||g          D             \  }}}d t          |||g|	|
|gg|||g          D             \  }}}d |||fD             \  }}}d t          |||gg          D             \  }}}t          j        t          j        t          j        ||          t          j        ||                    t          j        ||                    }|S )Nc              3  T   K   | ]#}t          j        |t          j                  V  $d S r   rO   rb  r|   r  r   s     ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>y  s0      'U'UQ(D(D'U'U'U'U'U'Urc   c              3  T   K   | ]#}t          j        |t          j                  V  $d S r   r  r  s     ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>z  s0      MMs|Au{;;MMMMMMrc   c              3  T   K   | ]#}t          j        |t          j                  V  $d S r   r  )r   r   s     ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>{  sA       (
 (
-.CLEK(((
 (
 (
 (
 (
 (
rc   c              3  n   K   | ]0\  }}}t          j        t          j        ||          |          V  1d S r   )rO   r  rr  )r   r  r   pads       ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  sR       "
 "
1c GCGAqMM3''"
 "
 "
 "
 "
 "
rc   c           
   3     K   | ]a\  }}}}t          j        t          j        ||          t          j        t          j        |t          j                  |                    V  bd S r   )rO   r  r   r`  r|   r  )r   r>  r   r   r  s        ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  sw       

 

 #q#s Kq!!373>#u{+K+KS#Q#Q 

 

 

 

 

 

rc   c              3  z   K   | ]6}t          j        |t          j        d t          j                            V  7dS r  rO   r  rb  r|   r  )r   r>  s     ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  sP       "
 "
 Ks|Au{;;<<"
 "
 "
 "
 "
 "
rc   c              3     K   | ]9\  }}t          j        |t          j        |t          j                            V  :d S r   rO   r  r`  r|   r  )r   r?  r   s      ra   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  sT       
 
S KS^C==>>
 
 
 
 
 
rc   )r  rO   rr  r  )pdr  r  stride_dr  r  pad_dr  r  kernel_dr  r  dstartr  r  dendr  r  r  depthr  r  r  rM  r"  s                      ra   r  z>avg_pool3d_backward.<locals>.compute_pool_size_without_paddingx  s   'U'Uf'U'U'U$(HMMWMMMue(
 (
2=(
 (
 (
$(H"
 "
 Rx8<ueU>S "
 "
 "


 

 '*(8X.&u%	' '	

 

 

dD"
 "
 &&1"
 "
 "

 
tT 2UFE4JKK
 
 
dD GCGD&))374+@+@AA374QWCXCX
 
 rc   c                X   | ^ }}}}d t          |||g          D             \  }}}d t          |||g!          D             \  }}}d t          |||g!          D             \  }}	}
d |||fD             \  }}}d t          ||	|
g g          D             \  }}	}
d }t                    D ]o}t                    D ][}t          "          D ]G}d t          |||g|||g          D             \  }}}}n,ssd         d         z  d	         z  }n |||          }t          j         g |t          j        t          j        |t          j        |t          j        dt          j	                                      d
          t          j        t          j        |t          j        |	t          j        dt          j	                                      d
          t          j        t          j        |t          j        |
t          j        dt          j	                                       d
                    |          }t          j
        t          j
        t          j        ||          t          j        ||	                    t          j        ||
                    }|5t          j        ||t          j        dt          j                            }t          j        |t          j        ||          |          }I]q|J |S )Nc              3  &   K   | ]\  }}||z   V  d S r   r   )r   r   r  s      ra   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s*      AAvq#1s7AAAAAArc   c              3     K   | ];\  }}}t          j        t          ||z
  |z   |          t          j                  V  <d S r   rO   r`  r&   r|   r  )r   r   r   r   s       ra   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s\       %
 %
1a N8AEAIq115;??%
 %
 %
 %
 %
 %
rc   c              3  |   K   | ]7\  }}t          j        t          ||          d z   t          j                  V  8dS r)  r  )r   r   r   s      ra   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  sU       
 
1 N8Aq>>A-u{;;
 
 
 
 
 
rc   c              3  z   K   | ]6}t          j        |t          j        d t          j                            V  7dS r  r  )r   pstarts     ra   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  sP       %
 %
 KQ < <==%
 %
 %
 %
 %
 %
rc   c              3     K   | ]9\  }}t          j        |t          j        |t          j                            V  :d S r   r  )r   pend
pooled_dims      ra   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  sT       
 
 j KcnZEEFF
 
 
 
 
 
rc   c              3     K   | ]9\  }}t          j        |t          j        |t          j                            V  :d S r   )rO   r   rb  r|   r  )r   r  p_s      ra   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  sT       " "&FB R(E(EFF" " " " " "rc   r   r+   r)   Fr  r_  )r  r  rO   ra  r  r  r  rb  r|   r  re  rd  r  r~  r   )#r"  r  r  r  r  pdstartr  r  pdendr  r  r  pd_r  r  r  r  r  r  r  rl  r  r  d_window_sizer  r  r  r  r  r  pooled_depthr  r   rM  r!  s#                        ra   rv   zavg_pool3d_backward.<locals>.fn  s   AqAAaAY)@)@AAA1a%
 %
1ay+v>>%
 %
 %
!'

 
Q1Iv..
 
 
ue
%
 %
"GW5%
 %
 %
!'
 
$'u%m\'R% %
 
 
ue '' 8	V 8	VC]++ 7V 7V // 6V 6VC" "*-$gw7#sC+ +" " "JBB (3 0* N+ N +AQ ?+a. P A A"b" M M;#!' # 5$'K(*CGE3<5;;W;W,X,X%& %& %1*/!" !" !" !$ 5$'K(*CGE3<5;;W;W,X,X%& %& %2*/!" !" !"  !$ 5$'K(*CGE3<5;;W;W,X,X%& %& %1*/!" !" !"! 4 7 D< 8E!2!2CF2u4E4EFFr5)) D  '#&9 $S%-(H(H$ $ $'9T378T3J3JH#U#Um6V7Vp ###rc   rq  )ri   r@   r   r   rb  r  r`  r-  rg   r   r  fallback_avg_pool3d_backwardr<   r   rl   )!r%  r   r  rM  r  r  r  r  r*  _d_outceil_mode_dr  ceil_mode_hr  ceil_mode_wr   r0  r   r  rv   rj  r  r  r  r  r  r  r  r  r  r   r!  r"  s!     ``` ``             @@@@@@@@@@@@ra   avg_pool3d_backwardr
  5  s9    #'71'<'<'<>X'<'<<  ))k9-----a#####{q    v;;!w<<1qzz||&&&&$%JJLL!VUFE&q+vw	 FK '; FK 'q+vw	 FK ))++Kg,,K+KKK4?4H4H4J4J1Qm\AJJLL!!HKKMME3 3 3 3 3
 q3 3 3/M=-  -/-?KS+	
 	
 		
# # # # # # # # # #JU U U U U U U U U U U U U U U U U Un 
	%%''	
 
 
B Irc   c                   |                                  }t          |t                    r|g}n|st          t	          |                    }t	          |          dk    r t          |          dv sJ d|             g S t          |          }t          t	          |                    D ]}||         dk     r.||xx         t	          |          rt	          |          ndz  cc<   d||         cxk    rt	          |          k     s#n t	          |          dk    r||         dk    sJ t	          t          |                    t	          |          k    s
J d            |S )Nr   )r   r  r9  zinvalid axis: r+   zreduction axis not unique)r   ri   r   r  r   r   rg   r$   )r   rs  r   rp   s       ra   _validate_reduction_axisr    sb   ::<<D$  v  SYY
4yyA~~T{{////1H$1H1H///	::D3t99 M M7Q;;GGGCII4s4yyy14GGGDG''''c$ii''''CIINNtAw!|||Lz$  CII---/J---Krc   c                 
 |t          | |          } |                                 t          t                   t	          | |                    }g }g g }g t          t                              D ]g}||v r1                    |           |                    |                    7                    |           |                    |                    h
fd}r)t                    }	D ]}t          j
        j        |	|<   n|}	|                                 
t          |                                 |p|                                 |                                 ||	|          S )Nc                    t          |          t                    k    sJ r0t                     t          	          k    sJ  fdD              t                     t                    k    sJ d gt                     t          |          z   z  }t          j        t                     t          |                    D ]
\  }}|||<    |          S )Nc                     g | ]
}|         S r   r   )r   rp   rJ  s     ra   r   z9_make_reduction_inner.<locals>.loader.<locals>.<listcomp>%  s    000!U1X000rc   )r   r  r  r  )
rJ  reduction_indexr  r"  varinner_loaderkeepdimskept_idxreduced_idxr   s
   `    ra   r  z%_make_reduction_inner.<locals>.loader!  s    ?##s;'7'77777 	1u::T****0000x000E5zzS]]****Fc%jj3+?+??@	!%  #k?"C"C
 
 	! 	!HC !IcNN|I&&&rc   )rs   r  r  rp  ro  r  )r  r   r$   r   r  r  r   rm   rg   r   r:  r;  r`  r"  rl   r   )r   rs  r  r   rS  
kept_sizesreduced_sizesrp   r  r0  r  r  r  r   s     `       @@@@ra   _make_reduction_innerr    s   Q::<<Dc?3At<<==DJHMK3t99 ' '99q!!!  a))))OOAd1g&&&&' ' ' ' ' ' ' ' '  :: 	& 	&A'+HQKK	& ==??L||~~'81;;==++--&   rc   rD  rQ   c                     dd d fd}|S )NFr   c                   t          | |||          }t          j        d| d|}t          |j        j        t                    r|                                 |S )Nrs  r  r   rS  )rD  r  r   )r  r=   r   ri   rk   r  )r   rs  r  r   r   rV  rS  rD  s         ra   r|  zmake_reduction.<locals>.innerA  sz    &"7
 
 
 !XAXXQWXXK
 
 	 NNrc   r   r   )rD  rS  r|  s   `` ra   make_reductionr  @  s:    T          Lrc   c                  |t          | |          } t          | |          }t          |                                 |                                 f|                                 f|                                 |          S )N)rs   dtypes	inner_fnsr   rs  )r  r<  r"  rl   r   r`  r   )r   rs  r   s      ra   _make_scan_innerr   T  sr    QD!!D||~~==??$ZZ\\   rc   r   c               r   |t          | |          } |                                 t          | |          }|                                 }|t          j        t          j        fv rt          | t          j                  } t          | ||          }t          fd|D                       }t          j        ||                                 |                                           }t          j        |t          |                                                    }t          t!          ||          |          S )Nc              3  (   K   | ]}|         V  d S r   r   r   rp   r   s     ra   r   zmean.<locals>.<genexpr>m  '      00a$q'000000rc   rI  )r  r   r  r   r|   ru  rt  rF  sum_rM   r.   r5   rl   r4   r   rg   div)r   rs  keepdimr   output_dtype
sum_resultdenomr   s          @ra   rG  rG  b  s    Q::<<D#At,,D;;==Lu~666Q$$aw''J0000400000Ee1;;==XXXEeT**=*=*?*?%@%@AAEC
E**L999rc   c                  
 |d}|                                  
t          | |          }t          | |d          }|r|                                 t	          t          | |                    }t          |||          }t          
fd|D                       }|rt          j	        ||z
  d          }t          j        ||                                 |                                           }t          j        |t!          |                                                     }t#          ||          }	|s|	fS |r|nt%          ||          }|	|fS )Nr+   T)r'  c              3  (   K   | ]}|         V  d S r   r   r#  s     ra   r   z var_mean_sum_.<locals>.<genexpr>  r$  rc   r   rI  )r   r  rG  r  squarer  r%  rM   r   r  r.   r5   r   rl   r4   r   rg   r&  r   )r   rs  
correctionr'  return_meanx_meandiffsr)  r*  x_varr   s             @ra   var_mean_sum_r3  s  sI   
::<<D#At,,D!T4(((F 3q&>>""EeT7++J0000400000E 1	%*,a00e1;;==XXXEeT**=*=*?*?%@%@AAE
E""E x9VVGFD$9$9F&=rc   c                   t          | |          }t          | ||d d           }|d         }t          |d                   }t          |t          j                  o/t          |          t          j        k     ot          |          dk    S )Nr  ro  r  r+   )	r  r  rM   ri   r   re  r   r,   r  )r   rs  r'  r   ro  reduction_numels         ra   use_two_step_variancer6    s    #At,,D"	wd$  F HF#F+=$>??O?EM22 	'  6#EE	'&!!Q&rc   c               j   dt          | ||d d           }|                    d          }|                    d           |                    d           t          j        j        d|fd|                                 d|\  }}}	|                                 |                                 |                                 t          | |          }t          fd|D                       d	 fd
}
 t          |
          |          }|r|                                 ||fS |fS )Nr+   r  rp  r  r  welford_reduce)r  rD  r   c              3  (   K   | ]}|         V  d S r   r   r#  s     ra   r   z$var_mean_welford_.<locals>.<genexpr>  s'      11q47111111rc   c                    t          | t          j                  r9| j        s2t	          j        t	          j        | t          j                  |          S t	          j	        | |          S r   )
ri   r   r   	is_numberrO   r  r`  r|   ra  rb  rK  s     ra   get_constant_or_index_exprz5var_mean_welford_.<locals>.get_constant_or_index_expr  sS    a$$ 	GQ[ 	G<q%+ > >FFF|Au%%%rc   c                               }           }t          j        d          }| t          j        |||z
            z  S r  )rO   rb  r  )rk   cNzeror.  r   r<  rnumels       ra   r]  z#var_mean_welford_.<locals>.scale_fn  sS    &&z599&&vu55|Au%%ck$A....rc   r   )r  r  r.   WelfordReductionr   r   r  r   r  rM   r}  )r   rs  r.  r'  r/  r   r  rG  m2r   r]  r  r   r<  rA  r   s     `         @@@@ra   var_mean_welford_rD    sz   
"	wd$  F ZZ
##F
JJ{
JJ{%, )'kkmm  	 KD"a JJLLLKKMME::<<D#At,,D1111D11111F& & &
/ / / / / / / / #.
"
"2
&
&C Dy6Mrc   c               8   |                                  t                    }t          | |d          } t          | ||||          }t	          | ||          rt          di |nt          di |}t          fd|D                       }|s|d         n|S )NFr  )r   rs  r.  r'  r/  )rs  r'  c              3  <   K   | ]}t          |d           V  dS )Fr  Nr  )r   r   r  s     ra   r   z#var_mean_helper_.<locals>.<genexpr>  s2      FF!8Ayu555FFFFFFrc   r   r   )r   r   r  r"  r6  r3  rD  r   )	r   rs  r.  r'  r/  compute_dtyper   rA  r  s	           @ra   var_mean_helper_rH    s    I))44MM...A
  F !w???	)(((( 
 FFFFvFFFFFF'36!99V3rc   )r.  r'  c               *    t          | |||d          S )NFrs  r.  r'  r/  rH  r   rs  r.  r'  s       ra   var_rM    s$    	W%   rc   c               *    t          | |||d          S )NTrJ  rK  rL  s       ra   var_meanrO    s$    	W$   rc   c                .   |dk     r$t          t          j        |           | |          S |dk    rt          j        d|          S |dk    r| S t          | |dz  |          }t          j        ||          }|dz  dk    rt          j        ||           }|S )Nr   r+   r)   )pow_recursiverO   r  rb  rr  )r   rB  r   rV  s       ra   rQ  rQ    s    1uuS^A..E:::Avv|Au%%%Avv1a1fe,,FWVV$$F	A!||##Mrc   c                ,    t          j        | |          S r   )rO   powr  r@  s     ra   
pow_nativerU    s    71a==rc   )r   c                X    t          t                    r0t                    k    rt           t                              S t          t                    rdk    rt	                     S t          t                    rdk    rt                     S t          d  fD                       }t          |          }t          t                    odcxk     odk     nc p|odk    }|ri                                  fd}t          j
                                                                          |                                           S t           t                    rL dk    rt          d          S  d	k    r0t!                                                    rt#                    S |rZt           t                    rt%                     S t          t                    rt'                     S t)                     S t+                     S )
Nr\  r+   c              3  r   K   | ]2}t          |t          j                  |                                V  3d S r   )ri   r.   r@   r   r   s     ra   r   zpow.<locals>.<genexpr>  s;      NN1*Q2M2MNNNNNNNrc   i    r   c                \    t           |                                                     S r   )rQ  r   )r"  r  r@  r  s    ra   rv   zpow.<locals>.fn  s%     a???rc   rq  r)   )ri   rF  r   rS  sqrtr  rP  r   r`  r<   r   rl   r   r   r   r
  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorrU  )r  r@  r   is_integer_powembed_exponentrv   r  s   ``    @ra   rS  rS    s0   !U SVV1c!ff~~	Au		 !s((Aww	As		 QQxx NNANNNNNE%e,,N  3'' a"32AF   
	@ 	@ 	@ 	@ 	@ 	@ 	@ <<>>++--::<<	
 
 
 	
 !V 66Q??"66nQ[[]]33677N 4a   	4&q!,,,6"" 	4-a333-a333arc   c                <   t          | t                    r| j        }n| }t          |t                    r|j        }t          |t          j                  st          j        |                                 |                                 |	                                | 
                                          }t          |t          t          f          sJ |j        }t          |t          j                  sJ t          |t          j                  ri|                                sU|                                sAt          |j        t          j                  s"|                                 |j        |_        | S t          j                            |||           | S )Nrq  unsafe_alias)ri   r@   rk   r.   r   r<   r   rl   r   r`  r   r2   r8   is_input_bufferis_module_buffer	NopKernelr  r  realize_into)changedr  rc  changed_datar_   s        ra   r
  r
  7  s   '9%% |#y!! hc2=)) 
.%%''##%%__&&##%%	
 
 
 $: 677777i#r}-----,.. 	$$&&	 ((**	 l'66		 	H!..\ /    Nrc   c                >    t          | t          | |                    S r   )r
  r
  )r   r|  s     ra   r  r  \  s    Q	!Z00111rc   c                    | |u r| S t          ||                                           }t          ||                                           }t	          ||                                           }t          | |          S r   r  rl   r  r   r>  r   r
  )r  r  r  s      ra   r+  r+  a  sg    
czz

C))
*
*C
3
(
(C
cllnn
%
%CS#rc   c                ,    t          j        | |          S r   )rO   floordivrT  s     ra   rn  rn  l      <1rc   c                ,    t          j        | |          S r   )rO   r  rT  s     ra   r  r  q  ro  rc   c                   t          |           ot          |          }t          |           ot          |          }|dk    r;|r
J d            |rt          | |          nt          t	          | |                    S |dk    r;|r
J d            |rt          | |          nt          t	          | |                    S t	          | |          S )Nr  z5floordiv operands can not be boolean at the same timer  z5truncdiv operands can not be boolean at the same time)r   r   rn  r  r&  r  r  )r  r@  rounding_modeboth_integerboth_booleans        ra   div_moderu  v  s    "1%%</!*<*<L"1%%</!*<*<L XX!XXX!-Cx1~~~5Q3C3CCXX!XXX!-Cx1~~~5Q3C3CCq!99rc   c                    t          |           ot          |          }|rt          | |          S t          t          j        j                  } t          |          | |          S r   )r   logical_andr:   r  rr  r  r}  )r  r@  	both_boolrv   s       ra   rr  rr    s`    ""9q'9'9I (1a   *++!~b!!!Q'''rc   r  Optional[ir.Constant]c                L   t          | t          j                  rt          | j                  S t          | t          j                  r!t          |                                           S t          | t          j                  r| S t          | t          j                  sdS t          j
        j                            |                                           }t          j        |          5  t!          j        t          j        dd          5   | j        |                                  }ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   t          |t          j
        j        j                  sJ t          |j        t          j                  r|j        S dS )z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)ri   r.   r8   get_constant_valuerk   r2   rI  r  Loopsr|   	_inductorops_handlerExtractConstantsHandlerrl   rP   set_ops_handlerr   objectr  rp  inner_fn_argsvirtualizedOpsValuer  )r   rE  rn   s      ra   r|  r|    s   
 !R]## *!!&)))!R[!! 3!!--//222!R[!!  a"" to)AA!,,..QQG	'""- -R&(8$??- - aj!//++,	- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - c5?6?@@@@@#)R[)) y4s6   ,!ED6*E6D:	:E=D:	>EEEc                   t          d | |fD                       }|rt          | |          S t          |          x}j|                                 j        dk    rM|j        dk    r(t          j        t          d          |j                  }n
d|j        z  }t          | |          S d } t          |          | |          S )Nc              3  R   K   | ]"}t          |          pt          |          V  #d S r   )r   r   r   s     ra   r   zdiv_prim.<locals>.<genexpr>  s7      OO1oa((>OA,>,>OOOOOOrc   r   r   infr  c                     t          j        |  S r   )rO   ra  r  s    ra   rv   zdiv_prim.<locals>.fn  s    {D!!rc   )r,  r  r|  rl   r   r  mathcopysignrF  rr  r}  )r  r@  is_integralr  r  rv   s         ra   div_primr    s    OOAOOOOOK 1~~ &a(((5!,,..:MQV:V:V=AuU||W]CCJJw},J1j!!!" " " >"a###rc   c                b    t          | |ft          j                  \  } }t          | |          S rj  )rU  r   INT_TO_FLOATr  rT  s     ra   r&  r&    s9     	
A$C$P  DAq Aq>>rc   c                    t          |           pt          |           }|rd }nd } t          |          | |          S )Nc                ,    t          j        | |          S r   )rO   modrT  s     ra   rv   zfmod.<locals>.fn  s    71a== rc   c                ,    t          j        | |          S r   )rO   fmodrT  s     ra   rv   zfmod.<locals>.fn  s    8Aq>>!rc   )r   r   r}  )r  r@  r  rv   s       ra   r  r    sc    !!$$:(:(:K "	! 	! 	! 	!
	" 	" 	" >"a###rc   c                   t          |                                           s!t          |                                           r|t          j        }t          d|          } || |||          S )Nr9  r  r   r   r   r   r|   ra  r  r   rs  r  r   rv   s        ra   r%  r%    sk     	''+;AKKMM+J+J
-	U	;	;	;B2axu----rc   c                   t          |                                           s!t          |                                           r|t          j        }t          |                                           dk    r.|dv sJ |p|                                 }t          | |d          S d }t          | ||          }t          j
        j        d	i |d|i\  }|t          | ||          S |S )
Nr   r   r]  Tr  c                >    | \  }|\  }t          j        ||          fS r   )rO   r   a_tupleb_tupler  r@  s       ra   
combine_fnzcumsum.<locals>.combine_fn  #    1rc   rs  r   r  r   r   r   )r   r   r   r|   ra  r   r   r  r   r.   Scanr   fallback_cumsumr   rs  r   r  r   rV  s         ra   cumsumr    s     	''+;AKKMM+J+J
-
1::<<Aw&5t,,,,     
 ad%888F????J???IV~qd%8888Mrc   c                   t          |                                           s!t          |                                           r|t          j        }t          |                                           dk    r.|dv sJ |p|                                 }t          | |d          S d }t          | ||          }t          j
        j        d	i |d|i\  }|t          | ||          S |S )
Nr   r  Tr  c                >    | \  }|\  }t          j        ||          fS r   )rO   rr  r  s       ra   r  zcumprod.<locals>.combine_fn  r  rc   r  r  r  r   )r   r   r   r|   ra  r   r   r  r   r.   r  r   fallback_cumprodr  s         ra   cumprodr    s     	''+;AKKMM+J+J
-
1::<<Aw&5t,,,,     
 ad%888F????J???IV~t59999Mrc   c                *   d }|                                  }t          |                                           dk    r|dv sJ t          |           S t	          | ||          }t          j        j        di |d|i\  }|t          | |          S |S )Nc           	         | \  }|\  }t          j        ||          }t          j        ||          }||k    t          j        |           z  }t          j        |t          j        t          j        ||z
                      |z   |          fS r   )rO   r  r  r  r  log1pexp)r  r  r  r@  min_vmax_vrl  s          ra   log_add_exp_helperz(logcumsumexp.<locals>.log_add_exp_helper(  s    Aq!!Aq!!CIe$4$4#45	$	#'%%-*@*@ A AE I1MMOOrc   r   r  r  r  r  r   )	r   r   r   r  r   r.   r  r   fallback_logcumsumexp)r   r   r  r   r   rV  s         ra   logcumsumexpr  &  s    P P P KKMME
1::<<Ag~~~~Qxxac777FGGGG4FGGGIV~$QC0000Mrc   c                   t          |                                           dk    r0dv sJ t          |           t          | t          j                  fS |                                 }t          j        d|d          }t          | |          }|t          j        f|d<   | 
                                fd	f|d
<   t          j        j        di |d|i\  }}|t          |           S ||fS )Nr   r  r   r  Fr   arg_break_ties_leftr  r  c                N    t          j        |          t          j                  S r   rO   r`  r|   ra  r"  rs  s    ra   r^  zcummax.<locals>.<lambda>K      CN3t9ek:: rc   r  r  r  r   )r   r   r  r  r|   ra  r   r.   get_reduction_combine_fnr   r`  r  r   fallback_cummaxr   rs  r   r  r   r  r  s    `     ra   cummaxr  <  
   
1::<<AwQxxAU[99999KKMME,5  J ad%888Fu{+F8	::::F; gnEEvEE*EEEOFG~qd++++7?rc   c                   t          |                                           dk    r0dv sJ t          |           t          | t          j                  fS |                                 }t          j        d|d          }t          | |          }|t          j        f|d<   | 
                                fd	f|d
<   t          j        j        di |d|i\  }}|t          |           S ||fS )Nr   r  r   argminFr  r  r  c                N    t          j        |          t          j                  S r   r  r  s    ra   r^  zcummin.<locals>.<lambda>b  r  rc   r  r  r  r   )r   r   r  r  r|   ra  r   r.   r  r   r`  r  r   fallback_cumminr  s    `     ra   cumminr  S  r  rc   c                   t          |                                           s!t          |                                           r|t          j        }t          d|          } || |||          S )Nr:  r  r   r  r  s        ra   r:  r:  j  sk     	''+;AKKMM+J+J
-	e	<	<	<B2axu----rc   c                l    t          | t          j                  }  t          d          | ||          S )Nr-  rs  r  )r  r|   r   r  r   r   r'  s      ra   
reduce_anyr  u  s2    EJA >%  w????rc   c                r    |$t          | ||          t          | ||          fS t          | d |          S Nr  )reduce_amaxreduce_argmaxr  s      ra   
reduce_maxr  {  L    
g666!#888
 	

 qtg6666rc   c                r    |$t          | ||          t          | ||          fS t          | d |          S r  )reduce_aminreduce_argminr  s      ra   
reduce_minr    r  rc   xor_sumr)  r(  r  r  r  
logical_or)ry  rn  stabler   
descendingc          	        |d}|                                  }|                                 }t          t          |          |          }t          |          dk    r+t	          |           t          d|t          j        |          fS t          |          r||         nd}t          j	        j
                            |t          j        t          j                  j                  st          | |||          S t!          |ddt          j        |d          }dgt          |          z  }t          |          r|||<   t#          ||          }t%          ||          }t&          j                            || j        |j        f|                                 |                                f||||          \  }	}|	t          | |||          S |J |	t1          |t          j                  fS )NFr   r+   r  )r>  r@  r   rs   rF  )rs   r  r  r   rs  r  r  )r   rl   r   r   r  r}  r|   ra  rP   rY   r   statically_known_ltr  int16r)  sort_fallbackrG  r  r>  r.   Sortr   r   r`  r  )
r   r  r   r  r  rs   r/  r  
view_shaper  s
             ra   sort_stabler    s   ~JJLLE\\^^F
3u::s
+
+C
5zzQQxxq&%+u==== ZZ.uSzzQH7//%+ek:R:R:VWW OQv3:NNNNVSX  G s5zz!J
5zz #"
37J''GWe$$Ggnn'==??G$7$7$9$9: %  OFG ~Qv3:NNNN8GU[1111rc   c                (    t          | d||          S )NFr  )r  )r   r   r  s      ra   sortr    s    qCJGGGGrc   c                <    t          | |t          j        |          S )Nr   r   r{  )r  r   r  )r\   r   r{  s      ra   register_pointwise_numericr    s(    
;H'	   rc   r\   torch._ops.OpOverloadPacketc                `    t          | j                   t          | t          j                  S rj  )rL   r  r  r   r  r  s    ra    register_pointwise_numeric_ldf64r    s2    '444
;H   rc   r  logical_not)rn  )r   r   rS  identity)r  pointwise_overrides_datac              #  H  K   t           |         t          | j        d           }|d S fd}t          |t          j        j                  r>|                                D ]'}t          ||          }|j         ||          fV  (d S |j         ||          fV  d S )Nc                4    j         t          |           S d S r   )tritonr  )r\   rk   s    ra   make_triton_fallbackz6_get_pointwise_overrides.<locals>.make_triton_fallbackE  s!    ;#B''' rc   )	r  r   r   ri   r|   r}   r   r   r   )nsr   r\   r  olnamer{  rk   s         @ra   _get_pointwise_overridesr  ?  s      #D)D	TY	%	%B	z( ( ( ( ( "ej122 Ellnn 	I 	IFV$$Bd.0D0DR0H0HHHHHH	I 	I $*,@,@,D,DDDDDDDrc   r  c                z    | t           |<   t                              |            fd}t          | |           d S )Nc                      | i |}g }t          | d         |          D ]*\  }}|                    t          ||d                     +|S )Nr   Trb  )r  rm   r
  )rq   r   resultsmut_resultsr   rV  outplace_ops         ra   rv   z$register_foreach_inplace.<locals>.fn  sk    +t.v..tAw00 	J 	JKCyf4HHHIIIIrc   )rV   r  r   r%  )aten_opoutplace_aten_opr  rv   s     ` ra   register_foreach_inplacer    sT    07,-G$$$     w+++++rc   c                @    t          | d           fd            }|S )Nr  c                      | i |}t          || d                                                   }t          | d         |          S r  )r  r   r
  )rq   r   rV  r  s      ra   rv   zregister_inplace.<locals>.fn  sJ    d-f--&$q'"3"3"5"566a&)))rc   )r5  )r  r  rv   s    ` ra   register_inplacer    s;    wD999* * * * :9*
 Irc   c                    d S r   r   r'  s      ra   sym_constrain_ranger        4rc   c                    t           j        j        j        d         }t	          |t
          j                  sJ d|             |j        j        S Nr  z*Expect val to be torch.SymInt but got val=	rP   rY   rZ   rv  ri   r|   r  r_   r  r  r   r  s      ra   sym_sizer    sS    
'

#E
*C c5<((  :S:: ( 8=rc   c                    t           j        j        j        d         }t	          |t
          j                  sJ d|             |j        j        S r   r  r  s      ra   
sym_strider    sQ    
'

#E
*Cc5<((  :S:: ( 8=rc   c                *    |                                  S r   )r  )r  s    ra   	sym_numelr    s    ;;==rc   c                    t          j        |  S r   )r   Addr  s    ra   sym_sumr
    s    9drc   c                     t          d          )NzHelpful for debuggingr   )r  rq   r   s      ra   foobarr    s    
5
6
66rc   c                H    |                                   t          |           S r   )r  r  r   s    ra   _realizer    s    IIKKK88Orc   c                X    |                                   t          j        | |           | S r   )r  r.   ResizeStorageBytes)variabler0  s     ra   resize_storage_bytes_r    s,    (H---Orc   c                    |                                   |                                  t          j        t          j        | |                    S r   )r  r@   r   r.   SetSourceTensorKernel)r  source_tensors     ra   set__source_tensorr    s?    LLNNNB4T=IIJJJrc   r+  c                    | |u r| S t          ||                                           }t          ||                                           }t	          ||                                           }t          | |          S r   rl  )r  r  s     ra   
fsdp_copy_r  (  sg    #::JS^^--..sCMMOO,,S#,,..))c"""rc   c                 	
 t          | t                    sJ t          |t          t          f          sJ |t          j        }|t          j        k    rt          d|           |t          j        k    rt          |          dk    sJ |t          j
        k    rt          |          dk    sJ |                                 
|                                 }|                                 }t          | j        t          j                  r| j                                        | _        t	          j                    rat          j        j        j        rKt-          |          rt/          d          n.t1          |          rt	          j        |          j        ndndt6          j        j                            
d          rt?          |||          S tA          | 
gd	g          }|!                                	t          j"        #                    ||          }t          j$        ||||          %                                	
fd
}tM          j'        |||t          |                    }|S )Nzunsupported memory format: r   r   nanTr_  r   r'  r+   c                     |           t          j        t          j                  }t          j        t          j                  }t          j        ||          }t          j        |fd          S )Nc                       g          S r   r   )
flat_indexflat_loaders   ra   r^  z*resize.<locals>.inner_fn.<locals>.<lambda>l  s    ZL(A(A rc   )rO   r`  r|   ra  rd  rf  )	r"  flat_index_exprlimitrl  r  r  	old_numelout_indexeruninitialized_vals	       @ra   rp  zresize.<locals>.inner_fng  sl     [%%
.U[AAy%+66vou--z$ A A A A ACTUUUrc   rq  )(ri   r@   rg   r   r|   contiguous_formatpreserve_formatry  channels_lastr   channels_last_3dr  r   rQ  rk   r.   r2   rI  r  utilsdeterministicfill_uninitialized_memoryr   rF  r   r  r)  rP   rY   r   r0  r  rR  r`  r   stride_ordered_for_memory_formatrL  r  r<   r   )r   r   r<  r   rs   x_flat
out_striderp  rn   r  r!  r"  r#  s            @@@@ra   resizer.  3  s   a#####dT5M*****/---HHHIII+++4yyA~~~~...4yyA~~~~IKKMME""$$F!&"+&& &##%% 	244 K%?  %   	% %ee$$ 	% %E 2 2 6 $  w//	1== ID+5HHHH		
 	
 F $$&&K"CCD-XXJ.jAANNPPKV V V V V V V V 
UXd4jj  C Jrc   )auto_functionalizedc                    ddl m} |                    |          }t          j        | ||i ||           d |                                D             S )Nr   )kernel_side_table)
kernel_idxgridtma_descriptor_metadatakernel_argsc                D    i | ]\  }}t          |t                    ||S r   r   )r   r  r  s      ra   r  z'triton_kernel_wrap_.<locals>.<dictcomp>  s-    RRRcz#y7Q7QRCRRRrc   )*torch._higher_order_ops.triton_kernel_wrapr1  get_constant_argsr.   UserDefinedTritonKernelr  )r2  constant_args_idxr3  r4  r   r1  constant_argss          ra   triton_kernel_wrap_r<  z  sz     MLLLLL%778IJJM 7/v//	    SRV\\^^RRRRrc   c                X   t          d | g|D                       rFd}t          j        j        j                            dd           x}r| d| }|t          j        _        t          j        	                    | |||          }t          t          t          j	        |                    S )Nc              3  ^   K   | ](}t          |t                    ot          |          V  )d S r   r[  r   s     ra   r   zcond.<locals>.<genexpr>  s7      
M
Ma:a  1Yq\\
M
M
M
M
M
Mrc   z"control flow operator: torch.cond.stack_trace Found from : 
 )r-  rP   rY   rZ   rv  rw  disable_cudagraphs_reasonr.   Conditionalr   rg   mapr@   )predtrue_fnfalse_fnoperandsr   r?  rV  s          ra   r   r     s    

M
MD;L8;L
M
M
MMM 02'.377tLLL; 	988;88C,/)^""4(HEEFI$f--...rc   c                |   t          d ||z   D                       rFd}t          j        j        j                            dd           x}r| d| }|t          j        _        dd}t          j        	                    | ||||          }t          |t                    sJ t          t          ||                    S )	Nc              3  ^   K   | ](}t          |t                    ot          |          V  )d S r   r[  r   s     ra   r   zwhile_loop.<locals>.<genexpr>  sK         	1f.)A,,     rc   z(control flow operator: torch.while_loop.r?  r@  rn   r   c                   t          | t                    r| S t          | t          j                  rt          |           S t          | t          j                  rt          j        |           S t          dt          |                      )NzNYI unsupported output type: )ri   r@   r.   r   MultiOutputr   ry  r   )rn   s    ra   _map_outputzwhile_loop.<locals>._map_output  s~    c9%% 	LJR]++ 	LS>>!R^,, 	L#C(((JtCyyJJKKKrc   )rn   r   )r-  rP   rY   rZ   rv  rw  rA  r.   	WhileLoopr   ri   r   rg   rC  )	cond_fnbody_fncarried_inputsadditional_inputsstack_outputr   r?  rL  rV  s	            ra   
while_looprS    s    
  "33     0 9'.377tLLL; 	988;88C,/)L L L L \  .*;\ F fh'''''K(()))rc   )rR  subgraph_fnir.Subgraph
identifierc                ~    t          j        j        | g|R  }t          t	          t
          j        |                    S r   )r.   InvokeSubgraphr   rg   rC  r@   )rT  rV  rG  rV  s       ra   invoke_subgraphrY    s8    %k=H===FI$f--...rc   )schemec                  d }t           j        j        j                            dd           }|J t          | j        j        j                  D ]o\  }}|j        dk    r||         t           j        j	        |<   ,|j        dk    rt           j        
                    |          \  }}t          j        ||                                          D ]}	|	                                 |j        r6t           j        j                            |	                                           t           j        j                            |	                                           t(          j        j                            t           j        |||          }>t           j                            |          t           j        j	        |<   q|S )Nquant_optionsrX  rA  )rP   rY   rZ   rv  rw  rh   r  nodesr\   envfetch_args_kwargs_from_envr  r  r  r  codegen_low_precisionlow_precision_codegen_opsr   r  invoke_quant_opsr|   rZ  InterpreterrA  run_node)
rT  rZ  rG  rA  r\  rp   r_   rq   r   r   s
             ra   invoke_quant_tracerre    so   FG(-11/4HHM$$$[5;ABB 7 747m## (AGKW  7==dCCLD&_T6==??;; E E		 6 RG599!:N:N:P:PQQQ(,,Q-A-A-C-CDDDDX)00$fMMFF ! 0 0 6 6AGKMrc   r  rQ  tuple[torch.Tensor]c                  	 ddl mm} t          |          dk    rt	          d          fdt          j        ||          D             } || |          		fd}t          |d         dd           }t          d |D                       |d	<   t          d
 |D                       |d<   t          j
        j        d|dd|}|d         t	          d          |S )Nr+   )InputDescriptorlower_pointwise_subgraphr   zSUnable to generate code for associative_scan op, because there are lifted argumentsc                r    g | ]3} |                                 |                                           4S )r'  )r   rl   )r   r   rh  s     ra   r   z$associative_scan.<locals>.<listcomp>  sH        	akkmmALLNNCCC  rc   c                `     g t          j        |           t          j        |          R  S r   )rA  r\  )lhsrhslowered_combine_fns     ra   wrapped_combine_fnz,associative_scan.<locals>.wrapped_combine_fn  sC    !! 
$$
$$
 
 
 	
rc   r  c              3  >   K   | ]}|                                 V  d S r   r  r   s     ra   r   z#associative_scan.<locals>.<genexpr>  s*      77qQ[[]]777777rc   r  c              3  >   K   | ]}|                                 V  d S r   r_  r   s     ra   r   z#associative_scan.<locals>.<genexpr>  s*      <<A<<<<<<rc   r  F)r  can_fallback_to_atenz/Unable to generate code for associative_scan opr   )r  rh  ri  r   ry  r  r  r   r   r.   r  r   )
r  xsrQ  ri  subgraph_inputsro  r   rV  rh  rn  s
           @@ra   associative_scanru    sS    MLLLLLLL
!!a
 
 	
   R((  O 21*oNN
 
 
 
 
 be!4888F77B77777F8<<<<<<<F;W^ %"   F
 ayLMMMMrc   c                    d S r   r   )tokenss    ra   _sink_tokensrx    r  rc   c                .   t          j        j        |g|R i |}ddlm}  ||||          }|J t
          j        j        |         }||fS t          j	        t           j
        t          j        |          }t          |t                    s||fS |g|R S )Nr   )get_effect_key)r.   EffectfulKernelr   torch._higher_order_ops.effectsrz  rP   rY   effectful_opsrA  r  rK  r@   ri   r   )tokenr\   rq   r   rV  rz  effect_typeeffectful_kernels           ra   with_effectsr    s    &r;D;;;F;;F>>>>>> .T622K"""w,[9~ ""!".)2BFKKF fe$$ + &)) *6***rc   )register_comm_loweringsc                v   t          | |ddd          }|d         }t          j        j                            t          |                    }t          j        j        di |d|d\  }}|dk    rNt          j        j        	                    |          t          j        k    rt          j        d| d|d	|\  }}||fS t          j        t!          j        d
                     t%          | |d          }	t'          t(          j                 t-          | |	                    }
t/          |
|d          }|	|fS )zn
    Lowering inductor_prims.prepare_softmax_online to compute max/sum in one pass if no split is needed.
    TNr  r  online_softmax_reduce)rD  r5  r+   r)   )r  
num_outputreduction_hintz
            Online softmax is disabled on the fly since Inductor decides to
            split the reduction. Cut an issue to PyTorch if this is an
            important use case and you want to speed it up with online
            softmax.
            )r  r   )r  rP   rY   r   simplifyrM   r.   r=   
num_splitsr&  r,   r  r9   r   rH  rI  textwrapdedentr  rT   r  r  r  r%  )r   r   r   r  rA  hint	num_split
max_tensor
sum_tensorrH  r  xsums               ra   prepare_softmax_onliner  $  so   
 #	d$d  F 01W&&}5E'F'FGGFl-  
.   OD) 	QG&&v..&2TTT!7!> "
Qt"
 "
?E"
 "

J :%% 	O 		
 		
 		
 1cD111!#a,,//Ct,,,Tzrc   )rD  )quantized_lowerings)mkldnn_lowerings)jagged_loweringsc              #    K   t          | t          j        j                  s
J d            t                              |           }	  t          |           t          |                      dV  |r|t          | <   dS t                              |            dS # |r|t          | <   w t                              |            w xY w)z^
    A context manager to force fallback an op. Used in unit test
    for FallbackKernel.
    z+Only OpOverload to make the clean up easierN)	ri   r|   r}   r~   rT   rw  r5  r  r  )r\   old_handlers     ra   force_fallbackr  q  s       b%*/00  5 0 --##K".r22333 	'IbMMMMM"  	'IbMMMM"s   )B )C)rd   re   )rv   rw   rx   ry   )r   r   )r   r   )r   r@   rs   r   rx   r@   )rq   r   r   r   r   r   r   r   r   r   rx   r   )r   r   )r   r   rx   r2  )NN)NNNFN)F)r   r  r   r  r  r   )r   r@   r   r  )r   r@   rs   r   r   )r   r@   r  r4  rx   r@   )r   r   r8  r+   Tr  )r  r@   rq  r@   rr  r@   rs  r   rt  r   ru  r   r   r  rx   r  )r  r@   rq  r@   rr  r@   rs  r   rt  r   ru  r   r   r  r  r  rx   r  )r  r@   r  rF  r  r   rt  r   ru  r   r   r  rx   r  )r  r@   r  rF  r  r   rt  r   ru  r   r   r  r  r  rx   r  )r  r@   r  r@   r  r@   rt  r   ru  r   r   r  rx   r  )r  r@   r  r@   r  r@   rt  r   ru  r   r   r  r  r  rx   r  )r   r   r+   )r  r   r  r   r  r   r9  )T)rt   rK  )r_   rV  )NTF)rs   r   )r   r  r  r@   r  r  r  r   )
r  r   r  r   r   r  r  r@   r  r   )r  r@   rx   r  )r  r@   rx   r  )r  r@   r  r@   r  r   r  r   r  r  r  r  rx   r  )r  r@   r  r@   r  r   r  r   )r   r   rJ  r   )r   NNr+   )NNN)r]  FF)r   r   )rk  r  r   r   r  r  r1  r   )r   r   r  r  )r   r   r1  r   )r)   F)rS  rT  rU  r   rV  r   )rq  rf  )ro  rf  rp  rf  )rv  rf  ro  rf  rp  rf  )r   r@   r  r  r|  rF  rx   r  )rp   r  r  r  )rp   r  r  r  )rp   r  r  r  r  r  )Nr  N)
r  r@   r  r  r  r  r  r  rx   r  )Nr   r+   F)r_  )NNNN)r   r   FTN)rD  rQ   r   )r   r  rx   ry  )r]  F)r\   r  )rT  rU  rV  r  )rT  rU  )r  rU  rQ  rf  r  (  
__future__r   
contextlibr  r   r  loggingr  r  rn  r  rH  collectionsr   collections.abcr   r   typingr   r   r	   r
   r   r   r   typing_extensionsr   unittest.mockr   r   r|   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreer(  _pytreerA  torch._dynamo.utilsr   (torch._higher_order_ops.associative_scanr   r7  r   torch._library.utilsr   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r    %torch.fx.experimental.symbolic_shapesr!   r"   r#   torch.utils._ordered_setr$   torch.utils._sympy.functionsr%   r&   r'   r(   _dynamo.utilsr*    r,   r-   r.   r/   decompositionr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   r  rO   rP   r  rQ   rR   rS   FALLBACK_ALLOW_LIST	getLoggerr  rv  rT   __annotations__rU   r}   r~   r*  r  tr_c10dr  r   _higher_order_opsr  r^   r  rV   quantized_decomposedrb   ru   r   r   r   r   r   rR  rW  rd  r  bmmconvolutionconvolution_backwardr  r  r,  r  rs  ru  _int_mmr  r  r  r  ra  ru  r~  r~  	complex32	complex64r   rt  r   r   r   r   r   r   r   r   r  r%  r1  rO  r5  rC  rU  r}  r  r  r  r  r  r  r   r  r  
device_putr  r  r  r  r  r  aliasdetachdetach_liftview_ofr  r   r  r   r  r  r  r  r  r  r  r  r  r>  r#  r%  r3  _unsafe_viewreshaper7  slicerA  rT  rp  quantize_per_channelr  r  r   r  _functional_assert_asyncr  dequantize_per_channelr  quantize_per_tensorr  dequantize_per_tensorr  r  r  r  r  r  r  r  r  r"  r$  r'  r2  r  r6  r<  r<  r  cacherJ  rP  rU  rf  r|  r  rngprimsr  r  r  	bernoullir  r  r  	lru_cacher  r  r  r  r  r  r  r  r  randintforce_stride_orderr  r  r  r  r  lookup_seedr  randomr  r  r  r  r  r  r  	NO_OPMATHr  r  r   r  r
  r   r   r:  _adaptive_avg_pool3dadaptive_max_pool3d*_scaled_dot_product_attention_math_for_mpsuniformexponential_pdist_forwardsoft_margin_loss_backward_fused_rms_normxpuis_availableembedding_dense_backward_cdist_forward_cdist_backward
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation_grouped_mm
_cudnn_rnn_cudnn_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backwardr  r  kthvaluetopkr  median	nanmedianr  resize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_stategraphsafe_run_with_rng_statemasked_scattermasked_scatter_backwardrb  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   r  r+  r)  #_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward0_scaled_dot_product_fused_attention_overrideable9_scaled_dot_product_fused_attention_overrideable_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducerepeat_interleaver  r  rA  rc  rG  rL  rT  rV  scalar_tensorrh  
LongTensorrj  rq  rs  ru  r}  r
  r  r/  r  r  r  rf  
zeros_liker  r  r  r  r  r  r  r  r  r  r  r  rJ  r  r  r  r  r  r  r  r  r   fallback__unsafe_masked_indexr%  ,fallback__unsafe_masked_index_put_accumulater;  r-  r  r5  r/  rA  r?  rE  r<  re  rk  rn  ry  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  r9  rC  rQ  rW  rk  re  r{  ry  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r   rG  r3  r6  rD  rH  r  rM  rO  rQ  rU  rS  Tensor_Tensorr^  Scalarr\  Tensor_Scalarr]  r
  r  r+  rn  r  r&  ru  rr  r|  r  true_divider  r  r9  r%  r  r  r  r  r  r  r  r  r  r  r:  r-  r  r)  r  r(  r  r  rH  r  rI  r  r  r  r  r  r   r  r  r  r  rsqrtr  r[  expm1relur:  rZ  r-  r  cossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfr  tantanhrw  r  r  logical_xorr  r  	clamp_min	clamp_maxnegr  	remaindersignsignbit	_neg_viewlerd  rc  rT  rJ  necoshsinhacosacoshasinasinhatan2atanatanhr  erfcerfinvhypotlog10log2	nextaftercodegen.commonr  r  r  r   r\   r   r{  _foreach_addListforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_rsqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyforeach_copyr  _foreach_add__foreach_mul__foreach_div__foreach_copy_r  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r  r  r   r  r  r  methodfuncr
  r  _inductor_testr  r  r  r  set_source_Tensorr  fsdpr  r.  *torch._higher_order_ops.auto_functionalizer/  r<  higher_orderr   rS  while_loop_stack_outputr4  rY  invoke_quantre  ru  rx  r  comm_loweringr  r  rD  r  register_quantized_opsregister_woq_mm_opsr  register_onednn_fusion_opsr  register_jagged_opscontextmanagerr  r   rc   ra   <module>r	     sM   " " " " " " "                    				   # # # # # # . . . . . . . . O O O O O O O O O O O O O O O O O O ' ' ' ' ' '         + + + +  $ $ $ $ $ $ $ $ $ ( ( ( ( ( ( H H H H H H U U U U U U : : : : : :                          M L L L L L L L         
 0 / / / / / U U U U U U U U U U U U , , , , , , 8 8 8 8 8 8 8 8 8 8 8 8 = = = = = = = =                                     &                                      +****** WT]]Yt__ !j    g!!FH	 H H H H       .Juz,-//	y~
)
	9
5:#89;; /j./
)* 
 8j!6799 NP  P P P Py5      ,   	B 	B 	BF F F

 
 
3 3 3   !$$-&  , {z{{{}}}
 .   " " "# # #           (O O O Od  24 4 4 4r  	(/    ,# # #,0 0 0 0j  $S S S Sl7 7 7 7v RW
E 
E 
E 
E 
E 5*7TRRR) ) SR)X 5-4HHH) ) ) IH) @E 5 5 5 5 5 5( 49?===& & & >=& ;@e K K K K K K 5#>>>F F F F ?>F 
7? $* * * *Z  >         4:DIII  JI0 4)UPTUUU  VU( DJT\49emTUU  VU 74 ,&do&&s+++ 4<T:::B B B ;:B, 4$$???" " " @?" DM?##   $# 4:C C C 4:C C C 49! ! ! 4:! ! ! 4:%&&% % '&% 4:! ! ! 4;D999> > :9>4 5)tDDD
 
 ED
 4>t<<<# # =<# 4;1 1  1h 4$$???49$7774<T:::1 1 1 ;: 87 @?1 4<T:::> > ;:> 4:4888V V V 98V 4?===J J J >=J$ 4#>>>   ?> 4'TBBB   CB
B B B BJ '<RVWWW/ / / XW/d  $ 4%)**$ $ +*$ 40455$ $ 65$ /T   (,/ / / / / /d ,4$  ! ! ! !H .6D   (,! ! ! ! ! !H ,3  * * * *Z .54   (,, , , , , ,^ 48y: y: y: y:x 4=d;;;3E 3E 3E 3E <;3El 4%4@@@6 6 6 6 A@6 4(dCCC    DC 4;D9996C 6C :96Cr 4:4888   984 4(dCCC      DC  4;D999   :9 4;D999D D :9D8 4>t<<<  =< 4?===  >=    48      $       >
3 
3 
3 
3 
3%6 %6 %6 %6 %6P/N /N /N /Nd	, 	, 	, 59%1tLLL#+ #+ ML#+L 4&DAAAG G BAG 4?===  >= 4>#>>>' ' ?>' 4<    !  QB B B/ / /
 )():;; **49+>?? ))$**<== ++DJ,@AA  dl    49J J J 4:J J J >4$OOOL L POL
 >&DAAAK K K BAK >'TBBBJ J CBJ
 >-4HHH	 	 IH	 >(dCCCRS      DC6 >)tDDDLM     ED8   . . . . 4$+FFF
 "&^ ^ ^ ^ ^ GF^B N(G(Q   7 7 7 7 7 7t          "    $D D DP d' ( ( ( d& ' ' ' d= > > > dl ' ' ' ' d&U 3 3 3 3 d! " " " d,5 9 9 9 9 d" / / / /9 M%E    d! " " " d" # # # do    d!) * * * d+3 4 4 4 dj    dn# $ $ $ d)1 2 2 2 d,4 5 5 5 dk    d$5 1 1 1 1 d . . . d')@ A A A do} - - - d&(: ; ; ; d!#5 6 6 6 d.0B C C C d* + + + d= > > > d= > > > d2 3 3 3 d= > > > d3 4 4 4 d0- @ @ @ d0 1 1 1 d/ 0 0 0 d/ 0 0 0 d1 2 2 2 d1 2 2 2 d- . . . d- . . . d- . . . d.0B C C C d0 1 1 1 d+] ; ; ; d" # # # di    di    dm    di    di    dk    dn    dm    dl    do    d    d- . . . d  ! ! ! d' ( ( ( d# $ $ $ dn    d& ' ' ' d" # # # d$ % % % dn    d" # # # d# $ $ $ d* + + + d    dn    dj    d' ( ( ( d/ 0 0 0 d       d# $ $ $ d% & & & d# $ $ $ d! " " " dj    dm    dl" # # # dhU + + + + d(- 8 8 8 el$; < < < el$7 8 8 8 el$A B B B
 d! " " " d* + + + d"$6 7 7 7 dj    d' ( ( ( d; < < < dn    do    dj    08	   
 9A	   
 ,4	   
 5=	   
 ,4	   
 5=	   
 4<	   
 =E	   
 9A	   
 BJ	   
 d+3_ E E E d,4o F F F d/7 I I I d08/ J J J d       d$+T B B B B
 49$777
 
 
 87
 4:"       ( 74"## 3+d*++E222 5:  ( 4&DAAA   BA: 4%4@@@@ @ @ A@@F   EL$"4566tDU 5 5 5 5 765p 5?##4 4 4 $#4 5#$$+ + %$+ 4+,,)" )" -,)"X 4&''  (' 4/00%)59$    10  <K K K  8 EK,-- 
    .-$  .> > > 0t//0B0B50I0IJJ
11!4455	 2 21 5 566
   4>"" $T$4     #" 4%&&Td    '&> 4)**"4    +*" 5%-..A A /.A EJ	*++: : ,+:
 4;D999! ! ! :9!H 4>t<<<   =<B# # #F F F F FR  "1 "1 "1 "1J 4:4888
 
 98
 4%4@@@/ / A@/ 4>t<<<   =< 4)**   +*; ; ;   4?===   >= >4$OOO   PO} } } }@ !1 0%5! ! !  0@/?,4%0 0 0 ,
 4,$GGG  HG, 4;QUVVV	P 	P WV	P 1 1 1 4*EEE   FE 4<T:::9 9 9 ;:9 !     B 4=d;;;DH : : : : : <;:" 4#>>>3 3 3 ?>3 4$$???6 6 6 @?6 4&DAAAP P P BAP 4'TBBBPT p p p p p CBpn , , , , ,^ 4*233> > > > 43> 419::J J J J ;:J 4*233RVI I I I 43I 419::RVU U U U ;:U 4*233 !% $ $S S S S 43S 419:: !% $ $	 	 	 	 ;:	7 7 7 59$%%  &%*\ \ \ \~ 4'TBBB4 4 4 CB4n          :>   < MQ     <   JN@ @ @ @ @:H H HV 5:PTUUU 5 5 5 VU5D   > 	1t    $  < 4/TJJJ 
 
 
 KJ
 4/TJJJ 
 
 
 KJ
 -=,<)1- - - ) 48dSSSN N TSNb   8B B B& & &R2 2 2j  0/%5      
 4,--B B .-BJ  0/$%      
 4+,,N N -,Nb) ) )X 4-..V V /.V 4-..V V /.VA A AH 43;<<BF* * * =<*Z '&O    '&O   
 4?===    >=* 4?===    >=*w w wt  0/$%      
 4+FFF b b b GFbJ  0/$%      
 4+FFF D D D GFDN  $. . .b    (   49:t : : : : :   4   ( ( (V4 4 4( DHei())T5     *) 4=!!u     "!      .-H    '&txERRR --H   
 48t,,,, , -,,^" " " "J 4:2 2 2 4:4888   98       48t,,,   -, DH:...( ( /.(   @ EI;$///$ $ 0/$, 	tx'7D  
  
 DIuz*d;;;$ $ <;$  DHei()). . . . . *). #"4;#677##DL$899 (():)BCC ""4;#677""4;#677 4;    . 4<     ! . 4$%%  &%* 4;D999   :9, 4;D999   :9, 49. . . . . . 48@ @ @ @
 486667 7 7 767 486667 7 7 767 !  %-    	!:!: ; ; ;*	**>>%+@+@AA*	**>>%+@+@AA.!!$+..N85;???  /!!$+..N85;???  H$L   ! !1uMMM 49#>>>!re $2 $2 $2 $2 ?>$2N 49$$???H H H @?H       	#"4:..&&tx00!!$),,""4:..$)$$
*
*4<
8
8''	22		DK	(	(t444     * * *     * * *""  !122''(?@@   -    00
(()ABB   !122  4; ' ' '  **  *I*V      4: & & &  48 $ $ $  49 % % %     * * *  *	   ! *	    O*	  
 ! *	   
T\
*
*

T\
*
* !  $. ! !' * * * !  $. ! !' * * *""""''88
  4> " " "$)LLL  49     4<uz B B B B !  $. ! !# & & &  47%* = = = =  47%* = = = =  47%* = = = =uzBBB  47%* = = = =  47%* = = = =  49 % % %  49 % % %  49 % % %  4: & & &  49 % % %  4: & & &  4: & & &  49 % % %  4: & & &  4= ) ) )  49 % % %  4; ' ' '  4: & & &  4: & & &  49 % % %  4> * * * D D D D D D D DE E E$ % 
 
D4L4Ld5 5 
 
0 	 3+		
 	
 	
 	
 	
 5M4Lt5 5 
 
0 	 3+		
 	
 	
 	
 	

 .-CT    0/ct     4,3Sd K K K K--d.?.DcJJ   4,3S 9 9 9//0A0H#NN   4,13 7 7 7  4,3S 9 9 9  4,4c : : :  4,4c : : :  4,3S 9 9 9  4,13 7 7 7  4,<c B B B--d.?.DcJJ   4,3S 9 9 9//0A0H#NN   4-t 4 4 4  4. 6 6 6  405w ? ? ?  407 A A A  405w ? ? ?  407 A A A  427 A A A  4297 C C C  427 A A A  4297 C C C  43Z @ @ @  4-t 4 4 4))$*<dCC, , ,  T.35E    t079K    T.35E    t079K    T.35E    t079K    !3!;\  
    C        "K 0 0 0  )+= > > >  "K 0 0 0  !: . . .  *,? @ @ @  "K 0 0 0  C        !3 ' ' '  & 1 1 1  "K 0 0 0  "K 0 0 0  !: . . .  "K 0 0 0  C        T " " "   ( ( (    $,   , , , "  $/ " "#5 6 6 6   $+  z * * * "  $/ " "#6 7 7 7   $,   , , ,   - - -  !4? 3 3 3  t{ + + +  !4? 3 3 3   - - - 4+,,   -, 4=$%%  &%$ 4?&''  (' 4>""  #" (M')) 8 8LFD1((0011$7777 5=!!  "! 4<  7 7 ! 7 59+344  54
 59%;<<  =< 59>&455K K 65K 759>7## 
#uy~+344# # 54# 59>())%) = = = = *)=@ K J J J J J ! " " " 122S S 32S( 59).DIII/ / JI/ 59)4$OOO* * * PO*6  	I2  )JT2224 4 4
 59)9tTTT/ / / UT/
 5*7TRRRDH      SR8 &DAAA      BA F 59?/788  98 59)6DQQQ+ + RQ+* 3 2 2 2 2 2      >8dSSS2 2 TS2l            ! ! ! ! ! ! +  * , , , '  ' ) ) )       ,  + - - -       %  $ & & &      rc   