
    `i                     $   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZmZm	Z	m
Z
 d dlmZmZ d dlZd dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZmZmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d d	l'm(Z( d d
l)m*Z* d dlm+Z+m,Z,m-Z- d dl.m/Z/m0Z0 ddl1m2Z2m3Z3 ddlm4Z4m5Z5m6Z6  e	d          Z7 ed          Z8e
ej9        j:        ej9        j;        f         Z<ee=d<    ej>        e?          Z@ejA        jB        ZBejA        jC        ZCejA        jD        ZDejA        jE        ZEejA        jF        ZF eg eBjG        eBjH        eBjI        eBjJ        eBjK        eBjL        eBjM        eBjN        eBjO        eBjP        eBjQ        eBjR        eBjS        eBjT        eBjU        eBjV        eBjW        eBjX        eBjY        eBjZ        eBj[        eBj\        eBj]        eBj^        eBj_        eBj`        eBja        eBjb        eBjc        eBjd        eBje        eBjf        eBjg        eBjh        e(eBji        eBjj        eBjk        eBjl        jm        eBjn        jo        eDjp        eEjq                  Zri  e            erZseBjt        eBju        eBjv        eBjw        jx        eBjy        eBjz        eBj{        eBj        eBj|        eBj}        eBj~        eBj        eBj        j        eBj        eBj        eBj        eBj        gZee
ej9        j        ej9        j;        f                  e=d<    eese           de
e<ee<         f         deee8e7f         gee8e7f         f         fdZ eeBj        g          dej        dej        demdemdedej        fd            Z eeBj        jx        g          ddddej        deej        j                 d eej        j                 ddfd!            Z eeBj        g          e 	 	 dd"ej        deej        j                 d eej        j                 dej        fd#                        Z eeBj        g          d$ee
emej        f                  d%ej        j        d&edej        fd'            Z eeBj|        g          dd(d"ej        d)emd*ej        d+ej        d,ej        j        dej        fd-            Z| eeBj        jx        g          d$ee
emej        f                  d.eem         d&edej        fd/            Z eeBj        g          dej        d0ej        d1ej        d2eem         d3e
emeem         f         d4e
emeem         f         d5e
emeem         f         d6ed7eem         d8emd9ee         deej        ej        ej        f         fd:            Z eeBj        j        g          dd"ej        d;emdej        fd<            Z eeBj        g          e	 dd=ej        d>ej        d?eej                 dej        fd@                        Z eeBj        g          e	 	 	 dd=ej        dAej        dBej        d?eej                 dCej        j        d,ej        j        dej        fdD                        Z eeBj        g          e	 dd=ej        dEej        d?eej                 dej        fdF                        Z eeBj        jx        g          	 ddGeej                 d)emdej        fdH            Z eeBj        g          d"ej        dej        fdI            Z eeBj        g          dd(d"ej        dJej        d,eej        j                 dej        fdK            Z eeBj        g          d=ej        dej        fdL            Z eeBj        eBj        g          d=ej        dej        fdM            Z eeBj        eCj        g          d=ej        dNej        dej        fdO            Z eeBj        eCj        g          d=ej        dNej        dej        fdP            Z eeBj                  	 	 dd=ej        d)eem         dRedej        fdS            Z eeBj                  	 	 dd=ej        d)eem         dRedej        fdT            Z eeBj        g          d=ej        d)emdUemdVemdej        f
dW            Z eeBj        jx        g          d=ej        d$ee
emej        f                  dej        fdX            Z eeBj        j        g          d=ej        dYej        dej        fdZ            Zd=ej        deej        ej        f         fd[Z eeBj                  ddddQdQej        d\d=ej        d%e
emef         dYeej                 d]eej                 d^eej                 d_ed`edaej        dej        fdb            Zddej        dcddedeej        f         d=ej        dYeej                 d^eej                 daej        d&edej        fdfZ eeBj                  d=ej        d&edej        fdg            Z eeBj                  d=ej        d&edej        fdh            Z eeBj        jx                  d=ej        diemd&edej        fdj            Z eeBj        j                  d=ej        dkemdiemd&edej        f
dl            Z eeBj        jx                  diemd$ee
emej        f                  d&edej        fdm            Z eeDjp        jx                  	 dd0ej        d1ej        dneej                 dej        fdo            Zp eeEjq        jx                  d0ej        dpej        dqej        d1ej        drej        dsej        dnej        dtej        duej        dvemdej        fdw            Zq eejA        jD        j                  dxej        dej        fdy            Z eeBj        g          e	 	 	 ddzej        d{ej        d|emd}emd~edej        fd                        Z eeBj        j                  	 dd=eej                 deej                 deej                 dedeej                 f
d            Z eeBj        j                  	 dd=eej                 deej                 deej                 dedeej                 f
d            Z eeBj        j                  deej                 deej                 d1ej        j        deej                 fd            Z eeBj        j                  deej                 deej                 deej        j                 deej                 fd            ZeBj        jx                            ej        j        j                   eeBj                  d0ej        d1ej        dnej        ej                 dej        ej                 dej        ej                 dedededeej        ej        ej        f         fd                        Ze j        deeedeef         f         fd            Zdeeedeef         f         fdZ eeBj                  d=ej        dej        dej        dej        fd            Z eeFj        j                  d0ej        demdemdedYej        deej        ej        f         fd            Z eeBj                  	 dd=ej        d*ej        dej        dedej        f
d            Z eeBj                  	 dd=ej        d*ej        dej        dedej        f
d            Z eeBjy        jx                  edej        dej        d)emdej        dej        f
d                        Zy eeBj                  ddd=ej        d)emd*ej        dej        dededej        fd            Zd"ej        deem         d3ee
emeem         f                  d4e
emeem         f         d5e
emeem         f         ded)emdeej        ej        f         fdZ eeBj                  	 	 	 	 dd"ej        deem         d3ee
emeem         f                  d4e
emeem         f         d5e
emeem         f         dedeej        ej        f         fd            Z eeBj                  	 	 	 	 dd"ej        deem         d3ee
emeem         f                  d4e
emeem         f         d5e
emeem         f         dedeej        ej        f         fd            Z eeBj                  d"ej        deem         deej        ej        f         fd            Z eeBj        j                  dQdQddddej        d=ej        j        dededee         deej                 dej        fd            Z eeBj                  	 	 	 	 dd=ej        dej        dedededeej                 deej        ej        f         fd            Z eeBj        j                  	 ddej        deem         dej        fd            ZdS )    N)AnyCallableOptionalTypeVarUnion)	ParamSpec	TypeAlias)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addembedding_dense_backwardpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_args)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDtype_to_dtype)guard_or_falsestatically_known_true   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_P_GenericOperatordecomps_to_excludeopsreturnc                     t          | t                    r| n| gD ]&}|t          v rt                              d|            't          j        | t                    S )Nzduplicate decomp: %s)
isinstancelistdecompositionslogwarningdecompregister_decomposition)r&   ops     q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/_inductor/decomposition.pyr/   r/      s\      T**5cc 5 5KK.444(n===    grad_outputindicesnum_weightspadding_idxscale_grad_by_freqc                 D    | j         rt          S t          | ||||          S N)is_xpuNotImplementeddecomp_embedding_dense_backward)r3   r4   r5   r6   r7   s        r1   _embedding_dense_backwardr=      s3      *Wk;8J  r2   )minmaxsymbolr>   r?   c                    d S r9    )r@   r>   r?   s      r1   sym_constrain_range_for_sizerC      s	     Fr2   xc                 b    ||                      |          } ||                     |          } | S r9   )	clamp_min	clamp_max)rD   r>   r?   s      r1   clamprH      s5     KK
KKHr2   size
fill_valuekwargsc                     |                     d          }|2t          t          |                    |d<   t          j        | |fi |S t
          S )Ndtype)getr   typetorchfullr;   )rI   rJ   rK   rM   s       r1   rQ   rQ      sS     JJwE}'Z(8(899wz$
55f555r2   alphadimindextensorrS   c                    t                      s| j        t          j        k    rt          S t          | |||d|          S )NF)inplacerS   )r   rM   rP   bfloat16r;   r   )rD   rT   rU   rV   rS   s        r1   	index_addrZ      s@     ;; M17en44!S%eLLLLr2   physical_layoutc                      dgt                     z  }t          |          D ]
\  }}|||<   t          j         fd|D             fi |                    |          S )Nr   c                      g | ]
}|         S rB   rB   ).0lrI   s     r1   
<listcomp>z"empty_permuted.<locals>.<listcomp>   s    999AQ999r2   )len	enumeraterP   emptypermute)rI   r[   rK   permpr_   s   `     r1   empty_permutedrg      sw     3T?D/**  1Q;9999999DDVDDLLTRRRr2   inputweight
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                 b   |
d         rt          | j        j                  st          S t                              | dgt          t          d|                                                     z             }t          	                    | |||||||||	|
d         |
d         dg          \  }}}|||fS )N   r   r   F)
r   devicerO   r;   atensumr*   rangerT   convolution_backward)r3   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   	grad_biasgrad_inpgrad_weight_s                  r1   rx   rx      s     q> (:(?!@!@ qcDq+//:K:K1L1L,M,M&MNNI#88	QQ/   Hk1 k9--r2   decimalsc                 R    d|z  }t                               | |z            d|z  z  S )Ng      $@g      ?)ru   round)rD   r}   ten_pow_decimalss      r1   	round_decr   	  s/    X~::a**++s5E/EFFr2   selfbatch2r   c                    t           j        r| j        j        dvr}t	          | j        d         dk              st	          |j        d         dk              rA|                     d          |                    d          z                      d          }|S | j        j        dk    rt	          |                     d          dk              rt	          |                    d          dk              rnt          d         dxx         dz  cc<   t          j        |                     d          |                    d          z  dd	
                              d          S t          S )Ncpumpsr   rs   rT   r   inductordecompose_bmmTrT   keepdim)r   coordinate_descent_tuningrt   rO   r   shape	unsqueezerv   rI   r   rP   squeezer;   )r   r   r   outs       r1   bmmr     sY    ' DK,<N,R,R A!!344 	8MLOq 9
 9
 	 >>"%%(8(8(;(;;@@Q@GGCJ{5   1!233 	8MKKOOq 9
 9
 	 Z 111Q61119Q&.."4"44!T  ill r2   mat1mat2betac                 *   | j         j        dk    r|t          |                    d          dk              rt          |                    d          dk              ryt          d         dxx         dz  cc<   t          j        |                    d          |                    d          z  dd                              d          }||z  || z  z   S t          |                    d          dk              rt          |                    d          d	k              rkt          |                    d          d	k              rEt          d         dxx         dz  cc<   |j
        |z                      dd          }||z  || z  z   S t          S )
Nr   r   r   r   r   decompose_addmmTr      )rt   rO   r   rI   r   rP   rv   r   r   r   Tr;   )r   r   r   r   r   rS   r   s          r1   addmmr   )  s    {5   1!233 	-8MIIbMMQ9
 9
 	- Z !2333q8333)Q$,,r"2"224  ill  3;,,!$))A,,!"344	-tyy||r122	- tyy||r122	-
 Z !2333q83336D=%%!T%::C3;,,r2   input2c                    t           j        r| j        j        dvr{t	          | j        d         dk              st	          |j        d         dk              r?|                     d          |                    d          z                      d          S | j        j        dk    rt	          |                     d          dk              rt	          |                     d          dk              rt	          |                    d          dk              rj| j	        |j	        k    rZt          t          j        |           t          j        |          z   dk              r t          d	         d
xx         dz  cc<   | |z  S t	          |                     d          dk              rt	          |                    d          dk              rnt          d	         d
xx         dz  cc<   t          j        |                     d          |                    d          z  dd                              d          S t          S )Nr   r   r   rs   r   r   r       r   decompose_mmTr   )r   r   rt   rO   r   r   r   rv   rI   rM   r   rP   numelr   r   r;   )r   r   r   s      r1   mmr   G  s#    ' HDK,<N,R,R A!!344 	H8MLOq 9
 9
 	H NN1%%(8(8(;(;;@@Q@GGG{5  !$))B--1"455	!%diillQ&677	! &fkk!nn&9::	! v|++D 1 1EK4G4G GBNOO , Z 000A5000&=  1!233 	8MKKOOq 9
 9
 	 Z 000A50009Q&.."4"44!T  ill r2   tensorsc                    dt           j        dt          ffd}t          t	          ||                     t                    dk    rYt          | dt          j        id         }d         }||j	        k    r|
                                n|                    |          S dt                    cxk     rt          |           k     r#n n t          j                                      S t                    dk    rt          fdD                       rd         }t          |j                  }dk     rt          |j                  z   n|                    t                                |                              j        |                     dz             
                                S t*          S )	NrD   r'   c                     t          | j                  dk    r t          | j        d         dk              rdS t          | j                  k     r t          | j                 dk              rdS dS )Nr   r   FT)ra   r   r   )rD   rT   s    r1   non_empty_tensorzcat.<locals>.non_empty_tensoru  si    " qw<<1
a!@!@5QW.1B"C"C5tr2   r   type_promotion_kindr   rM   c              3   ,   K   | ]}|d          u V  dS )r   NrB   )r^   tfiltered_tensorss     r1   	<genexpr>zcat.<locals>.<genexpr>  s?       ) )%&a  ) ) ) ) ) )r2   )rP   Tensorboolr*   filterra   r   r   DEFAULTrM   clonetoru   catdefaultallr   insertr   expandflattenr;   )r   rT   r   promoted_dtype
filtered_tinpr   r   s    `     @r1   r   r   p  s   
EL T      2 F#3W==>>
!!+
 ? G
 
  &a(
 !111 ^44	

 
S!""	1	1	1	1S\\	1	1	1	1	1x 0#666 q  S ) ) ) )*:) ) ) & &  q!SY&)AggcC	NN""3S#.//000(s}}S!!(%088cAgFFLLNNN r2   c                    |                                  rWt          j        t          j        | j                  t          d          t          j        | j        | j                            S t          | t          j
                  \  }}t          j        t          j        || j                  }t          j        | dk     |d          }t          j        t          j        |           t          d          |          S )Nnan)r   rM   rt   r   g        )
is_complexrP   whereisnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpirt   )rD   r|   rM   r   rets        r1   angler     s    ||~~ 
{Kuu{1616/J/J
 
 	
 "	;H  HAu 
	TWE!(	C	C	CB
+a!eR
%
%C;u{1~~uU||S999r2   yc                   t          j        |           o|                                 }t          j        |          o|                                }|r|st          S d}| j        dk    r|j        dk    rd}| j        dk    r|                     d          } |j        dk    r|                    d          }|}|||z  }t          j        | j        |j                  }dt           j        dt           j        fd}| dz   } |dz   } || 	                    | j
        j                            }	 ||	                    |j
        j                            }
t          j        |	|
z   d	          	                    |          }|r|d         S |S )
NFr   Tr   rV   r'   c                     | j         ^ }}|dz  dk    rt          d          g ||dz  dR }|                     |          }|S )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]rs   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)rV   initial_dimslast_dim	new_shapereshaped_tensors        r1   reshape_tensor_complexz#add.<locals>.reshape_tensor_complex  sj     #),x a<1 c  
 6l5HM5155	 ++i00r2   )	start_dim)rP   	is_tensorr   r;   ndimreshapepromote_typesrM   r   r   r   r   )rD   r   rS   x_is_complex_tensory_is_complex_tensoroutput_size_zerozcomplex_typer   
x_reshaped
z_reshapedresults               r1   addr     s     /!,,?/!,,? &9 v{{qv{{v{{IIaLLv{{IIaLL	AAI&qw88Lu|     $ 	
AA	AA''qv|(<(<==J''qv|(<(<==J]:
2bAAAFF|TTF ayMr2   c                 <    |                                  rt          S | S r9   )r   r;   r   s    r1   conj_physicalr     s     Kr2   c                     | S r9   rB   r   s    r1   liftr     s    Kr2   otherc                 `    t          j        t          j        |          || k    z  | |          S r9   rP   r   r   r   r   s     r1   fminr   	  )    ;u{5))UT\:D%HHHr2   c                 `    t          j        t          j        |          || k     z  | |          S r9   r   r   s     r1   fmaxr     r   r2   Fr   c                 h    | j         t          j        k    rt          j        | ||          S t          S Nr   )rM   rP   r   anyr;   r   rT   r   s      r1   amaxr     0     zUZy38888r2   c                 h    | j         t          j        k    rt          j        | ||          S t          S r   )rM   rP   r   r   r;   r   s      r1   aminr     r   r2   startlengthc                 T    t          j        | |||                                          S r9   )rP   narrowr   )r   rT   r   r   s       r1   narrow_copyr   )  s&     <c5&1177999r2   c                 \    t                               | |                                          S r9   )ru   r   r   )r   rI   s     r1   view_copy_defaultr   3  s$    
 99T4  &&(((r2   rM   c                 P    |                      |                                          S r9   )r   r   )r   rM   s     r1   view_copy_dtyper  ;  s     
 775>>!!!r2   c                      t          j                   } fd|D             }dgt          |          z  }t          |          D ]
\  }}|||<   ||fS )Nc                 *    g | ]}j         |         S rB   )r   )r^   r_   r   s     r1   r`   z/_get_shape_permutation_like.<locals>.<listcomp>G  s    444qTZ]444r2   r   )utils3compute_elementwise_output_logical_to_physical_permra   rb   )r   r[   r   permutationrf   r_   s   `     r1   _get_shape_permutation_liker  C  sr     OPTUUO4444O444E#E

"K/**  1A;r2   )rM   layoutrt   
pin_memoryrequires_gradmemory_formatr  rt   r	  r
  r  c          	         || j         n|}|| j        n|}|| j        n|}|t          j        k    r6t          j        | j        ||||||          }|                    |          S |t          j        k    sJ t          |           \  }	}
t          j        |	||||||          }|
t          t          t          |
                              k    r|S |                    |
                                          S )N)rM   r  rt   r	  r
  r  )rM   r  rt   rP   preserve_formatrQ   r   r   stridedr  r*   rw   ra   rd   r   )r   rJ   rM   r  rt   r	  r
  r  r   r   r  s              r1   	full_liker  P  s     -DJJUE"NT[[F"NT[[F---J!'
 
 
 yy}y555 &&&&8>>{!'
 
 
 $uS%5%5667777M~~k**00222r2   )rM   rt   r  rand_fn.c                   ||j         n|}||j        n|}|t          j        k    r% | |j        f||d|                    |          S t          |          \  }} | |f||d|}|t          t          t          |                              k    r|S |
                    |                                          S )Nr   r  )rM   rt   rP   r  r   r   r  r*   rw   ra   rd   r   )	r  r   rM   rt   r  rK   r   r  r   s	            r1   
_rand_liker  }  s      -DJJUE"NT[[F---wJ

 
 	
 

 "="
)
)	* 5T::E;W  	 F d5[!1!1223333>>+&&,,...r2   c                 2    t          t          j        | fi |S r9   )r  rP   randr   rK   s     r1   	rand_liker    s    ej$11&111r2   c                 2    t          t          j        | fi |S r9   )r  rP   randnr  s     r1   
randn_liker    s    ek4226222r2   highc                 d    t          t          j        t          j        j        d|          | fi |S Nr   r  	functoolspartialru   randintlow)r   r  rK   s      r1   randint_liker#    s.    i'(8!TBBDSSFSSSr2   r"  c                 d    t          t          j        t          j        j        ||          | fi |S r9   r  )r   r"  r  rK   s       r1   randint_like_lowr%    s0     i'(8#tDDdUUfUUUr2   c                 4    t          j        j        d| |fi |S r  )ru   r!  r"  )r  rI   rK   s      r1   r!  r!    s#     <AtT44V444r2   biasc                     t           j        j                            |          }t           j        j                            | |||                                d                   S r  )rP   r&   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrI   )rh   ri   r'  packed_weights       r1   #linear_dynamic_fp16_unpacked_weightr-    sO     I(MMfUUM9AA}dFKKMM!$4  r2   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          t           j        j                            ||||          }
t           j        j                            | |||
|||	          S r9   )rP   r&   r)  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)rh   r.  r/  ri   r0  r1  r'  r2  r3  r4  r,  s              r1   wrapped_quantized_linearr8    s\     I(@@/ M 9CC  r2   packedc                     dt           j        dt           j        fd} || dddf                   } || ddd f                   }| dd df                             t           j                  |z  |z   S )Nu8r'   c                 &     fddD             \  }}}}t           j        dk    r7||dz  z   |dz  z   |dz  z                       t          j                  d         S |dz  |dz  z   |dz  z   |z                       t          j                  d         S )Nc              3   f   K   | ]+}d |f                              t          j                  V  ,dS .N)r   rP   int32)r^   nr;  s     r1   r   zPq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>  s9      GGQbajmmEK00GGGGGGr2   )r   r   rs      little   r      r>  )sys	byteorderr   rP   float32)r;  rD   r   r   ws   `    r1   bitcast_u8_to_f32z=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32  s    GGGG,GGG
1a=H$$aLAG,R8>>u}MMiXX"Wb)Q!V4q8>>u}MMiXXr2   .i)rP   r   r   rG  )r9  rI  scalesoffsetss       r1   "q_embedding_bag_byte_unpack_decomprM    s    Yel Yu| Y Y Y Y vc2b5j122FsBCCx 011G#ss(u}--6@@r2   agridinterpolation_modepadding_modealign_cornersc                     | j         t          j         d          k    o%|dk    o|                     t          j                   }t	          | |||||          }|S )Nr   r   r  )rO  rP  rQ  rR  _expand_grid)rt   rP   is_contiguouscontiguous_formatdecomp_grid_sampler_2d)rN  rO  rP  rQ  rR  rT  outputs          r1   grid_sampler_2drY    sv     	
EL''' 	C!#	COO%*AOBBL $	-!#!  F Mr2   left_tensorsright_tensorsscalarc                     t           j                            | t           j                            ||          |          S NrR   )ru   _foreach_addList_foreach_mulr   rZ  r[  r\  s       r1   _foreach_addcmul_scalarrc    >     !!d$$\=AA "   r2   c                     t           j                            | t           j                            ||          |          S r^  )ru   r_  r`  _foreach_divrb  s       r1   _foreach_addcdiv_scalarrg    rd  r2   start_tensorsend_tensorsc           	          t           j                            | t           j                            t           j                            ||           |                    S r9   )ru   r_  r`  ra  Scalar_foreach_sub)rh  ri  ri   s      r1   _foreach_lerp_scalarrm  (  sP     !!  "";>>	
 	
  r2   scalarsc           	          t           j                            | t           j                            t           j                            ||           |                    S r9   )ru   r_  r`  ra  
ScalarListrl  )rh  ri  rn  s      r1   _foreach_lerp_scalarlistrq  6  sP     !!$$"";>>	
 	
  r2   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          t                               | |||||||          \  }}	}
|r||	|
fS ||                    d          |                    d          fS )Nr   )ru   native_batch_norm	new_zeros)rh   ri   r'  rr  rs  rt  ru  rv  rN  bcs              r1   miopen_batch_normr}  D  s|     $$"	 	GAq!  1ay	 r2   c                  "    i t           t          S r9   )r+   r   rB   r2   r1   fast_random_decompsr  d  s    5n5 455r2   c                  D    t           j        rt          S t                      S )z"decomps can change based on config)r   fallback_randomr+   r  rB   r2   r1   select_decomp_tabler  k  s        r2   masksourcec                    ddl m}m}  || j        |j                  rt
                              | |g          \  } }|                    d                              d          dz
  }d | ||fD             \  }}}t
          	                    |||gd          }	t          j        ||	|                              | j                  S t          S )Nr   )BackendFeaturehas_backend_featurer   r   c              3   >   K   | ]}|                                 V  d S r9   )r   )r^   rD   s     r1   r   z!masked_scatter.<locals>.<genexpr>  s*      ,W,WQQYY[[,W,W,W,W,W,Wr2   )codegen.commonr  r  rt   MASKED_SCATTER_WITH_INDEXru   broadcast_tensorsr   cumsum_unsafe_masked_indexrP   r   r   r   r;   )
r   r  r  r  r  
source_idx	self_flat	mask_flatsource_flatr   s
             r1   masked_scatterr  r  s     DCCCCCCC4;(PQQ J ++T4L99
d\\"%%,,Q//!3
,W,W4vBV,W,W,W)	9k**;	J<QRSS{9fi88==djIIIr2   	quant_min	quant_maxepsc                    t          j        |           \  }}||z
  t          ||z
            z  }t          j        |t          j        |g                    }|t          j        ||z                                t           j                  z
  }t          j        |||          }|                    t           j	                  |                    t           j
                  fS r9   )rP   aminmaxr   r?   r   r   r   intrH   float64int64)	rh   r  r  r  rM   min_valmax_valscale
zero_points	            r1   choose_qparams_tensorr    s     }U++GWw%	I(=">">>EIeU\3%0011EU[599<<UYGGGJZI>>J88EM""JMM%+$>$>>>r2   
accumulatec                     |                                  }t          j        ||g|                    |j                  |          }|                    | j                  S r9   )r   rP   	index_putr   r   )r   rU   r  r  	flatteneds        r1   putr    sS     IE7FNN5;77 I TZ(((r2   c                 h    t                               | |||          }|                     |          S )N)r  )ru   r  copy_)r   rU   r  r  r   s        r1   put_r    s-     ((4:(
>
>C::c??r2   rX  input_dtypec                     | |z  }t          j        ||d          }t          j        | ||          }| j        |k    r|                    |          }|                                S )NTr   )rP   rv   r   fmarM   r   
contiguous)r3   rX  rT   r  new_grad_outputsum_new_grad
grad_inputs          r1   _softmax_backward_datar    sm     "F*O9_#tDDDL#VG\?KKJ K'']];//
  """r2   Tinclude_selfsrcreduction_typer  c                   |dk    rt          | j                  s| j        j        p| j        j        }t	          j        |          }|r,| }t	          j        |                               |||          }	nZ|                     ||d          }t	          j        |                               |||          }	|		                    |	dk     d          }	|                    |||          }|r||	z  n||	z  S t          t          j        j        || j        |j        |j        j        d          rt           S | j        |dz   d                                          | j        d |                                         z  }
|                                g| j        |dz   d          | j        d |         R }g t'          | j        |z
  | j                  dt'          d| j        |z
            R }|                    t          j                                      |
                              |                              |          }|                     |||||          S )Nmeanr   r   Tr  )r    rM   is_floating_pointr   rP   	ones_likerZ   
index_fill
zeros_likemasked_fillr!   ru   scatter_reduce_twort   rO   r;   r   r   rw   r   r   r  repeat_interleaver   rd   scatter_reduce)r   rT   rU   r  r  r  true_divisiononesr   countsrepeatsindex_shapere   scatter_indexs                 r1   index_reducer    sE    (T
) ) 
4M
8Ms## 	7C_T**44S%FFFF//#ua00C%d++55c5$GGF''
A66FmmC,,,?sV||#-? 
	
   jq#))++dj#.>.D.D.F.FFG;;==L4:cAgii#8L4:dsd;KLLKNU49s?DI..NNE!TY_4M4MNND		7	#	#					  !    r2   kernel_size	ceil_modec                 F   |dk    rdg|z  }|dk    rdg|z  }|s|}t          ||          }t          ||          }t          ||          }t          ||          }t          j        t          j        |          }t
          j        j                            ||          s'|t          j	        t
          j
                  j        k    rt          S t                              | |||||          \  }}	t                              |	|| j        | d          |||          }
||
fS )Nr   r   )n_dim)r   r  reduceoperatormulrP   	_inductorlowering%should_fallback_max_pool_with_indicesiinfoint8r?   r;   prims!_low_memory_max_pool_with_offsets'_low_memory_max_pool_offsets_to_indicesr   )rD   r  rk   rl   rm   r  rT   window_sizevalsrL  r4   s              r1   _max_pool_with_indicesr    sA    1}}39!||#) {C00KHc**H7C((G&#&&F"8<==K 	 FFs 	G 	
 	
 UZ00444;;	 MD' ;;	 G =r2   c           	      .    t          | |||||d          S )Nrs   r   r  rD   r  rk   rl   rm   r  s         r1   max_pool2d_with_indicesr  -  *     "	;9!   r2   c           	      .    t          | |||||d          S )NrA  r   r  r  s         r1   max_pool3d_with_indicesr  ;  r  r2   output_sizec                 2   | j         ^ }}}|\  }}|dk    s|dk    r>g |||}|                     |          |                     |t          j                  fS ||z  dk    r.||z  dk    r%||z  ||z  g}t                              | |          S t          S )Nr   r   )r   	new_emptyrP   r  ru   r  r;   )	rD   r  batchh_inw_inh_outw_outo_sizer  s	            r1   adaptive_max_pool2dr  I  s     UD$LE5zzUaZZ'5'%''{{6""AKKekK$J$JJJe|qTE\Q..u}dem4++A{;;;r2   	out_int32rightsidesortersorted_sequencer  r  r  r  c                    t                               | t          j        |g| j                  ||||          d         S )Nrt   r  r   )ru   searchsortedrP   rV   rt   )r  r   r  r  r  r  s         r1   searchsorted_scalarr  [  sQ     dVO$:;;;    	 	r2         ?UUUUUU?noiselowerupper	generatorc                     |rW| dk    }t                               | |||          }t          j        || |z  |           }t          j        ||d          }	||	fS ||z   dz  }
t                               | |
          t          j                    fS )Nr   )r  r   rs   )ru   uniformrP   r   
leaky_relur   )r   r  r  r  rt  r  not_positiverrX  	noise_outnegative_slopes              r1   rrelu_with_noise_functionalr   o  s      EqyLLueyLAA\4!8T::Ka33	y  %-1,t^44elnnDDr2   repeatc                    t           j        j        rt          S |t	          |          t
          urt          S | j        j        dk    rt          S | j        t          j	        t          j
        fv sJ | j        dk    sJ |                     d          }t          j        || j                  }t          j        ||| j        t          j	        k    d          S )Nr   r   r   r  T)r  r  )r   tritonautotune_at_compile_timer;   rO   r  rt   rM   rP   r?  r  r   r  aranger  )r  r  r  poss       r1   repeat_interleave_Tensorr    s    
 }-  d;//s::}U""<EK55555;!]]1F
,{6=
9
9
9C ;D   r2   )NNrx  r9   )Nr   r   )NF)r   r   F)r   )F)Nr   r   F)r  r  FN)r  loggingr   r  rE  typingr   r   r   r   r   typing_extensionsr   r	   rP   torch._decomp_decompr.   torch._prims_common_prims_commonr  $torch.ao.quantization.fx._decomposedr
   r   r   torch._decomp.decompositionsr   rW  r   r   r<   r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   %torch.fx.experimental.symbolic_shapesr   r    r   r   r   r    r!   r"   r#   _opsOperatorBaseOpOverloadPacketr$   __annotations__	getLogger__name__r,   r&   ru   r  	quantizedr)  quantized_decomposed_adaptive_avg_pool2d_backwardindex_selectaddmvr  bitwise_and_bitwise_or_
clamp_min_distelu
empty_likeflipgeluhardtanhlcmr  linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardry  native_group_normnative_layer_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvecr-  r8  inductor_decompositionsr+   _unsafe_indexr  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  rG   rF   rZ   gluselect_scatterslice_scattersplitr   r   rv   unbindbaddbmmr%   r*   
OpOverloadr/   r   r=   rC   SymInttypesNumberrH   rQ   rg   rx   tupler   r}   r   r   rM   r   r   r   r   r   r   r   detach_r   r   r   r   r   	view_copyr   r  	ShapeType
StrideTyper  r  r  r   r  rt   r  r  r  r  r#  	low_dtyper%  r!  embedding_bag_byte_unpackrM  rY  _foreach_addcmulrk  rc  _foreach_addcdivrg  _foreach_lerprm  rp  rq  r}  py_impl_CDispatchKeyAutogradcachedictr  r  r  choose_qparamsrV   r  r  r  r  strr  r  r  r  r  r  r   	Generatorr  r  rB   r2   r1   <module>rf     sz           



  : : : : : : : : : : : : : : 2 2 2 2 2 2 2 2        # # # # # # + + + +         
              F E E E E E ( ( ( ( ( ( ( ( ( ( ( ( 7 7 7 7 7 7 . . . . . .         
 X W W W W W W W $ $ $ $ $ $ $ $          WT]]Yt__#	JUZ88 )    g!!y~	I	Y!
y5 ,,+*++ 	
+ 		+
 	+ 	+ 	+ 		+ 	+ 	+ 		+ 		+ 	+ 	+ 	+  	!+" 	#+$ 	-%+& 	%'+( 	0)+* 	1++, 	$-+. 	//+0 	"1+2 	 3+4 	5+6 	7+8 	9+: 	 ;+< 	=+> 	&?+@ 	A+B 		C+D 	
E+F 	G+H 	I+J 	K+L 	M+N 	O+P 	 $Q+R 	5S+T 	+U+- - \ K,,..J2IJ
 	,4<NN!NHJLHKL#W Duz4ej6QQRS   (  n&8 9 9 9>	&6!77	8>xB (2r6"223> > > > 6788\  	
  \   98" :BCDD )-(,	  L 
%+$	% 
%+$	%	
 
   ED %%# )-(,	 	|		%+$	%	 
%+$	%	 \		 	 	 $# &%	 $$	
uS%,&'
(	"	 	 \		 	 	 %$	 ()) !"M M M|M	M <M L	M ;M \M M M *)M* ,4566S
uS%,&'
(S#YS S \	S S S 76S 2344..<. L. S		.
 #tCy.!. 3S	>". CcN#. . I. . d. 5<u|34. . . 54.@ ,-..G G G GU\ G G G /.G
 
## (, 
,L $ \	    $#0 %%
 (,  ! 
,
, , $	
 +
 ; \    &%8 	"" (,   
, L  $  \	       #" N )*++ : :%,:	: \: : : ,+:z %%:U\ :el : : : &%:& 
##
 +/	7 7 7|7|7 EK&'	7
 \7 7 7 $#7t +,--     .- DL122u|     32 EJ/00Iu| IEL IU\ I I I 10I EJ/00Iu| IEL IU\ I I I 10I 	""  
,	#  \	   #" 	""  
,	#  \	   #" )*++:
,:	: : 	:
 \: : : ,+: /011)
,)
uS%,&'
() \) ) ) 21) -.//"
,";" \" " " 0/"
 
,
 
5?E,,-
  
  
  
  ''
 $(%)%)).)>)3 )3 )3
,)3c5j!)3 EK 	)3
 U\")3 U\")3 )3 )3 &)3 \)3 )3 )3 (')3` $(%)).)>/ / /c5<'(/
,/ EK 	/
 U\"/ &/ / \/ / / /@ ''2EL 2C 2EL 2 2 2 ('2 ((3U\ 3S 3U\ 3 3 3 )(3 )122Tu| T3 T# T%, T T T 32T )344V
,V V(+V7:V
\V V V 54V ,--5
5
uS%,&'
(5 5 \	5 5 5 .-5 	EMNN $( <L 5<
  \	   ON 
;CDD< l L	
 , | , | L  \   ED4 	+EFF
Au| 
A 
A 
A 
A GF
A -.//   |
,  	
  \    0/: -455
 	 
u|
u|$ % 	
 
%,   65 -455
 	 
u|
u|$ % 	
 
%,   65 *122
%
el#
 K
 
%,	
 
 
 32
 *566
%
el#
 %+$%
 
%,	
 
 
 76
 ''(<(EFF.//<L /%,
' /%,/	
 .  !&  5<u|34   0/ GF< 6T#xS'9"9: 6 6 6 6!T#xS'9"9: ! ! ! ! +,,
,
, L \	   -,$ ,;BCC?<?? ? 
	?
 ;? 5<%&? ? ? DC? !!
 	
) 
)
,
)<
) L
) 	
)
 \
) 
) 
) "!
) 	""
 	 
,< L 	
 \   #" 3;<<##L# 
# 	#
 \# # #  =<#( )** 1 1 1
,1	1 <1 
	1
 1 1 \1 1 1 +*1h1|1c1 U3S	>*+1 3S	>"	1
 CcN#1 1 
1 5<%&1 1 1 1h 455 /3%&&'
 
|
c
 U3S	>*+
 3S	>"	

 CcN#
 
 5<%&
 
 
 65
 455 /3%&&'
 
|
c
 U3S	>*+
 3S	>"	

 CcN#
 
 5<%&
 
 
 65
 011|"&s)
5<%&   21" )011
 %)	 	 	\	
+
	 		
 	 3-	 U\"	 \	 	 	 21	& 899 %+/E E
,E<E E 	E
 E (E 5<%&E E E :9E& .566 "& L# \   76  r2   