
    `i|e                       d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
mZmZmZmZmZ d dlZd dlZd dlmZ d dlmZmZ d dlmZ dd	lmZmZ dd
lmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* erd dl+m,Z,  ed          Z- G d dej.        j/                  Z0 G d de          Z1 G d de          Z2 G d de          Z3 G d d          Z4 G d d          Z5 G d de           Z6 G d de"          Z7dS )     )annotationsN)autoEnum)AnyCallable
NamedTupleOptionalTYPE_CHECKINGTypeVar)identity)Scope
TracerBase)SymT   )configdependencies)index_prevent_reordering)DefaultHandler
OpsHandlerWrapperHandler)cache_on_selfreduction_num_outputssympy_index_symbol_with_prefix
sympy_subs)opsV)SequenceTc                  d     e Zd Zeej        d                         Z fdZd	 fdZ fdZ	 xZ
S )
InterpreterShimc                 J    t           j                            t                    S N)torchfxsymbolic_tracer        m/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/_inductor/loop_body.py	_dummy_gmzInterpreterShim._dummy_gm&   s     x&&x000r'   c                    t                                          |                                 d           | | _        || _        || _        d| _        |j        | _        d | _	        d S )NF)garbage_collect_values)
super__init__r)   modulegraph
submodulesextra_traceback__getitem__
fetch_attrcurrent_node)selfr/   r0   	__class__s      r(   r-   zInterpreterShim.__init__+   sa     	))%HHH
$$$0 r'   ntorch.fx.Nodereturnr   c                T    || _         t                                          |          S r"   )r4   r,   run_node)r5   r7   r6   s     r(   r;   zInterpreterShim.run_node6   s#    ww"""r'   c                    t          j        |           5   t                      j        |i |cd d d            S # 1 swxY w Y   d S r"   )r   set_interpreter_handlerr,   run)r5   argskwargsr6   s      r(   r>   zInterpreterShim.run:   s    &t,, 	0 	0577;///	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0s   <A A )r7   r8   r9   r   )__name__
__module____qualname__staticmethod	functoolscacher)   r-   r;   r>   __classcell__r6   s   @r(   r    r    %   s        _1 1 _ \1	! 	! 	! 	! 	!# # # # # #0 0 0 0 0 0 0 0 0r'   r    c                       e Zd Z fdZ xZS )LightTracerc                    t                                                       t          j                            | j                  | _        t          dd           | _        i | _	        i | _
        d S )N)
tracer_cls )r,   r-   r#   r$   Graphr6   r/   r   scopemodule_stacknode_name_to_scope)r5   r6   s    r(   r-   zLightTracer.__init__A   sW    X^^t~^>>
2t__
"$r'   )rA   rB   rC   r-   rG   rH   s   @r(   rJ   rJ   @   s8        % % % % % % % % %r'   rJ   c                  .    e Zd ZU ded<   ded<   ded<   dS )MemoryEntrystr
index_nameOptional[str]buffer_namemodeN)rA   rB   rC   __annotations__r&   r'   r(   rS   rS   I   s3         OOOr'   rS   c                      e Zd Z e            Z e            Z e            Z e            Z e            Z e            Z	 e            Z
dS )MemoryUsageTypeN)rA   rB   rC   r   LOAD	LOAD_SEEDSTORESTORE_REDUCTION
INDEX_EXPRCHECK_BOUNDS	BUCKETIZEr&   r'   r(   r[   r[   O   s[        466DIDFFEdffOJ466LIIIr'   r[   c                  h    e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ded<    fdZd Zd@dZdAdZdBdZ	dCd Z
dBd!Zed"             Zed#             Zed$             Zd% Zd& Zd' Zd( Zd) Zd* Zd+ ZdDd-ZeZ	 	 dEdFd6Zd7 Zd8 Zd9 Zd: Zd; Zd< Z d= Z!d> Z"d? Z# xZ$S )GLoopBodyz
    Captures the body of a Loops subclass into an FX graph.  Persists any
    indexing simplifications and makes it easier to analyze loop bodies.
    zdict[str, sympy.Expr]indexing_exprszdict[sympy.Expr, str]indexing_exprs_namedict[str, Any]r0   zdict[str, LoopBodyBlock]	subblockszlist[sympy.Symbol]indirect_varszdict[sympy.Symbol, sympy.Expr]indirect_var_rangesLoopBodyBlock
root_blockz(dict[MemoryUsageType, list[MemoryEntry]]memory_usagecollections.Counter[str]	op_countsc                   t                                                       t          |                                          }|d t	          |                   |t	          |          d          f| _        || _        || _        || _        t          |t                    r|                     ||           n|                     ||           d | _        d S r"   )r,   r-   tuplevalueslensizes	iter_varsreduce_vars
var_ranges
isinstancerd   _init_with_copy_init_with_tracingindexing)r5   fnr?   rw   ru   rv   _flat_sizesr6   s          r(   r-   zLoopBody.__init__j   s    J--//00(#i..()I(()


 #&$b(## 	.  T****##B---r'   c                    i | _         i | _        d| j        i| _        i | _        g | _        i | _        d t          D             | _        t          j
                    | _        t          | ||          | _        | `dS )z9Do an FX trace of an arbitrary callable to construct self	get_indexc                    i | ]}|g S r&   r&   ).0ts     r(   
<dictcomp>z/LoopBody._init_with_tracing.<locals>.<dictcomp>   s    <<<qQ<<<r'   N)re   rf   r   r0   rh   ri   rj   r[   rm   collectionsCounterro   rk   rl   )r5   r|   r?   s      r(   rz   zLoopBody._init_with_tracing~   s|     #% &7CE <<O<<<$,..'b$77$$$r'   otherc                    |                     |          } fd|                                D              _         fd|j                                        D              _        |j         _        |j         _        |j         _        |j         _        |j        	                                _        i |j
        }|                    d           d j        i fd|                                D              _
        dS )z
        _init_with_tracing() is slow, so this is a fast path in the case
        where we are just reordering/merging/splitting the args of an
        existing LoopBody.
        c                l    i | ]0\  }}|t           j        j                            |j                  1S r&   )r   r/   sizevarssimplify_with_rangesrw   )r   nameexprr5   s      r(   r   z,LoopBody._init_with_copy.<locals>.<dictcomp>   sF     
 
 
d !'"77doNN
 
 
r'   c                B    i | ]\  }}||                               S r&   cloner   kvr5   s      r(   r   z,LoopBody._init_with_copy.<locals>.<dictcomp>   s)    OOOtq!!QWWT]]OOOr'   r   c                B    i | ]\  }}||                               S r&   r   r   s      r(   r   z,LoopBody._init_with_copy.<locals>.<dictcomp>   s)    ???DAqq!''$--???r'   N)indexing_from_argsitemsre   rh   ri   rj   rm   ro   rl   r   r0   popr   )r5   r   r?   re   r0   s   `    r(   ry   zLoopBody._init_with_copy   s    11$77
 
 
 
,2244
 
 
 POOOu7L7L7N7NOOO"0#(#< !.*0066)()
{###
????J,<,<,>,>???
r'   r   rT   c                @    | j                             |d          dk    S )Nr   )ro   getr5   r   s     r(   has_opzLoopBody.has_op   s    ~!!$**Q..r'   r9   c           	     <   | }| j         }|j        \  }}|\  }}g |j                                        }t          j        j                            ||t          |||                    \  }}	}
t          j        j                            ||t          |||                    \  }}}
t          j
        ||d          \  \  }}}t          | |	|           ||          g|||          }t          j
        ||d          \  \  }}}t          |||f|||          }|S )zU
        Merge both iteration and reduction loops and return a new LoopBody.
        r   )prefixp)rt   varsre   rr   r   r/   r   _simplify_loopsr   r   index_vars_no_squeezerd   )r5   old_body	old_sizesold_iter_varsold_reduce_varsold_iter_sizesold_reduce_sizesindex_exprs
iter_sizesiter_reindex_reduce_sizesreduce_reindexru   rv   rw   new_body
iter_vars2reduce_vars2var_ranges2	new_body2s                        r(   merge_loopszLoopBody.merge_loops   sd    J	)1&+4((9/66889&'g&6&F&F$[-PP'
 '
#
L! +,'*:*J*J$[/CSTT+
 +
'na. .z<PSTTT	
\)$$nn[&A&AB
 
 3?2TS3
 3
 3
/"\K z<0+z<
 
	 r'   	dimensionint	new_rangec                0   | | j         }|\           t                    }||<   |f}t          j        |ddi\  \  }}}d	fd}	t	          |	||f|||          }
t          j        |ddi\  \  }}}t	          |
||f|||          }	|	S )
z~
        Expand node on `dimension` to `new_range` and rely on index modular to avoid
        out-of-boundary access.
        r   r   indicesSequence[sympy.Expr]r9   r   c                 X   g t           j                            |           }t          |          t                    t          	          z   k    sJ |d t                             }|t                    d          }t	          |          }|         z  |<    ||          S r"   )	itertoolschainfrom_iterablers   list)
r   indexiter_idx
reduce_idxnew_iter_idxr   	iter_sizer   original_rangereduce_sizes
        r(   r   z>LoopBody.expand_dimension_for_pointwise_node.<locals>.new_body   s    =io33G<<=Eu::Y#k2B2B!BBBBB-s9~~-.Hs9~~//0J>>L&.y&9N&JL#8L*555r'   r   r   r   r9   r   )rt   r   r   r   rd   )r5   r   r   r   new_iter_size	new_sizesru   rv   rw   r   	loop_bodyr   r   r   r   r   r   r   s    `            @@@@r(   #expand_dimension_for_pointwise_nodez,LoopBody.expand_dimension_for_pointwise_node   s    J	!*	;"9-Y#,i "K0	/;/Q0
0
 0
, K*
		6 		6 		6 		6 		6 		6 		6 		6 		6 		6 y+.
I{
 
	
 3?2T3
3
 3
/"\K 
L1;
L
 
 r'   c                   ddl m} | | j        }t          |d                   t          |          k    sJ  ||          }|\   |          }|f}t	          j        |ddi\  \  }}}	d t          |          D             fdt          t          |                    D             dfd}
t          |
||f|	||          }t	          j        |ddi\  \  }}}t          |||f|||          }
|
S )zD
        Reorder iteration loops and return a new LoopBody.
        r   )same_reorderr   r   r   c                    i | ]\  }}||	S r&   r&   )r   abs      r(   r   z/LoopBody.reorder_iter_loops.<locals>.<dictcomp>&  s    ???$!QA???r'   c                     g | ]
}|         S r&   r&   )r   iinverse_orders     r(   
<listcomp>z/LoopBody.reorder_iter_loops.<locals>.<listcomp>'  s    IIIaq)IIIr'   r   r   r9   r   c                 <   g t           j                            |           }t          |          t                    t                    z   k    sJ |d t                             |t                    d          }fdD              |          S )Nc                     g | ]
}|         S r&   r&   )r   r   r   s     r(   r   zALoopBody.reorder_iter_loops.<locals>.new_body.<locals>.<listcomp>.  s    ;;;;;;r'   )r   r   r   rs   )r   r   r   r   r   r   r   r   s      @r(   r   z-LoopBody.reorder_iter_loops.<locals>.new_body)  s    =io33G<<=Eu::Y#k2B2B!BBBBB-s9~~-.Hs9~~//0J;;;;];;;H8Hj111r'   r   r   )	irr   rt   rs   r   r   	enumeraterangerd   )r5   	new_orderr   r   
reorder_fnr   r   ru   rv   rw   r   r   r   r   r   r   r   r   r   s                  @@@@r(   reorder_iter_loopszLoopBody.reorder_iter_loops  s    	%$$$$$J	9Q<  C	NN2222!\),,
!*	;"
9--"K0	/;/Q0
0
 0
, K*
 @?)I*>*>???IIII5Y3H3HIII	2 	2 	2 	2 	2 	2 	2 	2 	2 y+.
I{
 
	
 3?2T3
3
 3
/"\K 
L1;
L
 
 r'   c                B    | j         J | j        J | j         | j        fS r"   )ru   rv   r5   s    r(   r   zLoopBody.vars?  s/    ~)))+++~t///r'   c                    t          j        | j        j        fd | j                                        D                       }d |D             S )Nc              3  $   K   | ]}|j         V  d S r"   )r/   )r   blocks     r(   	<genexpr>z%LoopBody.get_nodes.<locals>.<genexpr>I  s$      >>UU[>>>>>>r'   c                &    g | ]}|j         D ]}|S r&   )nodes)r   r/   nodes      r(   r   z&LoopBody.get_nodes.<locals>.<listcomp>K  s'    EEEEEEEEEr'   )r   r   rl   r/   rh   rr   )r5   
all_graphss     r(   	get_nodeszLoopBody.get_nodesE  sT    __"$>>dn&;&;&=&=>>>
 

 FE*EEEEr'   c                $    ddl m}  ||           S )Nr   )	BoundVars)boundsr   )r5   r   s     r(   r   zLoopBody.boundsM  s$     	&%%%%%yr'   c                    t          | j        t          j                           D ]!}|j        |k    r| j        |j                 c S "t          |          r"   )reversedrm   r[   r\   rW   re   rU   KeyErrorr5   rW   entrys      r(   get_read_exprzLoopBody.get_read_exprT  s\    d/0DEFF 	= 	=E K//*5+;<<<< 0{###r'   c                    t          j        | j        t          j                 | j        t          j                           D ]!}|j        |k    r| j        |j                 c S "t          |          r"   )
r   r   rm   r[   r^   r_   rW   re   rU   r   r   s      r(   get_write_exprzLoopBody.get_write_expr[  ss    _o34o=>
 
 	= 	=E  K//*5+;<<<< 0{###r'   c                J      fd j         t          j                 D             S )Nc                4    g | ]}j         |j                 S r&   re   rU   r   r   r5   s     r(   r   z+LoopBody.get_read_exprs.<locals>.<listcomp>e  s4     
 
 
  01
 
 
r'   )rm   r[   r\   r   s   `r(   get_read_exprszLoopBody.get_read_exprsd  s9    
 
 
 
*?+?@
 
 
 	
r'   c                    g }t          | j        t          j                           D ]2}|j        |k    r%|                    | j        |j                            3|S r"   )r   rm   r[   r\   rW   appendre   rU   r5   rW   outr   s       r(   get_all_read_exprzLoopBody.get_all_read_exprj  s\    d/0DEFF 	B 	BE K//

4.u/?@AAA
r'   c                      fdt          j         j        t          j                  j        t          j                           D             S )Nc                4    g | ]}j         |j                 S r&   r   r   s     r(   r   z,LoopBody.get_write_exprs.<locals>.<listcomp>s  s4     
 
 
  01
 
 
r'   )r   r   rm   r[   r^   r_   r   s   `r(   get_write_exprszLoopBody.get_write_exprsr  sV    
 
 
 
"!/"78!/"AB 
 
 
 	
r'   c                    g }t          j        | j        t          j                 | j        t          j                           D ]2}|j        |k    r%|                    | j        |j	                            3|S r"   )
r   r   rm   r[   r^   r_   rW   r   re   rU   r   s       r(   get_all_write_exprzLoopBody.get_all_write_expr{  ss    _o34o=>
 
 	B 	BE  K//

4.u/?@AAA
r'   c                j   dt          | j                   g}|                    d | j                                        D                        |                    d t          j        d| j        fg| j                                                  D                        d	                    |          S )Nzvar_ranges = c                "    g | ]\  }}| d | S )z = r&   )r   r   vals      r(   r   z&LoopBody.debug_str.<locals>.<listcomp>  s*    UUUID#''#''UUUr'   c                >    g | ]\  }}|                     |          S r&   )	debug_str)r   r   r   s      r(   r   z&LoopBody.debug_str.<locals>.<listcomp>  s8       D% %%  r'   body
)
dictrw   extendre   r   r   r   rl   rh   join)r5   liness     r(   r  zLoopBody.debug_str  s    8do!6!6889UU9L9R9R9T9TUUUVVV #,?do./1E1E1G1G$ $  	
 	
 	
 yyr'   boolc                   t          | j        t          j                           dk    oYt          | j        t          j                           dk    o1t          | j                  dk    o| j                            d          S )zx
        True of this contains only a single loads and store.
        Note, this could involve a layout change.
        r   )loadstore)rs   rm   r[   r\   r^   r0   rl   contains_only_opsr   s    r(   is_memory_copyzLoopBody.is_memory_copy  sy     !/"6788A= ED%o&;<==BEDO$$)E 112CDD		
r'   Nr   
sympy.Exprmtyper[   rW   rV   rX   c                    | j                             |          }|s+dt          | j                   }|| j         |<   || j        |<   | j        |                             t          |||                     |S )Nr   )rf   r   rs   re   rm   r   rS   )r5   r   r  rW   rX   r   s         r(   add_index_exprzLoopBody.add_index_expr  s     '++D11 	-53t23355D-1D$T*(,D%% ''D+t(L(LMMMr'   c                    |d                                          r|| j        vr|}n| t          | j                   }|| j        |<   |S )zaNot actually for nn.Modules, but subblocks in generated code are mapped to FX call_module opcodes)	isnumericr0   rs   )r5   r   r   r   s       r(   add_submodulezLoopBody.add_submodule  sX    ":!! 	5fDO&C&CDD4c$/2244D %r'   c                    t          t          j        t          | j                            }|| j        vsJ | j                            |           || j        |<   |S r"   )r   r   INDIRECTrs   ri   rj   r   )r5   sizevars      r(   add_indirectzLoopBody.add_indirect  s[    ,T]C@R<S<STT$22222!!#&&&(, %
r'   c                    t                    t                    k    rdS | j        J fd| j                                        D             | _        dS )z,Swap in a variable used in indirect indexingNc                <    i | ]\  }}|t          |i          S r&   r   )r   r   r   newolds      r(   r   z-LoopBody.replace_indirect.<locals>.<dictcomp>  s-    XXX$!QJq3*55XXXr'   )rT   r{   r   )r5   r"  r!  s    ``r(   replace_indirectzLoopBody.replace_indirect  s^    s88s3xxF}(((XXXXX$-BUBUBWBWXXXr'   c                .    | j         J | j         |         S r"   )r{   r   s     r(   r   zLoopBody.get_index  s    }(((}T""r'   c                    g t           j                            |          }t          |          t           j                  k    sJ | j        f            t           fd|D                       sJ d j        d|            t          t           j                                        |                    fd j	        
                                D             S )Nc              3  *   K   | ]}|j         vV  d S r"   )rw   )r   r   r5   s     r(   r   z.LoopBody.indexing_from_args.<locals>.<genexpr>  s*      ;;1DO+;;;;;;r'   zself.var_ranges=z
, indices=c                8    i | ]\  }}|t          |          S r&   r   )r   r   r   replacementss      r(   r   z/LoopBody.indexing_from_args.<locals>.<dictcomp>  s9     
 
 
d *T<00
 
 
r'   )r   r   r   rs   rw   allr  zipkeysre   r   )r5   r   r   r(  s   `  @r(   r   zLoopBody.indexing_from_args  s    9)///8895zzS11111E4?3K111;;;;U;;;;; 	
 	
-t--7--	
 	
; C 4 4 6 6>>??
 
 
 
"17799
 
 
 	
r'   c                p    |                      |          | _        |                                 }d | _        |S r"   )r   r{   rl   )r5   r   results      r(   __call__zLoopBody.__call__  s2    //88""r'   c                r      fd}t          j        t          j                  |_        |S )Nc           	     t                         t          j                            |                      d S r"   )r#  r   r   indirect_indexing)new_varcheckr5   r  r  wrap_negs    r(   set_indirectz5LoopBody.bind_set_indirect_shim.<locals>.set_indirect  s@    !!QU,,WdE8LL    r'   )r  r  r3  r4  )rE   partialrd   bind_set_indirect_shimr   )r5   r  r  r3  r4  r5  s   ````` r(   r7  zLoopBody.bind_set_indirect_shim  sj    	 	 	 	 	 	 	 	 	
 '.+
 
 
 r'   c                \    fd}t          j        t          j                  |_        |S )Nc                F    t           j                            | |          S r"   )r   r   scan)dtypesrr   
combine_fns     r(   shimz%LoopBody.bind_scan_shim.<locals>.shim  s    5::fj&999r'   )r<  )rE   r6  rd   bind_scan_shimr   )r5   r<  r=  s    ` r(   r>  zLoopBody.bind_scan_shim  s=    	: 	: 	: 	: 	: &x'>:VVV
r'   c                `      fd}t          j        t          j                  |_        |S )Nc                \    t           j                            | j                 |          S r"   )r   r   maskedrh   )maskr   r   r5   s     r(   r=  z'LoopBody.bind_masked_shim.<locals>.shim  s"    5<<dnT&:EBBBr'   )r   )rE   r6  rd   bind_masked_shimr   )r5   r   r=  s   `` r(   rC  zLoopBody.bind_masked_shim  sI    	C 	C 	C 	C 	C 	C &x'@tLLL
r'   )r   rd   )r   rT   )r9   rd   )r   r   r   r   r9   rd   r9   r  NN)r   r  r  r[   rW   rV   rX   rV   )%rA   rB   rC   __doc__rY   r-   rz   ry   r   r   r   r   propertyr   r   r   r   r   r   r   r   r   r   r  r  __repr__r  r  r  r#  r   r   r.  r7  r>  rC  rG   rH   s   @r(   rd   rd   Z   s         
 *)))....''''%%%%7777::::''''    (% % %
 
 
 
2/ / / /9 9 9 9v- - - -^, , , ,\ 0 0 X0
 F F ]F   ]$ $ $$ $ $
 
 
  
 
 
       

 

 

 

 H &*"        Y Y Y# # #

 

 

            r'   rd   c                  8    e Zd ZdZddZd	 ZddZddZddZdS )rk   a  
    Captures the body of a Loops subclass into an FX graph.
    In normal cases there will be a 1:1 mapping between LoopBody and
    LoopBodyBlock, however in the case of ops.masked() the masked out
    operations will manifest as an extra LoopBodyBlock.
    r  rd   r|   Callable[..., Any]r?   	list[Any]c                   || _         t                      }|                    dddi           }ddlm} t          t          |||          |j                  }t          j	        r! ||| j         j
        | j         j                  }t          j        |          5  t          j         ||            d d d            n# 1 swxY w Y   |j        | _        d S )Nplaceholderr   r&   r   )IndexPropagation)r  rJ   create_proxyindex_propagationrN  CountOpsCaptureIndexingro   r   constant_and_index_propagationrw   rj   r   set_ops_handlerr   outputr/   )r5   r  r|   r?   tracer	proxy_opsrN  handlers           r(   r-   zLoopBodyBlock.__init__  s   	''ub"EE	777777ItV44N
 
 0 	&&-ty/L G w'' 	" 	" Jrr4y!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" \


s   B==CCc                    | j         }| j        j        }t          ||                              t          j                              S r"   )r/   r  r0   r    r>   r   get_ops_handler)r5   r/   r0   s      r(   r.  zLoopBodyBlock.__call__  s:    
Y)
uj1155a6G6I6IJJJr'   r   c           
         t           j                            | j        j        | j                  j        }t          j        dd|	                                
                    dd| d                    S )Nz;[^\n]*rM   zdef forward(zdef ()r#   r$   GraphModuler  r0   r/   coderesubstripreplace)r5   r   r^  s      r(   r  zLoopBodyBlock.debug_str  s`    x##DI$8$*EEJvJJLL  @@	
 
 	
r'   r9   r  c                l    t          fd| j                            d          D                       S )Nc              3  *   K   | ]}|j         v V  d S r"   )target)r   r   allowed_opss     r(   r   z2LoopBodyBlock.contains_only_ops.<locals>.<genexpr>)  s>       
 
 K;&
 
 
 
 
 
r'   call_method)op)r)  r/   
find_nodes)r5   rf  s    `r(   r  zLoopBodyBlock.contains_only_ops(  sN     
 
 
 

---??
 
 
 
 
 	
r'   c                    t                               t                     }|j                            i | j        d|i           |S )z'Shallow copy with a new parent LoopBodyr  )rk   __new____dict__update)r5   r  copys      r(   r   zLoopBodyBlock.clone.  sB    $$]33<<vt<<===r'   N)r  rd   r|   rJ  r?   rK  )r   rD  )r  rd   )	rA   rB   rC   rF  r-   r.  r  r  r   r&   r'   r(   rk   rk     s         " " " ".K K K
 
 
 

 
 
 
     r'   rk   c                      e Zd ZddZddZdS )rQ  innerOpsHandler[Any]countsrn   c                "    || _         || _        d S r"   )_inner_counts)r5   rp  rr  s      r(   r-   zCountOps.__init__6  s    r'   r   rT   r?   tuple[Any, ...]r@   rg   r9   r   c                b    | j         |xx         dz  cc<    t          | j        |          |i |S )Nr   )ru  getattrrt  )r5   r   r?   r@   s       r(   _defaultzCountOps._default:  sA    Ta)wt{D))4:6:::r'   N)rp  rq  rr  rn   )r   rT   r?   rv  r@   rg   r9   r   )rA   rB   rC   r-   ry  r&   r'   r(   rQ  rQ  5  s<           ; ; ; ; ; ;r'   rQ  c                       e Zd Zd Zd6 fdZd7dZd8dZd9dZd:dZd;dZ	d Z
d Zd Zd Z	 	 d<d=d*Zd>d-Zd?d0Zd1 Zd2 Zd@d4Zd5 Z xZS )ArR  rp  rq  r  rd   rV  rJ   c                f    t                                          |           || _        || _        d S r"   )r,   r-   r  rV  )r5   rp  r  rV  r6   s       r(   r-   zCaptureIndexing.__init__B  s/     		r'   r   r  r  r[   r@   r   c           	     b    | j                             dd | j        j        ||fi |fi           S )Ncall_moduler   )rV  rO  r  r  )r5   r   r  r@   s       r(   
_add_indexzCaptureIndexing._add_indexL  sD    {''%TY%dE<<V<<>	
 
 	
r'   r9   c                `    t           j        j                            || j        j                  S r"   )r   r/   r   r   r  rw   )r5   r   s     r(   	_simplifyzCaptureIndexing._simplifyT  s"    w44T49;OPPPr'   r   rT   r   c                    |                      |          }|                     |t          j        |          }| j                            ||          S NrW   )r  r~  r[   r\   rt  r  r5   r   r   s      r(   r  zCaptureIndexing.loadW  sE    u%%';NN{e,,,r'   r   c                    t          |t                    sJ | j                            t	          j        |          t          j        |           | j        	                    ||          S r  )
rx   r   r  r  sympyIntegerr[   r]   rt  	load_seedr  s      r(   r  zCaptureIndexing.load_seed\  se    %%%%%%	  M%  /"; 	! 	
 	
 	
 {$$T5111r'   Nc                    |                      |          }|                     |t          j        ||          }| j                            ||||          S )N)rW   rX   )r  r~  r[   r^   rt  r  )r5   r   r   valuerX   s        r(   r  zCaptureIndexing.storec  sU    u%%?(d   
 
 {  ueT:::r'   c                    |                      |          }|                     |t          j        |          }| j                            |||          S r  )r  r~  r[   r_   rt  store_reduction)r5   r   r   r  s       r(   r  zCaptureIndexing.store_reductionj  sQ    u%%?2   
 
 {**4>>>r'   c                    | j                             ||||          t          |          }|dk    r(t          fdt	          |          D                       S S )Nr   c              3  (   K   | ]}|         V  d S r"   r&   r   r   r-  s     r(   r   z,CaptureIndexing.reduction.<locals>.<genexpr>u  s'      ??q??????r'   )rt  	reductionr   rq   r   )r5   dtype	src_dtypereduction_typer  num_outputsr-  s         @r(   r  zCaptureIndexing.reductionq  sd    &&uiOO+N;;??????E+,>,>??????r'   c                4   |                      |          }t          |t          t          j        f          r(| j                            t          |          |          S |                     |t          j	                  }| j        
                    ||          S r"   )r  rx   r   r  r  rt  constantr~  r[   r`   
index_expr)r5   r   r  s      r(   r  zCaptureIndexing.index_exprx  sx    u%%ec5=122 	;;''E

E:::'ABB{%%eU333r'   c                    |                      |          }|                     |t          j                  }|                     |t          j                  }| j                            ||||          S r"   )r  r~  r[   ra   rt  check_bounds)r5   r   r  loweruppers        r(   r  zCaptureIndexing.check_bounds  s[    u%%'CDDt_%ABB{''tUEBBBr'   rr   r   
boundaries.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]boundary_indicesindexing_dtypetorch.dtyperightr  sorter Optional[tuple[str, sympy.Expr]]sorter_indicesOptional[T]c           	        |d         |                      |d         t          j        |d                   |                      |d         t          j        |d                   |                      |d         t          j        |d                   f}|6|d         |                      |d         t          j        |d                   f}| j                            |||||||          S )z3
        See [Note: Inductor bucketize op]
        r   r   r        )r~  r[   rb   rt  	bucketize)r5   rr   r  r  r  r  r  r  s           r(   r  zCaptureIndexing.bucketize  s    qMOO1)&qM   
 OO1)&qM   
 OO1)&qM   

$ q	1I8fQi    F {$$
 
 	
r'   masked_bodyrJ  c                
   | j                             dd          }| j                             |          | j         j        |<   t	          | j         |g           | j         j        |<   | j                            d|||fi           S )zR
        Recursively capture the masked out body in another LoopBodyBlock
        Nmasked_subblockr}  )r  r  rC  r0   rk   rh   rV  rO  )r5   
mask_proxyr  other_proxyr   s        r(   rA  zCaptureIndexing.masked  s     y&&t->??%)Y%?%?%E%E	T"$1$)["$M$M	D!{''4*k!:B
 
 	
r'   r<  =Callable[[tuple[Any, ...], tuple[Any, ...]], tuple[Any, ...]]c                   | j                             |          }| j                             |d          }| j                            d|||fi           t          fdt          t          |                    D                       S )Nr:  r}  c              3  (   K   | ]}|         V  d S r"   r&   r  s     r(   r   z'CaptureIndexing.scan.<locals>.<genexpr>  s'      @@1VAY@@@@@@r'   )r  r>  r  rV  rO  rq   r   rs   )r5   dtype_proxyr<  value_proxyr=  r   r-  s         @r(   r:  zCaptureIndexing.scan  s     y''
33y&&tV44))+&	
 
 @@@@c+.>.>(?(?@@@@@@r'   c                    | j                             ||||          t          fdt          t	          |                    D                       S )Nc              3  (   K   | ]}|         V  d S r"   r&   r  s     r(   r   z'CaptureIndexing.sort.<locals>.<genexpr>  s'      ;;1VAY;;;;;;r'   )rt  sortrq   r   rs   )r5   r;  rr   stable
descendingr-  s        @r(   r  zCaptureIndexing.sort  sO    !!&&&*EE;;;;c&kk(:(:;;;;;;r'   c                V    | j                             |          }|d         |d         fS )Nr   r   )rt  frexp)r5   r  r-  s      r(   r  zCaptureIndexing.frexp  s*    "";//q	6!9%%r'   Tc                    | j                             |          }| j                             ||||          }| j                            d| j                             |d|           |fi            |S )zx
        Flow data from tensors into indexing formulas.
        Introduce a call_module to update the indexing.
        r}  set_)r  r  r7  rV  rO  r  )r5   index_proxyr  r3  r4  r  r5  s          r(   r1  z!CaptureIndexing.indirect_indexing  s{     i$$T**y77T5(SS  I##L,,,??N		
 	
 	
 
r'   c                @    | j                             dd|i            d S )NrU  )rV  rO  )r5   r-  s     r(   rU  zCaptureIndexing.output  s$      8VR@@@@@r'   )rp  rq  r  rd   rV  rJ   )r   r  r  r[   r@   r   )r   r  r9   r  )r   rT   r   r  )r   rT   r   r   r"   rE  )rr   r   r  r  r  r   r  r  r  r  r  r  r  r  r9   r   )r  rJ  )r<  r  )TT)rA   rB   rC   r   r-   r~  r  r  r  r  r  r  r  r  r  rA  r:  r  r  r1  rU  rG   rH   s   @r(   rR  rR  ?  s       D     
 
 
 
Q Q Q Q- - - -
2 2 2 2; ; ; ;? ? ?  4 4 4C C C 48&*/
 /
 /
 /
 /
b	
 	
 	
 	
A A A A"< < <
& & &
    A A A A A A Ar'   rR  )8
__future__r   r   rE   r   r_  enumr   r   typingr   r   r   r	   r
   r   r  torch.fxr#   torch._dynamo.utilsr   torch.fx.proxyr   r   torch.utils._sympy.symbolr   rM   r   r   codegen.commonr   ops_handlerr   r   r   utilsr   r   r   r   virtualizedr   r   collections.abcr   r   r$   Interpreterr    rJ   rS   r[   rd   rk   rQ  rR  r&   r'   r(   <module>r     s   " " " " " "             				         N N N N N N N N N N N N N N N N   ( ( ( ( ( ( , , , , , , , , * * * * * * " " " " " " " " 4 4 4 4 4 4 C C C C C C C C C C                      )(((((( GCLL0 0 0 0 0eh* 0 0 06% % % % %* % % %    *       d   ] ] ] ] ] ] ] ]@8 8 8 8 8 8 8 8v; ; ; ; ;~ ; ; ;nA nA nA nA nAn nA nA nA nA nAr'   