
    `i>                   h	   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z" dd	l#m$Z$m%Z% ddl&Z&ddl'Z&ddl(Z&ddl)m*c m+Z, dd
l-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9 ddl:m;Z; ddl<m=Z=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZD ddlEmFZG ddlHmIZImJZJ ddlKmLZL ddlMmNZNmOZO ddlPmQZQ ddlRmFZF ddlSmTZT ddlUmVZV  e	jW        eX          ZYe&jZ        j[        Z[e&jZ        j\        Z\eZ]e"e]e&jP        j^        f         Z_ej`        a                    dd          Zb G d  d!e           Zc G d" d#e           Zd G d$ d%e           Ze e!d&          Zfe"e&jP        jg        jh        eif         Zj G d' d(          Zk ek            Zl	 ddd2Zm G d3 d4          Zn G d5 d6eo          Zpe"enepf         Zqdd:Zr G d; d<          Zs G d= d>e          Zt G d? d@et          Zu G dA dBet          Zv G dC dDet          Zw G dE dFet          Zx G dG dHet          ZyezedIf         Z{ G dJ dKey          Z| G dL dMe|          Z} G dN dOe|          Z~ G dP dQe|          Z G dR dSey          Z G dT dUe          Z G dV dWe          Z G dX dYe          Z G dZ d[et          Z G d\ d]et          Z G d^ d_et          Z G d` da          Z G db dce           Zej         G dd de                      Zej         G df dge                      Zej         G dh die                      Zej         G dj dke                      ZddnZddsZ	 ddd{Zedd|ddtfddZ eD            Zded<   ddZ ee          j        dz  dz  Zg Zded<   edd|dtfddZ eGj        dt          	 	 ddd            Z	 	 dddZefdtdddZefdtdddZddZ ej        d          ZddZddZddZddZddZddZ G d d          ZddZ	 	 	 	 dddZ e&j                    dddddÄ            Z e&j                    ddĄ            ZddǄZddȄZddʄZdd̈́ZddτZ eD            Zded<   	 dddԄZdd؄ZddڄZdS )a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)
Collection	GeneratorIterableMappingSequence)Path)AnyCallableNoReturnOptionalProtocolTypeVarUnion)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fx)guard_or_falsestatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)preserve_node_meta)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_type#TORCHINDUCTOR_PATTERN_MATCH_BACKENDinductorc                  "    e Zd ZU ded<   d	dZdS )
SearchFnstr__name__argsr   kwargsreturnc                    d S N selfr5   r6   s      s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/_inductor/pattern_matcher.py__call__zSearchFn.__call__`             Nr5   r   r6   r   r7   r   )r4   
__module____qualname____annotations__r>   r:   r@   r=   r2   r2   ]   s'         MMM======r@   r2   c                      e Zd ZddZdS )	ReplaceFnr5   r   r6   r7   c                    d S r9   r:   r;   s      r=   r>   zReplaceFn.__call__d   r?   r@   NrA   r4   rB   rC   r>   r:   r@   r=   rF   rF   c   s        ======r@   rF   c                      e Zd Zd
dZd	S )TraceFnfnUnion[SearchFn, ReplaceFn]r5   r   r6   r7   torch.fx.GraphModulec                    d S r9   r:   )r<   rK   r5   r6   s       r=   r>   zTraceFn.__call__h   s    "sr@   N)rK   rL   r5   r   r6   r   r7   rM   rH   r:   r@   r=   rJ   rJ   g   s(        # # # # # #r@   rJ   Tc                      e Zd ZddZdS )Multipler7   Nonec                @    dt                      vs| t          u sJ d S d S )NMULTIPLE)globalsrT   r<   s    r=   __init__zMultiple.__init__t   s-    **dh.>.>.>>**.>.>r@   Nr7   rR   )r4   rB   rC   rW   r:   r@   r=   rQ   rQ   s   s(        ? ? ? ? ? ?r@   rQ    new_metadict[str, Any]old_nodetorch.fx.Node	pass_namer3   r7   rR   c                   ddl m}m} t          j        j        dk    r|                     dg                                           }|                     ||||j	                             | 
                    d |j                                        D                        || d<   n6| 
                    d |j                                        D                        d|j        v r|j        d         | d<   d S d S )Nr   )
NodeSourceNodeSourceActionr,   	from_nodec              3  T   K   | ]#\  }}|t           j        j        j        v ||fV  $d S r9   torchfxproxy_COPY_META_FIELDS.0kvs      r=   	<genexpr>z!_transfer_meta.<locals>.<genexpr>   H       
 
1EHN444 F4444
 
r@   c              3  T   K   | ]#\  }}|t           j        j        j        v ||fV  $d S r9   rd   ri   s      r=   rm   z!_transfer_meta.<locals>.<genexpr>   rn   r@   stack_trace)torch.fx.tracebackr`   ra   r%   traceprovenance_tracking_levelgetcopyappendREPLACEupdatemetaitems)rZ   r\   r^   r`   ra   new_from_nodes         r=   _transfer_metar|   }   s1    @???????
 |-22 ["55::<<ZZ)=M=UVVWWW 
 
 ++--
 
 
 	
 	
 	

 !. 
 
 ++--
 
 
 	
 	
 	

 %%"*-"> &%r@   c                       e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   	 	 d,d- fdZed.d            Zd/dZd0dZ	d1dZ
d2dZd3dZd4d!Zd5d#Z	 	 d6d7d+Z xZS )8Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	list[Any]r5   r[   r6   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxzOptional[torch.fx.GraphModule]replacement_graphNOptional[Sequence[Any]]Optional[dict[str, Any]]r7   rR   c                    t                                                       || _        t          |pg           | _        |pi | _        g | _        i | _        || _        d | _	        d S r9   )
superrW   r   listr5   r6   r   r   r   r   )r<   r   r   r5   r6   	__class__s        r=   rW   zMatch.__init__   sc     	$$	l
!%r@   torch.fx.Graphc                    | j         j        S r9   )r   graphrV   s    r=   r   zMatch.graph   s    x~r@   otherc                   | j         r|t          | j                                                   t          |j                                                   z  D ].}| j         |         |j         |         k    rt          d|          /| j                            |j                   | j                            |j                   | j                             |j                    | j                            |j                   d S )Nzkwarg mismatch: {})	r6   r"   keysFailedMatchr5   extendr   rx   r   )r<   r   keys      r=   r   zMatch.extend   s    ; 	A!$+"2"2"4"455
5<CTCTCVCV8W8WW A A;s#u|C'888%&:C@@@ 9	$$$
%+&&&5<(((EM*****r@   c                L    | j         rt          | j                   gng | _         | S r9   )r5   tuplerV   s    r=   bundlezMatch.bundle   s'    *.);U49%%&&	r@   r3   c                (    d| j          d| j         dS )NzMatch(..., , )r5   r6   rV   s    r=   __repr__zMatch.__repr__   s    8TY88$+8888r@   c                    | j         }t          | j                  D ]%}|j        s|j        s|                    |           &d S r9   )r   reversedr   _erasedusers
erase_node)r<   r   ns      r=   erase_nodeszMatch.erase_nodes   sT    
$*%% 	$ 	$A9 $QW $  ###	$ 	$r@   list[Optional[torch.fx.Node]]c                4      fd j         j        D             S )Nc                <    g | ]}|j         j        |         nd S r9   )r   pattern_to_node)rj   pr<   s     r=   
<listcomp>z&Match.output_nodes.<locals>.<listcomp>   s<     
 
 
 -.MTX%a((t
 
 
r@   )r   outputsrV   s   `r=   output_nodeszMatch.output_nodes   s3    
 
 
 
X%
 
 
 	
r@   r]   c                X    t          d |                                 D                       S )Nc              3     K   | ]}||V  	d S r9   r:   )rj   r   s     r=   rm   z$Match.output_node.<locals>.<genexpr>   s'      88!a8A888888r@   )nextr   rV   s    r=   output_nodezMatch.output_node   s+    88t0022888888r@   Sequence[Any]c                T    t                               | | j        j        ||           d S r9   )ReplacementPatternEntryreplace_with_graphr   r   )r<   r   r5   s      r=   r   zMatch.replace_with_graph   s4     	 22$(."3T	
 	
 	
 	
 	
r@   Treplacement_fnrF   trace_fnOptional[TraceFn]run_functional_passesboolc                   ddl m}m} t          |j        |          r|j        |j        nt          j                    }dd}|5  |t          j        t          |	          } || j
                  rGi | j
        d         j        d
         \  }	}
i |
}
t          | j                  | j        }}dfd}t          j        j                            |||f|	|
f           t          j                            |fd          } |||          }t          j                            |d           } |||          }t+          |j        j
                  t+          |j        j
                  k    sJ t/          |j        j
        |j        j
                  D ]#\  }}d
|j        v r|j        d
         |j        d
<   $n-t          j                            |d           } |||          }t+          | j
                  dk    r1|j        j
        D ]$}t1          |j        | j
        d         d           %t2                              | | j        j        ||           ddd           dS # 1 swxY w Y   dS )a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVNr   r   r7   r   c                    t          |           dk    rdS | d         }d|j        vrdS |j        t          t          j        j        j        t          j        j        j        t          j        j        j	        g          v S )Nr,   Fr   eager_input_vals)
lenry   targetr"   re   opshigher_order triton_kernel_wrapper_functionalauto_functionalizedauto_functionalized_v2)r   nodes     r=   !should_propagate_eager_input_valszCMatch.replace_by_example.<locals>.should_propagate_eager_input_vals  sr    5zzQu8D!22u;*I*KI*>I*A# #  r@   )r   r   r   r]   valr   rR   c                T    t          | t          j        j                  r|| <   d S d S r9   )
isinstancere   rf   Node)r   r   node_to_vals     r=   recordz(Match.replace_by_example.<locals>.record%  s3    !$66 0,/D)))0 0r@   c                    |          S r9   r:   )argr   s    r=   <lambda>z*Match.replace_by_example.<locals>.<lambda>-  s    +cBR r@   c                    | j         d         S Nr   ry   r   s    r=   r   z*Match.replace_by_example.<locals>.<lambda>3  s    #(5/ r@   c                H    d| j         v r| j         d         n| j         d         S )Nr   example_valuer   r   s    r=   r   z*Match.replace_by_example.<locals>.<lambda>H  s+    (( !$/2 r@   r,   replace_by_examplerZ   r\   r^   )r   r   r7   r   )r   r]   r   r   r7   rR   )torch._inductor.virtualizedr   r   r   	fake_mode
contextlibnullcontext	functoolspartialfwd_onlyr   ry   r   r5   r6   re   utils_pytreetree_maprf   map_argr   r   zipr|   r   r   r   )r<   r   r5   r   r   r   r   contextr   	fake_argsfake_kwargs
match_argsmatch_kwargsr   example_valsgraph_with_eager_valsreplacementr\   new_noder   r   s                       @r=   r   zMatch.replace_by_example   sk    	?>>>>>>> q{K88*=>[=P KK')) 		 	 	 	  I	 I	$,4I   10<< 5E !)-A);<N)O&	;-o+0+;+;T[L
0 0 0 0 0 0 #,,Z6K8P    %x//6R6R6R6RSS )1(N(N%  %x//6Q6QRR&h'<lKK 06<==%+B B     +.)/5{7H7N+ +  &Hh *X]::<DM.=&89	  %x//3 3    'h~|DD4:!##$*0  A"!"!%A"6     $66	  II	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	 I	s   HI&&I*-I*)NN)
r   r   r   r   r5   r   r6   r   r7   rR   )r7   r   )r   r~   r7   rR   )r7   r~   r7   r3   rX   )r7   r   )r7   r]   )r   r   r5   r   r7   rR   NT)
r   rF   r5   r   r   r   r   r   r7   rR   )r4   rB   rC   __doc__rD   rW   propertyr   r   r   r   r   r   r   r   r   __classcell__r   s   @r=   r~   r~      s        	 	 OOO44445555 )-+/& & & & & & &&    X+ + + +   
9 9 9 9$ $ $ $
 
 
 
9 9 9 9
 
 
 
 '+&*n n n n n n n n nr@   r~   c                  6    e Zd ZU dZded<   dd	Zdd
ZddZdS )r   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    r3   format_stringr5   r   r6   r7   rR   c                z    || _         t          |          dk    rt          d|           || _        || _        d S )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r   r   RuntimeErrorr5   r6   )r<   r   r5   r6   s       r=   rW   zFailedMatch.__init__g  sQ    * }##xivxx   	r@   c                :     | j         j        | j        i | j        S r9   )r   formatr5   r6   rV   s    r=   __str__zFailedMatch.__str__r  s!    (t!($)Ct{CCCr@   r   c                    dS NFr:   rV   s    r=   __bool__zFailedMatch.__bool__u      ur@   N)r   r3   r5   r   r6   r   r7   rR   r   r7   r   )r4   rB   rC   r   rD   rW   r   r   r:   r@   r=   r   r   ]  sn           	 	 	 	D D D D     r@   r   mMatchResultTypeIs[Match]c                     t          |           S )z|
    TypeIs cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeIs.
    )r   )r  s    r=   is_matchr  |  s    
 77Nr@   c                  X    e Zd ZU dZded<   ded<   ded<   ded	<   	 dddZddZddZd
S )r   zC
    Internal state needed while running PatternExpr._match().
    list[Optional[PatternExpr]]r   z*dict[PatternExpr, Optional[torch.fx.Node]]r   r   r   zlist[NodeOrConstant]exclusive_node_setN*Optional[dict[PatternExpr, torch.fx.Node]]r7   rR   c               `    || _         |i nt          |          | _        || _        g | _        d S r9   )r   dictr   r   r	  )r<   r   r   r   s       r=   rW   zMatchContext.__init__  s9     %4%<rr$BWBW
"$r@   r   r   r   NodeOrConstantr  c                    || j         v r0| j         |         |k    rt          | |          S t          d          S |                    ||           }|| j         vsJ |r|nd| j         |<   |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r~   r   _match)r<   r   r   r  s       r=   matchzMatchContext.match  s    d***#G,44T7+++"#=>>>NN4&&d2222201(;tW%r@    dict[PatternExpr, torch.fx.Node]c                H    d | j                                         D             S )Nc                F    i | ]\  }}|                                 r|||S r9   )has_multiple_users)rj   r   r   s      r=   
<dictcomp>z;MatchContext.filter_multi_user_patterns.<locals>.<dictcomp>  sD     
 
 
))++
 150@ T0@0@0@r@   )r   rz   rV   s    r=   filter_multi_user_patternsz'MatchContext.filter_multi_user_patterns  s2    
 
!%!5!;!;!=!=
 
 
 	
r@   r9   )r   r  r   r
  r   r   r7   rR   )r   r   r   r  r7   r  )r7   r  )r4   rB   rC   r   rD   rW   r  r  r:   r@   r=   r   r     s           )(((????,,,,
 GK
% 
% 
% 
% 
%
 
 
 

 
 
 
 
 
r@   r   c                  R    e Zd ZdZedd            Zdd	ZddZddZddZ	ddZ
dS )r   z+
    Base class for types of patterns.
    r   r]   r   r   r7   r  c                    d S r9   r:   r<   r   r   s      r=   r  zPatternExpr._match  s    MPSr@   c                    	 t          | g|j                                      | |          S # t          $ r}|cY d }~S d }~ww xY wNr   )r   r   r  r   r<   r   es      r=   r  zPatternExpr.match  s\    	dj999??dKKK 	 	 	HHHHHH	s   *- 
A>AAr   c                    dS r   r:   rV   s    r=   r  zPatternExpr.has_multiple_users  r   r@   r3   c                     | j         j        dz   S )Nz())r   r4   rV   s    r=   r   zPatternExpr.__repr__  s    ~&--r@   searchedOrderedSet[torch.fx.Node].Generator[Optional[torch.fx.Node], None, None]c              #  >   K   | |j         v r|j         |          V  d S d S r9   )r   r<   r   r!  s      r=   find_anchor_nodeszPatternExpr.find_anchor_nodes  s8       3&&&%d++++++ '&r@   r   r   c                ,    t          || j                  S )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )r   r   )r<   r   s     r=   
pattern_eqzPatternExpr.pattern_eq  s     %000r@   Nr   r]   r   r   r7   r  r   r]   r7   r  r  r   r   r   r!  r"  r7   r#  r   r   r7   r   )r4   rB   rC   r   r   r  r  r  r   r&  r(  r:   r@   r=   r   r     s          PPP ^P      . . . ., , , ,1 1 1 1 1 1r@   r   c                      e Zd ZdZd
dZd	S )Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    r   r  r   r   r7   r  c                (    t          || |g          S )N)r5   r~   r  s      r=   r  z
Arg._match  s    S$dV,,,,r@   Nr   r  r   r   r7   r  r4   rB   rC   r   r  r:   r@   r=   r.  r.    s2         
- - - - - -r@   r.  c                  *    e Zd ZdZddZdd
ZddZdS )Ignoredz4
    Match an arg, but don't pass it to handler
    r   r  r   r   r7   r  c                "    t          ||           S r9   r0  r  s      r=   r  zIgnored._match  s    S$r@   r3   c                    dS )N*r:   rV   s    r=   r   zIgnored.__repr__  s    sr@   ppPatternPrettyPrinterc                    dS )Nz	Ignored()r:   )r<   r8  s     r=   pretty_printzIgnored.pretty_print  s    {r@   Nr1  r   r8  r9  r7   r3   )r4   rB   rC   r   r  r   r;  r:   r@   r=   r4  r4    sZ                        r@   r4  c                  @     e Zd ZdZd fdZddZddZd fdZ xZS )
KeywordArgD
    Capture a kwarg which will become an input to the handler.
    namer3   r7   rR   c                V    t                                                       || _        d S r9   r   rW   r@  r<   r@  r   s     r=   rW   zKeywordArg.__init__  $    			r@   c                    d| j         dS )NzKeywordArg(r   r@  rV   s    r=   r   zKeywordArg.__repr__  s    +TY++++r@   r   r  r   r   r  c                4    t          || | j        |i          S )Nr6   )r~   r@  r  s      r=   r  zKeywordArg._match  s    S$	4'89999r@   r   r   r   c                    t          j        t          |          }t                                          |          o| j        |j        k    S r9   typingcastr   r   r(  r@  r<   r   r   s     r=   r(  zKeywordArg.pattern_eq  :    D%((ww!!%((DTY%*-DDr@   r@  r3   r7   rR   r   r1  r,  	r4   rB   rC   r   rW   r   r  r(  r   r   s   @r=   r>  r>    s              , , , ,: : : :E E E E E E E E E Er@   r>  c                  L     e Zd ZU dZded<   d fdZddZddZd fdZ xZ	S )ExclusiveKeywordArgr?  r3   r@  r7   rR   c                V    t                                                       || _        d S r9   rB  rC  s     r=   rW   zExclusiveKeywordArg.__init__  rD  r@   c                    d| j         dS )NzExclusiveKeywordArg(r   rF  rV   s    r=   r   zExclusiveKeywordArg.__repr__	  s    4di4444r@   r   r  r   r   r  c                    ||j         v rt          d          S |j                             |           t          || | j        |i          S )Nzexclusive arg appears twicerH  )r	  r   rv   r~   r@  r  s      r=   r  zExclusiveKeywordArg._match  sR    3)))<===%%d+++S$	4'89999r@   r   r   r   c                    t          j        t          |          }t                                          |          o| j        |j        k    S r9   rJ  rM  s     r=   r(  zExclusiveKeywordArg.pattern_eq  rN  r@   rO  r   r1  r,  )
r4   rB   rC   r   rD   rW   r   r  r(  r   r   s   @r=   rR  rR    s           III     5 5 5 5: : : :E E E E E E E E E Er@   rR  c                       e Zd ZU dZded<   ded<   	 d d! fdZeed"d                        Zd"dZ	d"dZ
d#dZd$dZd%dZd&dZd' fdZ xZS )(_TargetExprz7
    Base class for filtering match by node.target
    zlist[FnsType]fnszOrderedSet[FnsType]fns_setr,   !Union[FnsType, Sequence[FnsType]]r   Union[Multiple, int]r7   rR   c                   t                                                       t          |          st          |t                    r|gnt          |          }|D ]Tt          t          j        j                  r3|	                    fd
                                D                        U|| _        t          |          | _        || _        d S )Nc              3  8   K   | ]}t          |          V  d S r9   getattrrj   overloadrK   s     r=   rm   z'_TargetExpr.__init__.<locals>.<genexpr>'  s-      PPX72x00PPPPPPr@   )r   rW   callabler   r3   r   re   _opsOpOverloadPacketr   	overloadsrY  r"   rZ  r   )r<   rY  r   rK   r   s      @r=   rW   z_TargetExpr.__init__   s     	}}K
3(<(<Ksee$s)) 	Q 	QB"ej9:: Q

PPPPPPPPPP!#


r@   r3   c                    d S r9   r:   rV   s    r=   opz_TargetExpr.op-  s    r@   c                   | j         d         }t          |t                    s|j        }t	          | j                   dk    rd| dS | j         d         t          t          |d           u rd| S | j         d         t          t          |d           u rd| S t          | j         d         t          j        j	                  rt          | j         d                   S |S )Nr   r,   [z, ...]torch.z	operator.)
rY  r   r3   r4   r   r`  re   operatorrd  
OpOverload)r<   
first_reprs     r=   fns_reprz_TargetExpr.fns_repr1  s    Xa[
*c** 	-#,Jtx==1)z))))Xa[GE:t<<<<(J(((Xa[GHj$????+z+++UZ%:;; 	tx{###r@   c                    | j         t          u rd}n| j         dk    rd| j          d}nd}| j        j         d|                                  | dS )Nz
, MULTIPLEr,   r   r   rY   ()r   rT   r   r4   ro  )r<   comma_userss     r=   r   z_TargetExpr.__repr__A  sc    :!!&KKZ1__,tz,,,KKK.)KKDMMOOK[KKKKr@   r   c                L    t          | j        t                    p
| j        dk    S )Nr,   )r   r   rQ   rV   s    r=   r  z_TargetExpr.has_multiple_usersJ  s    $*h//A4:>Ar@   r   r   r!  r"  r#  c                    t           r9   NotImplementedErrorr%  s      r=   r&  z_TargetExpr.find_anchor_nodesM  s
     "!r@   r   r]   c                    t          |t          j        j                  o%|j        | j        k    ot          |          | j        v S r9   )r   re   rf   r   rh  extract_targetrZ  )r<   r   s     r=   
_match_fnsz_TargetExpr._match_fnsR  s@    tUX]++ 547"5t$$4	
r@   c                j    | |j         v p*| j        t          u pt          |j                  | j        k    S r9   )r   r   rT   r   r  s      r=   _match_usersz_TargetExpr._match_usersY  s7    CK -zX%-4:$*,	
r@   r   r   c                    t          j        t          |          }t                                          |          o/| j        |j        k    o| j        |j        k    o| j        |j        k    S r9   )rK  rL  r   r   r(  rh  rY  r   rM  s     r=   r(  z_TargetExpr.pattern_eq`  sc    D%((GGu%% *58#*EI%* 
ek)		
r@   )r,   )rY  r[  r   r\  r7   rR   r   r  r+  r   r]   r7   r   )r   r]   r   r   r7   r   r,  )r4   rB   rC   r   rD   rW   r   r   rh  ro  r   r  r&  ry  r{  r(  r   r   s   @r=   rX  rX    s/               UV        ^ X    L L L LB B B B" " " "

 
 
 

 
 
 

 
 
 
 
 
 
 
 
 
r@   rX  .c                       e Zd ZdZddd% fdZed&d            Zed&d            Zd'dZd(dZ	d)dZ
d*d!Zd+ fd$Z xZS ),_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r,   )_usersrY  /Union[torch.fx.node.Target, str, Sequence[Any]]r5   r   r  Union[int, Multiple]r6   r7   rR   c                  t                                          ||           t          |          | _        t	          |          | _        t          d t          j        ||	                                          D                       r| j
        | _        n| j        | _        |                     | j        | j                  | _        d S )Nc              3  Z   K   | ]&}t          |t          t          t          f          V  'd S r9   )r   r  r   r   rj   xs     r=   rm   z+_TargetArgsExpr.__init__.<locals>.<genexpr>|  sG       
 
 q4u-..
 
 
 
 
 
r@   )r   rW   r   r5   r  r6   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)r<   rY  r  r5   r6   r   s        r=   rW   z_TargetArgsExpr.__init__r  s     	f%%%$KK	6ll 
 
_T6==??;;
 
 
 
 
 	/  .DLL.DL $TY D Dr@   r   Mapping[Any, Any]9tuple[Sequence[Any], Union[_SimpleSpec, pytree.TreeSpec]]c                    g | |                                 R }t          |           g|                                R }||fS r9   )r  r   r   )r5   r6   r  specs       r=   r  z_TargetArgsExpr.simple_flatten  sF     +4*&--//**D		*FKKMM**t|r@   c                    t           t          t          t          t          t          idfdt          j        | |ffd          }t          j        |          \  }}||fS )Nr  r   r7   c                    t          |           }                    |          }|#t          j         ||           fd          S | S )Nc                &    t          |           v S r9   typer  type_mappings    r=   r   zF_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>  s    d1gg&= r@   is_leaf)r  rt   pytreer   )r  cls
convert_fnconvert_typer  s      r=   r  z4_TargetArgsExpr.pytree_flatten.<locals>.convert_type  sa    q''C%))#..J% JqMM====   
 Hr@   c                &    t          |           v S r9   r  r  s    r=   r   z0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>  s    d1gg5 r@   r  )r  r   r7   r   )r   r   r   r   r  r  r   tree_flatten)r5   r6   normalized_args_treeflatr  r  r  s        @@r=   r  z_TargetArgsExpr.pytree_flatten  s    
 E%D*
		 		 		 		 		 		 		  &6N5555 
  
  

 ()=>>
dTzr@   r3   c                   |                                  gt          t          | j                  d | j                                        D             }| j        t          u r|                    d           n(| j        dk    r|                    d| j                    | j	        j
         dd                    |           dS )Nc                "    g | ]\  }}| d | S =r:   ri   s      r=   r   z,_TargetArgsExpr.__repr__.<locals>.<listcomp>  s&    999TQjjQjj999r@   _users=MULTIPLEr,   _users=rq  r   r   )ro  mapreprr5   r6   rz   r   rT   rv   r   r4   join)r<   r5   s     r=   r   z_TargetArgsExpr.__repr__  s    MMOO
ty!!
 :9T[%6%6%8%8999

 :!!KK)****Z1__KK.$*..///.)>>DIIdOO>>>>r@   r8  r9  c                   |                                  gfd| j        D             fd| j                                        D             }| j        t
          u r|                    d           n(| j        dk    r|                    d| j                    d}| j        j         d|	                    |           dS )	Nc              3  B   K   | ]}                     |          V  d S r9   r;  rj   r  r8  s     r=   rm   z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>  s/      44Qbooa  444444r@   c                J    g | ]\  }}| d                      |            S r  r  )rj   rk   rl   r8  s      r=   r   z0_TargetArgsExpr.pretty_print.<locals>.<listcomp>  s7    JJJda))R__Q''))JJJr@   r  r,   r  r   rq  r   )
ro  r5   r6   rz   r   rT   rv   r   r4   r  )r<   r8  r5   
joiner_strs    `  r=   r;  z_TargetArgsExpr.pretty_print  s    MMOO
4444$)444
 KJJJdk6G6G6I6IJJJ

 :!!KK)****Z1__KK.$*..///
.)DDJOOD,A,ADDDDr@   r   r]   r   r   r  c                                          |          r*t          |j                  t           j                  k    rt          d|           S                      ||          st          d           S |j        }|j        t                    t           j                  k     rddlm} t          |j	                  sJ  ||j	        |j        |j                  }|t          d|           S |\  }t          |          t           j                  k    r5t                    t           j                  k    r fdD             n t          d|           S  fdD              
                    |          \  }} j        \  }}	||	k    rt          d||	          S t          |          t          |          k    sJ t          |           }
t          t          j                    ||          D ]\  }}}t!          |t"                    r?|                    ||          }t'          |          s|c S |
                    |           Zt!          |t*          j        j                  s||k    rt          d||          c S |
j                            |           |j	        |
j         <   |
S )	Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionc                4    i | ]}|j         v ||         S r:   rH  rj   i_kwargsr<   s     r=   r  z*_TargetArgsExpr._match.<locals>.<dictcomp>  s+    RRRdkAQAQq'!*AQAQAQr@   c                4    i | ]}|j         v ||         S r:   rH  r  s     r=   r  z*_TargetArgsExpr._match.<locals>.<dictcomp>  s+    JJJdk9I9Iq'!*9I9I9Ir@   zargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})ry  r   r5   r   r{  r6   torch.fx.operator_schemasr  rc  r   r  r  r~   r   r  countr   r   r  r  r   re   rf   r   r   rv   r   )r<   r   r   _argsr  normalized_args_and_kwargs
node_items	node_spec
self_items	self_specr  r  r   
child_nodechild_matchr  s   `              @r=   r  z_TargetArgsExpr._match  s   t$$ 	UDI#di..(H(HGtTTT  s++ 	:2D999	+w<<#dk****DDDDDDDK(((((););TY* *& *1"#KTSWXXX!;wu::TY//CLLCDTDT4T4TRRRRRgRRRGG&@$   KJJJJgJJJG $UG < <
I $ 5
I	!!5y)LLL:#j//1111#t&))/*;*;Z&T&T 		 		"Aw
';// !ii<<,, '&&&&%%%%J66 *:O:O"94     ;P 	
t+	$r@   r!  r"  r#  c              #  r  K   | |j         v r|j         |          V  dS | j        d         D ]}t          |t                    ru|                    ||          D ]^}t          |t
          j        j                  s"|j        D ]4}||vr.| 	                    |          r|V  |
                    |           5_dS )a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   r  r   r   r&  re   rf   r   r   ry  add)r<   r   r!  r   
other_noder   s         r=   r&  z!_TargetArgsExpr.find_anchor_nodes  s       3&&&%d++++F,Q/ 		3 		3G';// 3")";";C"J"J 3 3J%j%(-@@ !  * 0 3 3x//#t44 3&*


 (T 2 2 2	3		3 		3r@   r   r   c                ,   t          j        t          |          }t                                          |          oX| j        d         |j        d         k    o<t          d t          | j        d         |j        d                   D                       S )Nr,   c              3  |   K   | ]7\  }}t          |t                    r|                    |          n||k    V  8d S r9   r   r   r(  rj   abs      r=   rm   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>  [        Aq $.a#=#=IQ16     r@   r   )rK  rL  r   r   r(  r  allr   rM  s     r=   r(  z_TargetArgsExpr.pattern_eq  s    D%((GGu%% %a(E,B1,EE   5a 8%:PQR:STT    	
r@   )
rY  r  r5   r   r  r  r6   r   r7   rR   )r5   r   r6   r  r7   r  r   r<  r)  r+  r,  )r4   rB   rC   r   rW   staticmethodr  r  r   r;  r  r&  r(  r   r   s   @r=   r  r  m  s         ()	E E E E E E E E&    \    \8
? 
? 
? 
?E E E E1 1 1 1f3 3 3 32	
 	
 	
 	
 	
 	
 	
 	
 	
 	
r@   r  c                      e Zd ZdZdZdS )CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNr4   rB   rC   r   rh  r:   r@   r=   r  r    s          
BBBr@   r  c                      e Zd ZdZdZdS )
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr  r:   r@   r=   r  r  $            
BBBr@   r  c                      e Zd ZdZdZdS )
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr  r:   r@   r=   r  r  ,  r  r@   r  c                      e Zd ZdZd
dZd	S )_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    r   r]   r   r   r7   r  c                   |                      |          st          d          S |                     ||          st          d          S t          ||           }|j                            |           |j        |j        | <   |j        	                    |j                   |j
                            |j
                   |S )Nfunction_mismatchmultiple_users)ry  r   r{  r~   r   rv   r   r   r5   r   r6   rx   )r<   r   r   r  s       r=   r  z_TargetExprVarArgs._match9  s    t$$ 	42333  s++ 	1/000#t	t+	$	di   	$$$r@   Nr)  r2  r:   r@   r=   r  r  4  s2              r@   r  c                      e Zd ZdZdS )CallFunctionVarArgsr  Nr4   rB   rC   rh  r:   r@   r=   r  r  H  s        	BBBr@   r  c                      e Zd ZdZdS )CallMethodVarArgsr  Nr  r:   r@   r=   r  r  L          	BBBr@   r  c                      e Zd ZdZdS )CallModuleVarArgsr  Nr  r:   r@   r=   r  r  P  r  r@   r  c                  B     e Zd ZdZdd fd	ZddZddZd fdZ xZS )ListOfz$
    Matches a repeated pattern
    Fr   r   r   r   r7   rR   c                    t                                                       t          |t                    sJ || _        || _        d S r9   )r   rW   r   r   r   r   )r<   r   r   r   s      r=   rW   zListOf.__init__Y  sA    ';/////r@   r3   c                0    | j         j         d| j         dS Nrq  r   )r   r4   r   rV   s    r=   r   zListOf.__repr___       .);;DL;;;;r@   r   r   r   r   r  c                z   t          |t          t          f          rt          |          dk    rt	          d          S t          ||           }|                                }d}t          |          D ]\  }}t          |j	        ||j
                  }|                    | j        |          }	|                                }t          |	          s| j        st	          d||	          c S zd}|                    |	                                           |st	          d          S |                                S )Nr   non_listFr  zlist[{}]: {}Tzlist: no_match)r   r   r   r   r   r~   r  	enumerater   r   r   r  r   r  r   r   r   )
r<   r   r   r  r   matchedr  r  	child_ctxr  s
             r=   r  zListOf._matchb  s=   $u.. 	+#d))q..z***#t 88::&t__ 	+ 	+MAz$_J4D  I $//$,
CCK'BBDDOK(( | G&~q+FFFFFGHH[''))**** 	1/000xxzzr@   r   r   c                    t          j        t          |          }t                                          |          o.| j                            |j                  o| j        |j        k    S r9   )rK  rL  r   r   r(  r   r   rM  s     r=   r(  zListOf.pattern_eqz  sY    D%((GGu%% .''66.-	
r@   F)r   r   r   r   r7   rR   r   )r   r   r   r   r7   r  r,  rP  r   s   @r=   r  r  T  s               < < < <   0
 
 
 
 
 
 
 
 
 
r@   r  c                  x     e Zd ZU ded<   d fdZedd            Zdd
ZddZd dZ	d!dZ
d"dZd# fdZ xZS )$MultiOutputPatternr  r   Sequence[Optional[PatternExpr]]r7   rR   c                   t                                                       t          |d         t                    sJ t	          d |D                       s
J |            t          |          | _        |d         j        | _        d S )Nr   c              3  H   K   | ]}|d u pt          |t                    V  d S r9   )r   r   r  s     r=   rm   z.MultiOutputPattern.__init__.<locals>.<genexpr>  s5      LLq19:
1k : :LLLLLLr@   )r   rW   r   rX  r  r   r   rh  )r<   r   r   s     r=   rW   zMultiOutputPattern.__init__  s|    '!*k22222LLGLLLLLUUgUULG}}!*-r@   -Union[Callable[..., Any], str, Sequence[Any]]c                Z    t          j        t          | j        d                   }|j        S Nr   )rK  rL  rX  r   rY  )r<   outputs     r=   rY  zMultiOutputPattern.fns  s#     [$,q/::zr@   r3   c                0    | j         j         d| j         dS r  )r   r4   r   rV   s    r=   r   zMultiOutputPattern.__repr__  r  r@   r8  r9  c                    fd| j         D             }dd }| j        j         d|                    |           }| d}|S )Nc                :    g | ]}                     |          S r:   r  r  s     r=   r   z3MultiOutputPattern.pretty_print.<locals>.<listcomp>  s%    999q""999r@   z,
z  z([z
]))r   r   r4   r  )r<   r8  r5   r  str_outs    `   r=   r;  zMultiOutputPattern.pretty_print  s`    9999DL999!4\\
^,GG
0E0EGG"""r@   r   r]   r   r   r  c                D   t          j        t          | j        d                   }|                    ||          }t          |          s|S | j        dd          D ]C}||                     ||          }t          |          s|c S |                    |           D|S )Nr   r,   )rK  rL  rX  r   r  r  _match_from_anchorsr   )r<   r   r   r  r  r   r  s          r=   r  zMultiOutputPattern._match  s    [$,q/::IIfd##{{ 	H|ABB' 	" 	"G227C@@KK(( #""""HH[!!!!r@   r   r   c                   t          |j                  }t          d          }|                    |t	                                D ]?}|                    ||          }t          |          r|c S t          |          |_        @|S )Nzno anchor found)r  r   r   r&  r"   r  r  )r<   r   r   priorr  r   s         r=   r  z&MultiOutputPattern._match_from_anchors  s     S())$%677--c:<<@@ 	. 	.D		'4((A{{ "&u++Cr@   c                    	 t          | j        |j                                      | |          S # t          $ r}|cY d }~S d }~ww xY wr  )r   r   r   r  r   r  s      r=   r  zMultiOutputPattern.match  s\    	DJ???EEdDQQQ 	 	 	HHHHHH	s   .1 
AAAAr   r   r   c                0   t          j        t          |          }t                                          |          oZt          | j                  t          |j                  k    o0t          d t          | j        |j                  D                       S )Nc              3  |   K   | ]7\  }}t          |t                    r|                    |          n||k    V  8d S r9   r  r  s      r=   rm   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>  r  r@   )	rK  rL  r   r   r(  r   r   r  r   rM  s     r=   r(  zMultiOutputPattern.pattern_eq  s    D%((GGu%% DL!!S%7%77  em<<    	
r@   )r   r  r7   rR   )r7   r  r   r<  r)  )r   r   r   r   r7   r  r*  r,  )r4   rB   rC   rD   rW   r   rY  r   r;  r  r  r  r(  r   r   s   @r=   r  r    s         ((((               X
< < < <             	
 	
 	
 	
 	
 	
 	
 	
 	
 	
r@   r  c                  P     e Zd ZdZd fdZedd            ZddZd fdZ xZ	S )RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    inner_patternrX  r7   rR   c                n    t                                                       || _        |j        | _        d S r9   )r   rW   r  rh  )r<   r  r   s     r=   rW   zRepeatedExpr.__init__  s/    *"r@   Sequence[FnsType]c                    | j         j        S r9   )r  rY  rV   s    r=   rY  zRepeatedExpr.fns  s    !%%r@   r   r]   r   r   r  c                   |                     | j        |          }t          |          s|S |j                            | j                   | j                            |t                                D ]Z}t          | g|j                                       | j        |          }t          |          s|c S |	                    |           [|S r  )
r  r  r  r   popr&  r"   r   r   r   )r<   r   r   r  anchor_nodeanchor_ms         r=   r  zRepeatedExpr._match  s    IId($//{{ 	H	
 	
 	
  -??Z\\RR 	 	K#TF$*===CC"K H H%%  HHXr@   r   r   r   c                    t          j        t          |          }t                                          |          o| j                            |j                  S r9   )rK  rL  r   r   r(  r  rM  s     r=   r(  zRepeatedExpr.pattern_eq  sN    D%((ww!!%(( 
T-?-J-J.
 .
 	
r@   )r  rX  r7   rR   )r7   r  r)  r,  )
r4   rB   rC   r   rW   r   rY  r  r(  r   r   s   @r=   r
  r
    s         # # # # # #
 & & & X&   "
 
 
 
 
 
 
 
 
 
r@   r
  c                  ^    e Zd ZdZddZeej        ddd
                        ZddZ	ddZ
dS )r9  z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    r7   rR   c                r    t           j        j                                        | _        i | _        i | _        d S r9   )re   rf   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_pprV   s    r=   rW   zPatternPrettyPrinter.__init__  s/    2244;= 8:r@   r  objr   output_namer3   c                    t                      t          | d          sJ |                               }fdj        D             }|                    | d|            d                    |          S )zU
        Serializes obj to python code with obj written out to `output_name`
        r;  )r8  c                J    g | ]}j         |          d j        |           S ) = )r  r  )rj   r   r8  s     r=   r   z,PatternPrettyPrinter.run.<locals>.<listcomp>	  sG     
 
 
 %c*IIr/B3/GII
 
 
r@   r  
)r9  hasattrr;  r  rv   r  )r  r  out_strr  r8  s       @r=   runzPatternPrettyPrinter.run  s     "##sN+++++""b"))
 
 
 
-
 
 

 	2222333yy   r@   r   c                    t          |t                    r3| j                            |          x}r|S |                     |          S t          |d          r|                    |           S t          |          S )Nr;  )r   r  r  rt   memoizer   r;  r  )r<   r  memoized_names      r=   r;  z!PatternPrettyPrinter.pretty_print  s|    c?++ 	) $ 8 < <S A AA} )$$||C(((3'' 	*##D)))Cyyr@   r  c                    |                     |           }|                                }dD ]}|                    |d          }| j                            |d           }|| j        |<   || j        |<   |S )N)zaten.rk  zprims.rY   )r;  ro  replacer  create_namer  r  )r<   r  obj_strobj_nameprefixtmp_names         r=   r$  zPatternPrettyPrinter.memoize  s~    ""4((<<>>3 	4 	4F''33HH>--h==(0 %%,c"r@   NrX   )r  )r  r   r  r3   r7   r3   )r  r   r7   r3   )r  r  r7   r3   )r4   rB   rC   r   rW   r  r   cacher"  r;  r$  r:   r@   r=   r9  r9    s         ; ; ; ;
 _! ! ! ! _ \!$	 	 	 		 	 	 	 	 	r@   r9  c                      e Zd ZddZdS )_PassDictsTyperk    tuple[str, torch.fx.node.Target]r7   list[PatternEntry]c                    d S r9   r:   )r<   rk   s     r=   __getitem__z_PassDictsType.__getitem__*  s     Sr@   N)rk   r0  r7   r1  )r4   rB   rC   r3  r:   r@   r=   r/  r/  )  s(        ! ! ! ! ! !r@   r/  c                  :    e Zd ZU ded<   ded<   ddZ	 	 dddZdS )PatternEntryr   r   Callable[[Match], bool]extra_checkr  r~   r   r   r   r]   r7   rR   c                    t           r9   ru  r<   r  r   r   s       r=   applyzPatternEntry.apply4  s    !!r@   NF
pass_dicts/Union[_PassDictsType, Sequence[_PassDictsType]]r   !Union[torch.fx.node.Target, None]prependr   c                   |@t          | j        d          sJ | j        j        D ]}|                     |||           d S t	          |t
          t          f          rlt          | j        d          sJ |r*|| j        j        |f                             d|            d S || j        j        |f         	                    |            d S t          j        t          t                   |          }|D ]}|                     |||           d S )NrY  r>  rh  r   )r   r   rY  registerr   r  PatternMatcherPassrh  insertrv   rK  rL  r   r/  )r<   r;  r   r>  rK   r  s         r=   rA  zPatternEntry.register7  s/    >4</////l& ? ?j"g>>>>? ?
T+=$>?? 		:4<..... CDLOV45<<QEEEEEDLOV45<<TBBBBBXn%=zJJJ : :a9999: :r@   r  r~   r   r   r   r]   r7   rR   r   )r;  r<  r   r=  r>  r   r7   rR   )r4   rB   rC   rD   r:  rA  r:   r@   r=   r5  r5  /  sc         ((((" " " " 59	: : : : : : :r@   r5  c                  "    e Zd ZU ded<   ddZdS )LoweringPatternEntryCallable[..., Any]handlerr  r~   r   r   r   r]   r7   rR   c                    t          j        | j                  t          j        | j        |                    }|                    |          5  |                    |t          |j                  |j                  }|j	        
                    |j	                   |                    |           d d d            n# 1 swxY w Y   |j        d         |u sJ |                                 d S )N)r   wrapsrH  r   inserting_beforer  r   r5   r6   ry   rx   replace_all_uses_withr   r   )r<   r  r   r   rH  r   s         r=   r:  zLoweringPatternEntry.applyQ  s   /)/$,//	0A$,PU0V0VWW##D)) 	4 	4--guUZ7H7H%,WWK##DI...&&{333	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 {2$&&&&s   A#B??CCNrD  )r4   rB   rC   rD   r:  r:   r@   r=   rF  rF  M  s6              r@   rF  c                  &    e Zd ZU dZded<   ddZdS )GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    rG  rH  r  r~   r   r   r   r]   r7   rR   c                    |                     |          5   | j        |g|j        R i |j         d d d            d S # 1 swxY w Y   d S r9   )rL  rH  r5   r6   r9  s       r=   r:  zGraphPatternEntry.applyc  s    ##D)) 	= 	=DL<<<<u|<<<	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=s   ?AANrD  )r4   rB   rC   r   rD   r:  r:   r@   r=   rO  rO  [  sB            = = = = = =r@   rO  c                  :    e Zd ZU ded<   edd            ZddZdS )r   zCallable[..., list[Any]]normalize_argsr  r~   r   r   r   +Union[torch.fx.Graph, torch.fx.GraphModule]r5   Sequence[torch.fx.Node]r7   rR   c                    G fddt           j        j                  }|                                 }t	          |          dk    r	|d         }n`|d         sJ t          |d         j        j                  fd|D             }t          |t          j
        d                    d         }dd                    |          5  t          |t           j        j                  sJ   ||          j         }t          |t           j        j                  r|g}dddfdt	          |          t	          |          k    r#t!          ||          D ]\  }	}
 |	|
           n't	          |          dk    sJ  |d         |           d d d            n# 1 swxY w Y   |                                  d S )Nc                  0     e Zd ZdZdZdZd fdZ xZS )<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNr   r]   r7   r   c                V   |j         dv r!t                                          |          S |j        }|                     |          \  }}|j         dk    rt          |          sJ                     |||          }t          |j        |d           d|j        v r|j        d         |j        d<   d|j        v rcd|j        vrZ|j        d         |j        d<   t          |j        d         t          j                  r d|j        v sJ |j        d         |j        d<   |S |j         dk    rd	d
lm} t                                          |||          }t          |t          j        j                  st#          d| d| d          j        J d }j                                        D ]\  }	}
||
u r|	} n|Ft          |t(                    sJ  |j        |          \  }}j                            ||                               |          S t#          d|           )N)placeholderr  r  Interpreter_Replacerr   r   r   tensor_metaget_attrr   )unique_graph_name_with_rootzNYI: replacement_graph.z is not a graph module. Got .z
unhandled )rh  r   run_noder   fetch_args_kwargs_from_envrc  r  r|   ry   r   re   Tensortorch._higher_order_ops.utilsr]  r\  rf   GraphModulerv  owning_modulenamed_modulesr3   register_module)r<   r   r   r5   r6   resultr]  sub_gm
graph_namer   mod_r   r   s               r=   r_  zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_nodex  s   7777 77++D111#>>tDDf7o--#F+++++"00vFFF"!'!%"8    *TY66:>)DV:W$67	))e6;.F.F-1Yu-=E*%di&6EE R#0DI#=#=#=#=9==9QFK6!M7j((      #WW--fdFCCF%feh.BCC 1cfccZ`ccc   !.:::!%J"'"5"C"C"E"E " "3!S==)*J!E ) "))&#66666(C(C!/) ): +;;JOOO >>*555)*=t*=*=>>>r@   r   r]   r7   r   )r4   rB   rC   r  r  r\  r_  r   )r   r   s   @r=   ReplacerrW  s  sR        KKH2? 2? 2? 2? 2? 2? 2? 2? 2? 2? 2?r@   rm  r,   r   c                |    g | ]8}t          |t          j        j                  !                    |          |f9S r:   )r   re   rf   r   index)rj   r   r   s     r=   r   z>ReplacementPatternEntry.replace_with_graph.<locals>.<listcomp>  sL       a//Q#  r@   )r   r   r]   tag_namer3   	tag_valueinput_stopsr"  r7   rR   c                (   | g}t          t          j        j                             }|ri|                                }||vrM||vrIt          |d          r9|                    |           ||j        |<   |                    |j	                   |gd S d S )Nry   )
r"   re   rf   r   r  r   r  ry   r   all_input_nodes)r   rp  rq  rr  queuevisitedr   s          r=   percolate_tagszBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags  s     FE /11G 	6iikkw&&;..V,, / KK$$$)2CHX&LL!4555  	6 	6 	6 	6 	6r@   r   c                    | j         dk    rd S | j        t          j        k    rd S t	          | j                  dk    sJ | j        d         S )Nr  r#   r,   )rh  r   rl  getitemr   r5   r   s    r=   maybe_getitemzAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitem  sN    7o--4;("222449~~****y|#r@   oldUnion[torch.fx.Node, None]new3Union[torch.fx.Node, Sequence[torch.fx.Node], None]c           	        | |J d S t          | t          j        j                  sJ |,|                     d                                |            d S t          |t          j        j                  rd|j        vr|j                            | j                   dD ]1}|| j        v r& 	||| j        |         t                               2|                     |                               |            d S t          | j
                                                  }|D ]0} |          }|t          d           
|||                    1                    |            d S )Nr   )	recomputeac_graph_idzPDeleted index from getitem, did you erase the index and not properly replace it?)r   re   rf   r   rM  r   ry   rx   r"   r   r   r   AssertionError)r|  r~  rp  old_usesuseridxr5   r   r{  rw  r'  s         r=   r'  z;ReplacementPatternEntry.replace_with_graph.<locals>.replace  s    ;;;;F!#ux}55555;--d333$$S)))Fc58=11 CH,,111 %A  #sx//*N #Xsx/A:dCSCS   --c222$$S)))F:  	 0 011$ , ,D'---C{,n   GD#c(++++  %%%%%r@   )
r   r]   rp  r3   rq  r3   rr  r"  r7   rR   rl  )r|  r}  r~  r  r7   rR   )re   rf   Interpreterr   r   r   r   r   minrl  
itemgetterrL  r   rc  r"  r   r   r   )r  r   r   r5   rm  r   	last_nodeindicesr   r|  r~  r{  r   rw  r'  s    ` `       @@@@r=   r   z*ReplacementPatternEntry.replace_with_graphl  s   7	? 7	? 7	? 7	? 7	? 7	? 7	?ux+ 7	? 7	? 7	?r ))++|!!$QII?""?a.455E   %  G
 G)<Q)?)?@@@CI	6 	6 	6 	6( ##I.. Z	6 Z	6/1EFFFFF9((#45594@K+ux}55 ,*m$ $ $ $E& E& E& E& E& E& E& E& E& E&N <  C$4$444 #L+ > > & &HCGC%%%%& <((A----Q555uZ	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6 Z	6x 	s   CF**F.1F.r   r]   c           
     |    |j         J |                     |||j          | j        |j        i |j                   d S r9   )r   r   rR  r5   r6   r9  s       r=   r:  zReplacementPatternEntry.apply,  sX    &222#D<u|<<		
 	
 	
 	
 	
r@   N)
r  r~   r   r   r   rS  r5   rT  r7   rR   rD  )r4   rB   rC   rD   r  r   r:  r:   r@   r=   r   r   h  sY         ,,,,} } } \}~
 
 
 
 
 
r@   r   r  r   c                    dS r   r:   )r  s    r=   _return_truer  6  s    4r@   	search_fnrG  r  r   c                H    t                               d| j        |           d S )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginfor4   )r  r  s     r=   log_trace_failurer  :  s.    HHJ	    r@   Fr   r   Optional[torch.fx.Graph]seen_patternsdict[str, list[Optional[str]]]skip_duplicatesc                   t                               |           |                              }|s.|                             |rt	          |          nd           dS ||rdS t          j        dfd           t	          |          }|D ]'|k    s	|r dS t          j        dfd           (|                    |           dS )aB  
    Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
    in matching, use the graph to distinguish equivalent search patterns.

    Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
    `skip_duplicates` is False and a duplicate is found.
    NFTc                     d  dS )NDuplicate pattern: z with no graphr:   )pattern_reprs   r=   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>[  s    F,FFF r@   c                     d d  dS )Nr  z with duplicated match graph  r:   )	graph_strr  s   r=   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>f  s    a,aaU^aaa r@   )r9  r"  rt   rv   r3   re   _check)r   r   r  r  equiv_pattern_reprsnew_graph_strr  r  s         @@r=   check_and_add_duplicate_patternr  B  s    (++G44L'++L99 l#**+H3u:::DIIIu} 	4FFFF	
 	
 	

 JJM( 
 
		)) 	44aaaaa	
 	
 	
 	
 }---5r@   r:   
replace_fnexample_inputsIterable[Any]r   r;  r<  r7  r6  scalar_workaround)Union[dict[str, Union[float, int]], None]exclusive_arg_namesSequence[str]search_fn_patternUnion[PatternExpr, None]c
           
         g t          j                   j                                        d f	d}
dfd	}t          u rt          j                    rd
S t          j        d
          5  d |D             t           |          \  }}n}d}t          |t                    r|n|gD ]F}t          |t                    r/t          ||r|j        nd|j        |	          r ddd           d
S Gt!          ||
|          }|                    |           |j        cddd           S # 1 swxY w Y   dS )a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    r  r~   r7   r   c           
     
	  	  t                    }|D ]%}| j        vrt          d| d j                   &t          t          j                             fd|D             d                     g }t          j        j                                      }|J |5  t                    D ]8\  }}t          |         t          j                  r|r(t          |         j                  r ddd           dS t          j        |                                         |                                         |         j        |         j        |          |<   t%          j        |         j        |                                                   D ]Lt          t          j                  r0t-          fd|D                       r|                               M:}|s|rhdfd}	  ||z             }	n3# t          $ r&}
t1          |
           Y d}
~
ddd           dS d}
~
ww xY wg }t3          t5          t7          |          t7                    z             |	j        j                  D ]\  }}|t7          |          k     r|                    |j                   3|	j                            |          5  |	j                             ||t7          |          z
                     }|j!        |_        |"                    |           |	j        #                    |           ddd           n# 1 swxY w Y   ||z   }nA	            }	n3# t          $ r&}
t1          |
           Y d}
~
ddd           dS d}
~
ww xY wtI          |	|          } %                                d         }|J |&                    |          }tO          |          rx |          rm            _(        t7           j                  dk    r6 j(        j        j        D ]$}tS          |j*         j        d         d           %	 ddd           dS 	 ddd           dS # 1 swxY w Y   dS )z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                *    g | ]}j         |         S r:   rH  )rj   r@  r  s     r=   r   z:register_replacement.<locals>.check_fn.<locals>.<listcomp>  s     999d#999r@   c                    | j         d         S r   r   )r   s    r=   r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVE] r@   NF)dtypedevicerequires_gradc              3  >   K   | ]}t          |k              V  d S r9   )r   )rj   r  rl   s     r=   rm   z9register_replacement.<locals>.check_fn.<locals>.<genexpr>  sA       ? ?>?1!q&99? ? ? ? ? ?r@   args_newr   r7   c                 X     | t          |           t                    z
  d           S r9   )r   )r  r5   r  s    r=   search_fn_newz=register_replacement.<locals>.check_fn.<locals>.search_fn_new  s,    (y(3x==3t993L3N3N*OPPr@   )argnamesr  r  r   r,   r   r   T)r  r   r7   r   )+r   r6   r   re   rf   r   _dynamor   detect_fake_moder  r   ra  r   r  empty_stridedsizestrider  r  r  shapeSymIntr  rv   r  r   ranger   r   r   r   inserting_afterrY  r@  rM  r   fx_to_patternr   r  r  r   r|   ry   )r  r  r@  sym_argsr   r  gradspecific_patternr  specific_graphr  sym_arg_namesrY  r   r   specific_pattern_matchr   r5   rl   argnames_staticr  r7  r  r  r  r  r  r   s   `                @@r=   check_fnz&register_replacement.<locals>.check_fn  s    (( 	 	D5<''"a9Aa aRWR^a a   ( H9999999;R;R 
 
 (*M'88>>	$$$ Z	 Z	$]33 / /4d1gu|44 / % 0a ? ? %$	Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 $1QQ(("1gm#Aw~&*  DG '_T!W]DGNN<L<LMM / /%a66 /3 ? ? ? ?CK? ? ? < < / %OOA...
  1# 1 )%Q Q Q Q Q Q Q%)1-D)Q)Q' % % %))Q777$uuuQZ	 Z	 Z	 Z	 Z	 Z	 Z	 Z	L%
 %'M*-c(mmc$ii788&,2+ + I I; s8}},,)001CDDD$+1AA+NN I I'5';'G'G (S]]): ;( (H /7mHO'==hGGG*0;;KHHHI I I I I I I I I I I I I I I  -x7HH%)1)T)B)B' % % %))Q777$uuuEZ	 Z	 Z	 Z	 Z	 Z	 Z	 Z	@% $1"%(;&7	$ $ $  %%''*D###%5%;%;D%A%A".// 
KK@V4W4W 
*2(:t*D*D'u{##q(("4:@  &%&V%*[^&3    
 sZ	 Z	 Z	 Z	 Z	 Z	 Z	 Z	t uZ	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	 Z	s   "AQ8C8Q8<HQ8
H<H7&Q87H<<BQ8A,M
>Q8
MQ8M
Q8M*)Q8*
N4NQ8NCQ8*Q88Q<?Q<r6   r   r   c                       fdD             }t          dt                     dz             D ]6}d|  vr n,|                                         d|                      7 rJ d             |S )Nc                :    g | ]}                     |          S r:   )r  )rj   r@  r6   s     r=   r   z@register_replacement.<locals>.normalize_args.<locals>.<listcomp>  s%    ===T

4  ===r@   r,   	tangents_zleftover kwargs: )r  r   rv   r  )r6   r5   r  r  s   `  r=   rR  z,register_replacement.<locals>.normalize_args  s    ====_===q#f++/** 	5 	5A1f,,KK

?q??334444999v9999zr@   Ffunctionalize_rng_opsc                P    g | ]#}t          |t          j                  o|j        $S r:   )r   re   ra  r  r  s     r=   r   z(register_replacement.<locals>.<listcomp>  s8     %
 %
 %
@AJq%,'';AO%
 %
 %
r@   N)r  )r   r7  rR  r  r~   r7   r   )r6   r   r7   r   )inspect	signature
parametersr   joint_fwd_bwdre   is_inference_mode_enabledfunctorch_configpatchgen_pattern_and_search_gmr   r   rB  r  r   r  r   rA  r   )r  r  r  r   r;  r7  r  r  r  r  r  rR  r   gmpattern_matcher_passr  r  s   `` ` ````      @@r=   register_replacementr  l  sq   2 H))44?DDFFGOr r r r r r r r r r r r r rh      =   *,, 	5 
	e	<	<	< " "%
 %
ES%
 %
 %
 $3!# KGRR (GB %Z::LJJ
	! 
	!  .0BCC !2 ",BHH(6$3	   ! !5" " " " " " " "8 * )
 
 

 	$$$E" " " " " " " " " " " " " " " " " "s   >A=E.EE
EzOrderedSet[str]_serialized_patternsunique_namer   c                   dd}t                                           st          dt                      |j        }ddlm}  |j        d          5  t          ||||          }d d d            n# 1 swxY w Y   t          	                    || 	          }	|t          vrd
}
t                              |           nd}
 |            }t          t           | dz  |
          5 }|
d
k    r|                    |           n|                    d           |                    |	           |                    d           d d d            n# 1 swxY w Y   |S )Nr7   r3   c                    t          j        d          } t          j        d                              |           }g }t          t          j        j                  D ]x}t          t          j        j        |          }	 t          |t                    r1t          |t          t          f          r|                    |           i# t          $ r Y uw xY wd                    |          }d| d}| | S )Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr   dirre   	_inductorpattern_matcherr`  r   r  
issubclassr   rX  rv   	TypeErrorr  )auto_generated_msgfile_templatepattern_matcher_importsr@  attrformatted_importss         r=   get_file_templatez-_serialize_pattern.<locals>.get_file_template:  s    %_
 
 !
 
 &'&
(
( 	  #%788 	 	D5?:DAADdD)) 9j;4/ / 9 ,224888    %MM*ABBjQbjjj4!2444s   ?AC
CCz0Could not find serialized patterns directory at r   r$   Fr  )r  wr  z.pyz

r  r   )SERIALIZED_PATTERN_PATHis_dirr   r4   torch._functorchr%   r  gen_patternr9  r"  r  r  openwrite)r  r  r  r   r  r  pattern_namer  r   serialized_pattern
write_moder  fs                r=   _serialize_patternr  3  s   &5 &5 &5 &5P #))++ 
X?VXX
 
 	
 %L;;;;;;			e	<	<	< V ViCTUUV V V V V V V V V V V V V V V .11'{1SS///
  ....
%%''M	%<(<(<(<<j	I	I QGGM""""GGFOOO	"###	               Ns%   A22A69A6&AEEE	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc
                   t          |          }dt          j        v rt          | ||||          }
n[|j        }t          j        d|           }|rt          ||           st          	                    d|            t          ||           }
t          j        |          D ]%}t          |t                    r|j        d |_        &t                               |||||
f           t%          |||||||||
|	
  
         d S )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  r  )r   osenvironr  r4   	importlibimport_moduler   r  warningr`  r  	tree_iterr   r)   constantr  rv   r  )r  r  r  r  r   r;  r7  r  r  r  patr  r  r   s                 r=   gen_register_replacementr    sO    >**N++ NH>O
 
 !)#K\KK
 
  	;// 	KKV   a%%//    c:&& 	 3<+C
  CL&&	NH.?E   '     r@   r  (tuple[PatternExpr, torch.fx.GraphModule]c                   g t          j        |           j                                        }|i }g }d}|D ]B}||v r|                    ||                    "|                    ||                    |dz  }C || |          }	t          |	t          t          t          t          j
        t          j        f|||          |	fS )Nr   r,   )ignore_typesr  r  r  )r  r  r  r   rv   r  intfloatr   re   r  r  )
r  r  r   r  r  r  flat_inputs	input_idxargname	search_gms
             r=   r  r    s     A"9--8==??@H KI  '''09::::~i8999NIIK00IudEL%+F/ 3	
 	
 	
 		 	r@   c                4    t          | ||||          d         S r  )r  )r  r  r   r  r  s        r=   r  r    s+     %>8->@S 	 	r@   r@  	pass_dictr>  2Callable[[Callable[..., Any]], Callable[..., Any]]c                     d fd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    rH  rG  r7   c                    t          |           sJ t          |                                          d| _        | S )Nr   r7  rH  r@  T)rc  rF  rA  _inductor_lowering_functionrH  r7  r  r   r>  s    r=   	decoratorz,register_lowering_pattern.<locals>.decorator  sQ         g	
 	
 	

(9g(
.
.
..2+r@   rH  rG  r7   rG  r:   r   r7  r  r>  r  s   ```` r=   register_lowering_patternr    s;             r@   c                     d fd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    rH  rG  r7   c                x    t          |           sJ t          |                                          | S )Nr  r@  )rc  rO  rA  r  s    r=   r  z)register_graph_pattern.<locals>.decorator  sI         g	
 	
 	

(9g(
.
.
.r@   r  r:   r  s   ```` r=   register_graph_patternr    s;             r@   r   r   c                H    |t          t          | j                            u S r9   )r   iterr   )r   r   s     r=   is_start_of_fx_graphr"  '  s    4U[))****r@   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)rh  torch._ops.OpOverloadc                    | j         dk    rdS | t          j        j        j        j        t          j        j        j        j        fv S )Nr0   F)r  re   r   r0   accumulate_grad_defaultresize_storage_bytes_)rh  s    r=   "fixme_incorrect_inductor_schema_opr(  1  sD    	|z!!u 	+3	08  r@   c                B   t          | j        t          j        j                  r%t          | j                  s| j        j        j        S t          | j        t          j        j	        j
                  rdS | j        dk    r=t          | j                  sJ t                              | j        j                  rdS nH| j        dk    r=t          | j        t                     sJ t                              | j                  rdS | j                            d          d uS )NFr  Tr  out)r   r   re   rd  rm  r(  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedrh  rc  _mutation_op_researchr4   r3   r6   rt   rz  s    r=   is_mutation_opr2  =  s   UZ*  0== {"--	U,?R
 
  uw/!!$$$$$!!$+"677 	4		M	!	!$+s+++++!!$+.. 	4;??5!!--r@   r  r  c                f    d| j         v sJ d|j         v sJ | j         d         |j         d         k    S Nmutation_region_idr   )r  r  s     r=   same_mutation_regionsr6  Q  sC    16))))16))))6&'162F+GGGr@   r  c                   |}d|j         vr0t          | |          s |j        }d|j         vrt          | |           |j                             dd          }||ur)|j        }t          |          r|dz  }||j         d<   ||u)|S )Nr5  r   r,   )ry   r"  prevrt   r   r2  )r   r   r   r5  s       r=   get_mutation_region_idr9  W  s    A
af
,
,5I%QR5S5S
,F af
,
,5I%QR5S5S
,$8!<<
4--F! 	$!#'9#$	 4--
 r@   c                R    dt          t          | j                            j        vS r4  )r   r!  r   ry   r  s    r=   "should_compute_mutation_region_idsr;  d  s"    tD,=,='>'>'CCCr@   c                Z    d}| j         D ] }t          |          r|dz  }||j        d<   !d S )Nr   r,   r5  )r   r2  ry   )r   r5  nds      r=   compute_mutation_region_idsr>  h  sN    k ; ;" 	$!#(:$%%; ;r@   c                  <     e Zd Z	 dd fdZdd
ZddZddZ xZS )rB  Nr^   Optional[str]r7   rR   c                    t                                                       t          t                    | _        || _        t          t                    | _        d S r9   )r   rW   r   r   patternsr^   r  )r<   r^   r   s     r=   rW   zPatternMatcherPass.__init__q  sQ     	  	 # >I=N=Nr@   itemr0  r1  c                    | j         |         S r9   )rB  )r<   rC  s     r=   r3  zPatternMatcherPass.__getitem__  s    }T""r@   r  +Union[torch.fx.GraphModule, torch.fx.Graph]r  c                   | j         sdS t          |t          j        j                  r|j        }nHt          |t          j        j                  r
|}|j        }nt          dt          |                     t          |          rt          |           t          j        t          |          }d}g }d}| j         D ]9\  }}|dk    rd}|                    |                    ||d                     :|r*|                    |                    dd                     | j        | j        nd}	t          |t          j        j                  sJ t%          ||	          5  t'          t(          j                            |          d	          D ]}
t/          |
          }|
j        dk    r|
j        |f| j         vr.t3          |
d
          r@| j         |
j        |f         D ]Y}|
j        r nN|j                            |
          }t;          |          r4t=          t?          tA          ||j!                                      dk    rjtD          j#        $                    d          |
j%        k    r(tL          '                    d|
|
j(        ||j                   t;          |          rtS          |*                    |                    rn|dz  }|+                    |||
           tX          tZ                   dxx         dz  cc<   tX          tZ                   dxx         t=          |j!                  z  cc<   [	 d d d            n# 1 swxY w Y   |S )Nr   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)rh  r   sort)rh  rG  r  )reverse)allow_cpu_inputsr,   !TORCHINDUCTOR_PATTERN_MATCH_DEBUGz
%s%s %s %spattern_matcher_countpattern_matcher_nodes).rB  r   re   rf   rc  r   Graphrd  r   r  r;  r>  r   r   r9  rv   
find_nodesr^   r    sortedr  r  from_iterablerx  rh  r.   r   r   r  r  r   r"   r  r   r  r   rt   r@  r  r  r5   r   r7  r:  r   backend)r<   r  r   get_mutation_region_id_partialr  r   has_call_modulerh  r   r^   r   entryr  s                r=   r:  zPatternMatcherPass.apply  s   } 	1b%(.// 	HEEEHN++ 	E$BBg]abd]e]egg   .e44 	/'...)2):"E*
 *
& - 	Q 	QJB]"""&U--F-OOPPPP 	ILL))])GGHHH&*n&@DNNFW	"eh233333#B	22 !	S !	Sy<<UCCTRRR  S  S'--7m++(== 
 9PUVVV !]DGV+<= S SE| ++D11A !!&s+I17'S'STT   
 !z~~&IJJdiWWL$	1emTTT{{ S~e6G6G6J6J'K'K S
Aud333 )*ABBBaGBBB )*ABBBc!'llRBBBA S!	S !	S !	S !	S !	S !	S !	S !	S !	S !	S !	S !	S !	S !	S !	SD s   %G MMMc                8    | j                                          d S r9   )rB  clearrV   s    r=   rV  zPatternMatcherPass.clear  s    r@   r9   )r^   r@  r7   rR   )rC  r0  r7   r1  )r  rE  r7   r  rX   )r4   rB   rC   rW   r3  r:  rV  r   r   s   @r=   rB  rB  p  s         $(O O O O O O O # # # #? ? ? ?B       r@   rB  r5   r   r6   r   c                     t           r9   ru  r   s     r=   _not_implementedrX    s    
r@   r  rE  r
  Sequence[type[Any]]r  c                   	 |pi }d |                                 D             t                    t          |          k    sJ 	 ddfd		t          j                     G 	fd
dt          j        j                  }t          | t          j        j                  sJ  ||           	                                }t          |t                    s!t          t          j        |                    S |S )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                    i | ]\  }}||	S r:   r:   ri   s      r=   r  z!fx_to_pattern.<locals>.<dictcomp>  s    HHHdaQHHHr@   Nr  rO   ignore_types_overrideOptional[Sequence[type[Any]]]r7   Union[T, KeywordArg, Ignored]c                8   ||n}t          | t          t          f          r| v rt          |                    S t	          |           |v rt                      S t          | t                    r)t          d | D                       r| rt                      S | S )Nc              3  @   K   | ]}t          |t                    V  d S r9   )r   r4  )rj   ys     r=   rm   z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>  s,      &I&I!z!W'='=&I&I&I&I&I&Ir@   )r   r  r  r>  r  r4  r   r  )r  r\  current_ignore_typesr
  inv_scalar_workarounds      r=   process_argz"fx_to_pattern.<locals>.process_arg  s     &;%F!!L 	 a%&& 	810E+E+E3A677777***99a 	3&I&Iq&I&I&I#I#I 	a 	99r@   c                  L     e Zd ZeZeZeZdfd	ZdfdZd fdZ	 xZ
S ) fx_to_pattern.<locals>.Converterr   r3   r5   r   r6   Mapping[str, Any]r7   &Union[ExclusiveKeywordArg, KeywordArg]c                   t                    }|t                    k     r	|         }n4r|                    d          sJ |}nt          j        dd|          }|}|v rt          |          S t          |          S )Ntangentz_\d+$rY   )r   r   
startswithresubrR  r>  )	r<   r   r5   r6   r   r@  r  argnumr  s	         r=   rY  z,fx_to_pattern.<locals>.Converter.placeholder  s     VA3x==  { ((33333"f55****4000!$'''r@   r   c                .   |t           j        k    r!t          d D                       fdfd}|t          j        ||f          \  }}t
          v r.fd	|D             }fd
|                                D             }t          |g|R i |S )Nc              3  ,   K   | ]}|t           u|V  d S r9   )r  )rj   ts     r=   rm   zAfx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s3       Q Q1C<<<<<<Q Qr@   r  rO   r\  r]  r7   r^  c                     | |          S r9   r:   )r  r\  rd  s     r=   process_arg_fn_implzKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     ';q*?@@@r@   c                &    g | ]} |          S r:   r:   )rj   r  process_arg_fns     r=   r   zBfx_to_pattern.<locals>.Converter.call_function.<locals>.<listcomp>  s#    888aq))888r@   c                .    i | ]\  }}| |          S r:   r:   )rj   rk   r  ru  s      r=   r  zBfx_to_pattern.<locals>.Converter.call_function.<locals>.<dictcomp>  s)    JJJ41a!^^A..JJJr@   r  rO   r\  r]  r7   r^  )rl  ry  r   r  r   r   rz   r  )r<   r   r5   r6   rs  ru  r
  rd  s        @r=   r  z.fx_to_pattern.<locals>.Converter.call_function  s     )N))) LQ Q Q#/Q Q Q L LA A A A A A A "5!?>D&>JJLD&|##88884888JJJJ6<<>>JJJ8888888r@   r   r]   r   c                   t                                          |          }|j        dk    rt          |t                    rv|j        d         }t          |t                    sJ t          |          t          |          k    sJ t          ||          D ]\  }}t          |j	                  |_	        nt          |j	                  |_	        |S )Nr  r   )
r   r_  rh  r   r   r5   r   r   r   r   )r<   r   rvr5   rr   r   s         r=   r_  z)fx_to_pattern.<locals>.Converter.run_node"  s    !!!$$BtxJr5$9$9vay!$
333332ww#d))++++!"dmm - -FAs!#)nnAGG- qw<<Ir@   )r   r3   r5   r   r6   rg  r7   rh  )r   r3   r5   r   r6   rg  r7   r   )r   r]   r7   r   )r4   rB   rC   rX  r  r  r\  rY  r  r_  r   )r   r  rn  r  r
  rd  s   @r=   	Converterrf    s        &&#	( 	( 	( 	( 	( 	( 	( 	((	9 	9 	9 	9 	9 	9 	96
	 
	 
	 
	 
	 
	 
	 
	 
	 
	r@   r{  r9   rw  )rz   r   r  r  re   rf   r  r   rc  r"  r   r  r  tree_leaves)
r  r
  r  r  r  r{  r   rn  rc  rd  s
    `` `  @@@r=   r  r    sK    */RHH.?.E.E.G.GHHH$%%->)?)????? FJ        _F> > > > > > > > > > >EH( > > >@ b%(./////imm!!Gg{++ ?!&"4W"="=>>>Nr@   T)r   get_decomp_fnrK   r   r}  Optional[Callable[..., Any]]rM   c                  t                      5  t                      5  |
 |            nt                      } t          | |d          | }ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   ddlm} |r) ||j                   |j                                         |                                 |S )z>Build a normalized inference graph, for use with fx_to_patternNreal)tracing_moder,   remove_noop_ops)	r   r!   r-   r   fx_passes.post_gradr  r   eliminate_dead_code	recompile)rK   r5   r   r}  decompositionsr  r  s          r=   r   r   5  sw    
"	#	# E E%7%9%9 E E,8MMOOO>Q>S>S 	 >WRf===tD	E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E 544444 '!!!
$$&&&LLNNNIs4   A10AA1A	A1!A	"A11A58A5c           
        ddfd
}t           j                            d          5   t          | d |t	                      dd          |  ddd           n# 1 swxY w Y   sJ ddlm}  |j                   ddlm	} t                      }t          t           j        j        j        j        t!          d          t!          d                    }t#          ||t$                                        |j                   |                    j                   t           j        j                                        j        _        j                                                                          S )z=Build a normalized training graph, for use with fx_to_patternNjoint_graphrM   inputsr   r6   r   r7   1tuple[torch.fx.GraphModule, torch.fx.GraphModule]c                F    rJ t          |           t          | |fi |S r9   )clone_graphr(   )r  r  r6   r  s      r=   record_joint_graphz)joint_fwd_bwd.<locals>.record_joint_graphT  s3     v%% f?????r@   c                     t          |           S r9   )r'   )gr  s     r=   r   zjoint_fwd_bwd.<locals>.<lambda>_  s    ++ r@   TF)partition_fnr  keep_inference_input_mutations
enable_logr,   r  )pointless_viewr   r  )r   rH  r7  )r  rM   r  r   r6   r   r7   r  )re   _guardstracingr&   r-   r  r  r   fx_passes.joint_graphr  rB  r  r   atenviewr&  r>  rO  r  rA  rB  r:  rf   CodeGen_codegenr  r  )rK   r5   r  r  r  matcher_passr   r  s          @r=   r  r  O  s    *.B@ @ @ @ @ @ 
		t	$	$  	
+++.00+/	
 	
 	
 	 	               II2444444OBH555555%''L	#Z%6%6
68J8J G \  h|$%%%rx    ..00BHH  """LLNNNIs   &AA"Ar   list[torch.fx.node.Argument]c                l    g }t           j                            | j        | j        f|j                   |S r9   )re   rf   r   r5   r6   rv   )r   r5   s     r=   r  r  ~  s/    )+D	Hafah'555Kr@   c                   t          t          | j                            }t          t          j        j                             t          t                     }d }|r|                                }fdt          |          D             }|r"||d                  
                    |           nm                    |           |r|j        |ur|
                    |           |}|                    t          |                    |d                               ||s%t                    t          | j                  k    sJ d S )Nc                    g | ]}|v|	S r:   r:   )rj   r  readys     r=   r   z+stable_topological_sort.<locals>.<listcomp>  s    @@@Q%qr@   rJ  r:   )r   r   r   r"   re   rf   r   r   r  r  rv   r  r   r   r   )r   pendingwaitingcursorr   waiting_forr  s         @r=   stable_topological_sortr    sB    8EK(())G ux}%''E $G F
 <{{}}@@@@%++@@@ 	< KO$++D1111IIdOOO $&+T11d###F NN8GKKb$9$9::;;;  <  93u::U[)9)9999999r@   Callable[[], Any]c                l     t           j        t          j                   d fd                        }|S )z0Wrapper around lazy init functions in fx_passes/r7   r   c                    t           t                                                   } t          j                            d           5  t                      5  t                      5               }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   | t           t          <   |S r9   )r   rQ  ru   re   r  r  r   r*   )counters_refrg  rK   s     r=   	lazy_initz%init_once_fakemode.<locals>.lazy_init  sc     (--//]""4(( 	 	*@*B*B 	 	NDTDT 	 	RTTF	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 )sZ   B(B#A:.B:A>>BA>BB(B	B(B	B((B,/B,)r7   r   )r   r-  rK  )rK   r  s   ` r=   init_once_fakemoder    sL     __R	 	 	 	 	  _	 r@   r@  Callable[[Match], Any]c                     d fd}|S )z2Function for extra_check to put pass behind a flagr  r~   r7   r   c                .    t          t                    S r9   )r`  r%   )r  r@  s    r=   
flag_checkzconfig_flag.<locals>.flag_check  s    vt$$$r@   )r  r~   r7   r   r:   )r@  r  s   ` r=   config_flagr    s)    % % % % % % r@   input_graphc                b     G d dt                     } ||                                           S )Nc                        e Zd Zd fdZ xZS )clone_graph.<locals>.CopyGraphr\   r]   r7   c                .   t                                          |          }t          |t          j        j                  rS|j        j                            |j                   | j	        j
                            |j        d           |j        _        |S r9   )r   r_  r   re   rf   Proxyr   ry   rx   	new_graph_graph_namespacer(  r@  )r<   r\   r   r   s      r=   r_  z'clone_graph.<locals>.CopyGraph.run_node  sx    ww''11H(EHN33 "))(-888%)^%D%P%PM4& &" Or@   )r\   r]   r7   r]   )r4   rB   rC   r_  r   r   s   @r=   	CopyGraphr    s=        	 	 	 	 	 	 	 	 	 	r@   r  )r+   	transform)r  r  s     r=   r  r    sJ        K    9[!!++---r@   _seen_patterns
arg_number
kwarg_namer@  c                    t          | j                  |k    r| j        |         S |d S | j                            |          S r9   )r   r5   r6   rt   )r   r  r  s      r=   get_arg_valuer    sC     49~~
""y$$		t{z***r@   r   Iterable[torch.fx.Node]r   c                    gt          t          j        j                  r3                    fd                                D                        fd| D             S )Nc                0    g | ]}t          |          S r:   r_  ra  s     r=   r   z filter_nodes.<locals>.<listcomp>  s#    IIIhGB))IIIr@   c                &    g | ]}|j         v |S r:   )r   )rj   r   rY  s     r=   r   z filter_nodes.<locals>.<listcomp>  s%    999TdkS&8&8D&8&8&8r@   )r   re   rd  re  r   rf  )r   rK   rY  s    `@r=   filter_nodesr    sl    $C"ej122 K

IIII",,..IIIJJJ9999U9999r@   torch.fx.node.Targetc                    | j         dk    r@t          | j        t                    sJ t	          | j        j        | j                  j        S | j        S )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r  )rh  r   r   r3   r   r   rd  r   rz  s    r=   rx  rx    sK    
 w-$+s+++++14;??II;r@   )rY   )rZ   r[   r\   r]   r^   r3   r7   rR   )r  r  r7   r  r  )r  rG  r  r   r7   rR   r  )
r   r   r   r  r  r  r  r   r7   r   )r  r2   r  rF   r  r  r   rJ   r;  r<  r7  r6  r  r  r  r  r  r  r  r   r7   r   )r  r3   r  r2   r  r   r   rJ   r  r  r7   r   )r  r3   r  r2   r  rF   r  r  r   rJ   r;  r<  r7  r6  r  r  r  r  r  r   r7   rR   )Nr:   )r  r2   r  r   r   rJ   r  r  r  r  r7   r  )r  r2   r  r   r   rJ   r  r  r  r  r7   r   )
r   r   r7  r6  r  r/  r>  r   r7   r  )r   r   r   r]   r7   r   )rh  r#  r7   r   r}  )r  r]   r  r]   r7   r   )r   r   r   r]   r7   r  )r   r   r7   r   )r   r   r7   rR   )r5   r   r6   r   r7   r   )r:   r:   Nr:   )r  rE  r
  rY  r  r  r  r  r  r  r7   r   )
rK   rG  r5   r   r   r   r}  r~  r7   rM   )rK   rG  r5   r   r7   rM   )r   r]   r7   r  )rK   rG  r7   r  )r@  r3   r7   r  )r  rM   r7   rM   r9   )r   r]   r  r  r  r@  r7   r   )r   r  rK   r   r7   r   )r   r]   r7   r  )r   
__future__r   r   dataclassesr   r  r  r  loggingrl  r  rl  r  rK  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   pathlibr   r   r   r   r   r   r   r   typing_extensionsr   r   re   torch._guardstorch.fxtorch.utils._pytreer   r   r  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   r   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr    rq   r!   torch.utils._ordered_setr"   
_functorchr%   r  _functorch.aot_autogradr&   r'   _functorch.partitionersr(   _subclassesr)   r*   rf   r+   rY   decompositionr-   loweringr.   	getLoggerr4   r  r   r  primsConstantr   r  r   rt   rQ  r2   rF   rJ   rO   r   Targetr3   FnsTyperQ   rT   r|   r~   r   r   r  r  r   r   r.  r4  r>  rR  rX  r   _SimpleSpecr  r  r  r  r  r  r  r  r  r  r
  r9  r/  	dataclassr5  rF  rO  r   r  r  r  r  r  rD   r  __file__parentr  r  r  r  r  r  r  r  r"  compiler0  r(  r2  r6  r9  r;  r>  rB  rX  r  no_gradr   enable_gradr  r  r  r  r  r  r  r  r  rx  r:   r@   r=   <module>r     s`  ! ! !F # " " " " "                        				 				   # # # # # # # # # # # # # # N N N N N N N N N N N N N N       N N N N N N N N N N N N N N N N N N * * * * * * * *       $ $ $ $ $ $ $ $ $ ; ; ; ; ; ; ( ( ( ( ( ( 0 0 0 0 0 0 @ @ @ @ @ @ 6 6 6 6 6 6 W W W W W W W W + + + + + + I I I I I I I I K K K K K K 1 1 1 1 1 1 / / / / / / 3 3 3 3 3 3 C C C C C C C C 7 7 7 7 7 7 4 4 4 4 4 4 4 4             . . . . . . ; ; ; ; ; ; g!!y~	x./
*..>

K
K> > > > >x > > >> > > > > > > ># # # # #h # # # GCLL $c)
*? ? ? ? ? ? ? ? 8:: IK? ? ? ? ?8A A A A A A A AH    ,   8 E;&'   '
 '
 '
 '
 '
 '
 '
 '
T 1  1  1  1  1#  1  1  1F- - - - -+ - - -    k   E E E E E E E E(E E E E E+ E E E4O
 O
 O
 O
 O
+ O
 O
 O
d CHol
 l
 l
 l
 l
k l
 l
 l
^    ?                        (    ,       *       *   ,
 ,
 ,
 ,
 ,
[ ,
 ,
 ,
^F
 F
 F
 F
 F
 F
 F
 F
R#
 #
 #
 #
 #
; #
 #
 #
L4 4 4 4 4 4 4 4n! ! ! ! !X ! ! ! : : : : : : : :: 
 
 
 
 
< 
 
 
 	= 	= 	= 	= 	= 	= 	= 	= J
 J
 J
 J
 J
l J
 J
 J
Z       "	' ' ' ' '` ,8CG)+26!A A A A AH )3
  4 4 4 4L L L L^ $x../+=@UU       $ ,8CG)+!5 5 5 5 5p e444
 DH)+    54L DH)+		 		 		 		 		 ,8
      4 ,8
      ,+ + + + "*VWW	 	 	 	. . . .(H H H H
 
 
 
D D D D; ; ; ;V V V V V V V Vr    )+ CG)+e e e e eP 
 #'26     2 + + + +\   !: !: !: !:H   &   . . . . #-*,, . . . . GK+ + + + +: : : :     r@   