
    `iJ             &       ~%   U d dl Z d dlZd dlZd dlT d dlmZ d dlZd dlmc mZ	 d dl
mZ d dlmZ ddlmZ g Zee         ed<   i Zeeef         ed	<   ddZ	 	 	 ddZddZdeddfdZdededededdf
dZd Zd Zd Z e j        ee          Z dee!         fdZ"d Z#d Z$d Z%d Z& e ej'        j(        j)        j*        ej'        j(        j+        j*        ej'        j(        j,        j*        ej'        j(        j-        j*        ej'        j(        j.        j*        ej'        j(        j/        j*        ej'        j(        j0        j*        gd          d              Z1 e ej'        j2        j3        j*        d          d!             Z4 e ej'        j(        j5        j*        gd          d"             Z6 e ej'        j(        j7        j*        d          d#             Z8  e ej'        j(        j7        j9        d$          e8            e ej'        j(        j:        j*        d$          d%             Z; e ej'        j(        j<        j*        d&          d'             Z= e ej'        j(        j>        j*        d(          d)             Z? e ej'        j(        j@        j*        d*          d+             ZA e ej'        j(        jB        jC        d,          d-             ZD e ej'        j(        jE        j*        d          d.             ZF e ej'        j(        jG        j*        d/          d0             ZH  e ej'        j(        jI        j*        d          e$            e ej'        j(        jJ        j*        ej'        j(        jK        j*        ej'        j(        jL        j*        ej'        j(        jM        j*        ej'        j(        jN        j*        gd          d1             ZO  e ej'        j(        jP        j*        d2          eO             e ej'        j(        jQ        j*        d3          eO             e ej'        j(        jQ        jR        d4          eO            e ej'        j(        jS        j*        d          d5             ZT e ej'        j(        jU        j*        d6          d7             ZV e ej'        j(        jW        j*        d6          d8             ZX e ej'        j(        jY        j*        d9          d:             ZZ e ej'        j(        j[        j*        d;          d<             Z\ e ej'        j(        j]        j*        d=          d>             Z^ e ej'        j(        j_        j`        d?          d@             Za e ej'        j(        j_        j*        dA          dB             Zb e ej'        j(        jc        jd        dC          dD             Ze e ej'        j(        jf        j*        dE          dF             Zg e ej'        j(        jh        j*        dG          dH             Zh e ej'        j(        ji        j*        dI          dJ             Zj e ej'        j(        jk        jl        dK          dL             Zm e ej'        j(        jn        j,        dM          dN             Zo e ej'        j(        jp        j*        dO          dP             Zq e ej'        j(        jr        j*        dQ          dR             Zs e ej'        j(        jt        j*        dS          dT             Zu e ej'        j(        jv        j*        dU          dV             Zw e ej'        j(        jx        j*        dW          dX             Zy e ej'        j(        jz        j*        dY          dZ             Z{ e ej'        j(        j|        j*        d[          d\             Z| e ej'        j(        j}        j*        d]          d^             Z} e ej'        j(        j~        j        d_          d`             Z e ej'        j(        j        j*        da          db             Z e ej'        j(        j        j*        da          dc             Z e ej'        j(        j        j*        dd          de             Zdf Z e ej'        j(        j        j*        dA          dg             Z e ej'        j(        j        j        dh          di             Z e ej'        j(        j        jl        dj          dk             Z e ej'        j(        j        j*        dl          dm             Z e ej'        j(        j        j*        ej'        j(        j        j*        gd[          dn             Z e ej'        j(        j        j*        do          dp             Z e ej'        j(        j        j*        dq          dr             Z e ej'        j(        j        jl        ds          dt             Z e ej'        j(        j        jd        du          dv             Z e ej'        j(        j        j*        dw           e ej'        j(        j        j*        dw          dx                         Z e ej'        j(        j        j*        dy          dz             Z e ej'        j(        j        j,        dh          d{             Z e ej'        j(        j        j*        dA          d|             Z e ej'        j(        j        j        d}          d~             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j        d}          d             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        dQ          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j        d          d             Z e ej'        j(        j        j        d          d             Z e ej'        j(        j        jd        d          d             Z e ej'        j(        j        j*        d          d             Zd dlmZmZ d dlmZ  ej        e          	 	 ddejd        dejd        dejd        de!dededeeef         dededeejd        ejd        ejd        f         fd            Z ej        e          	 	 ddejd        dejd        dejd        dejd        dejd        dejd        dejd        dee!ef         dedededeeef         dededeejd        ejd        ejd        eeejd                 df         f         fd            Z e ej'        j(        j        j*        dæ          dĄ             Zej                            ddƦ          5 Z(e(                    dedȦ           e(                    dedɦ           e(                    dedʦ           ddd           dS # 1 swxY w Y   dS )    N)*)Optional)normalize_function)#jagged_scaled_dot_product_attention   NestedTensor__all__JAGGED_OPS_TABLEFc                 L    ddl m} t          |t          t          f          rU t          |           fd|D                       } t          |          t                              |                    S |r | |          }|dk    r| k     sJ |dk    rdz
  n|dz
  S )Nr   canonicalize_dimsc              3   :   K   | ]}t          |          V  d S N_outer_to_inner_dim).0dndim
ragged_dims     n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/nested/_internal/ops.py	<genexpr>z&_outer_to_inner_dim.<locals>.<genexpr>   s0      QQ.tQ
CCQQQQQQ    r   )torch._prims_commonr   
isinstancetuplelisttypedictfromkeys)r   dimr   canonicalizer   outputs   ` `   r   r   r      s    555555#t}%% 3cQQQQQSQQQQQtF||DMM&11222 +c**!88d


" !AXX:>>372r   Tc                     ddl m}  || |          }||k    r|st          | d          |dk    r|st          | d          |rt          | ||          n|}	|r
|dk    }
|	|
fS |	S )Nr   r   z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)r   r   RuntimeErrorr   )r   r!   r   op_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr   wrappedretoperating_on_batchs              r   _wrap_jagged_dimr-   '   s     655555c**G*%5gWWWXXX	AogRRRSSS  	D':666 
  ) %\'((Jr   c                 X   	 ddl m	 t          |t          t          f          sJ dt          |                       	 fd|D             }d|v }|v }t          fd|D                       }t          t                               fd|D                                 }||||fS )z
    For NestedTensor operators,
    wraps dimensions to non-negative values,
    and returns metadata related to reduction dimension(s).
    r   r   z<_wrap_jagged_dims(): cannot iterate over dimensions of type c                 (    g | ]} |          S  r0   )r   r   r   r   s     r   
<listcomp>z%_wrap_jagged_dims.<locals>.<listcomp>P   s4       '($""  r   c              3   0   K   | ]}|d k    o|k    V  dS )r   Nr0   )r   r   
ragged_idxs     r   r   z$_wrap_jagged_dims.<locals>.<genexpr>V   s0      PPaqAv9!z/PPPPPPr   c              3   :   K   | ]}t          |          V  d S r   r   )r   r   r   r3   s     r   r   z$_wrap_jagged_dims.<locals>.<genexpr>Z   s0      UU1)$:>>UUUUUUr   )	r   r   r   r   r   r   anyr   r    )
r   dimsr&   r3   wrapped_dimsoperate_on_batchoperate_on_raggedoperate_on_non_batchouter_to_inner_dimr   s
   `  `     @r   _wrap_jagged_dimsr<   D   s    655555dUDM**  StDzzSS *    ,0  L L("l2PPPP<PPPPP UUUUUUUUUU  /1BDXXXr   
schema_strreturnc                    |                      d          }d |D                                 d          }t          |          |z
  }|d         dk    r|d d         }nqt          |          |k    r t          |          t          |          k    s>t          d|j         d|  d| d	t          |           d
t          |           d          d d d d d}t          |          D ]\  }}	|	                     d          \  }
}|                    d          }|r
|d d         n|}||                                vrt          d|           |t          |          k    r |st          d|j         d|  d|
           ||         |ffd	} |||                   s.dddddd}t          d|j         d|  d|
 d||                    d S )N, c                 8    g | ]}|                     d           S )?)endswithr   xs     r   r1   z check_schema.<locals>.<listcomp>b   s"    BBBQCBBBr   T...NestedTensor (z): expected at least z arguments and at most z arguments, but got: z
 argumentsc                 b    t          | t          j                  ot          | t                     S r   )r   torchTensorr	   rE   s    r   <lambda>zcheck_schema.<locals>.<lambda>q   s$    z!U\22V:a;V;V7V r   c                 T    t          | t                    o| j        d u o
| j        dk    S Nr   )r   r	   _lengths_ragged_idxrM   s    r   rN   zcheck_schema.<locals>.<lambda>r   s1    
1l33 J$MQ r   c                 ,    t          | t                    S r   r   r	   rM   s    r   rN   zcheck_schema.<locals>.<lambda>u   s    J|
 
 r   c                     dS )NTr0   rM   s    r   rN   zcheck_schema.<locals>.<lambda>x   s     r   )tjtjt_allr5   z: rB   zUnknown arg type: z) missing required argument: c                 <    |r| d u p
 |           S  |           S r   r0   )rE   is_optional	_check_fns     r   check_fnzcheck_schema.<locals>.check_fn   s0     $Dy0IIaLL0 y||#r   tensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz
<any type>)rV   zt?rW   rX   r5   z): expected z	 to be a )	splitcountlen
ValueError__name__	enumeraterC   keysAssertionError)r=   funcargskwargsnamed_arg_typesnum_optional_argsmin_argsarg_type_check_fnsinamed_arg_typenamearg_typerZ   normalized_arg_typer\   type_to_descr[   s                   @r   check_schemars   `   s    &&t,,OBB/BBBHHNN?##&77H re##)#2#.D		X%%#d))s?7K7K*K*K) ) )
 ) )QY ) )),_)=)=) )t99) ) )   WV 
 
 ~	 	 '77 # #>'--d33h'',,/:Hhssmm&8&=&=&?&??? !K6I!K!KLLLD		>>  9DM 9 9J 9 9269 9   &':;	$/ 	$ 	$ 	$ 	$ 	$ 	$ xQ   	'=6# L , , ,
 , , , ,), ,  	/# #r   aa_namebb_namec           	          |j         |j                 |j         |j                 k    rt          d| j         d| d| d          d S )NrH   z: expected  and z' to have the same exact offsets tensor.)_sizerR   r%   rb   )rf   rt   ru   rv   rw   s        r   check_ragged_dim_samer{      sk     	wq}!777)DM ) )f ) )6 ) ) )
 
 	
 87r   c                     | j         dz   }| j        d |         }|d |         }t          |          t          |          k    o&t          d t	          ||          D                       S )Nr   c              3   4   K   | ]\  }}||k    p|d k    V  dS )rF   Nr0   )r   nsss      r   r   z%raggedness_matches.<locals>.<genexpr>   s3      HH52qB!GqBwHHHHHHr   )rR   rz   r`   allzip)ntsizeend	nt_raggedsize_raggeds        r   raggedness_matchesr      sn    
.1
C#Itt*Ky>>S--- HHC	;,G,GHHHHHr   c                     |                                  dk    rO| j        d         dk    r>|                     d          } |                                  dk    r| j        d         dk    >| S Nr   r   )r!   shapesqueeze)rV   s    r   squeeze_leading_onesr      sX    & %%''A++!'!*//IIaLL %%''A++!'!*//Hr   c                 z     t          t                    sgt           t                    s g  fd}|S )Nc                 F     D ]} fd}D ]} ||          ||<    S )Nc                       fd}|S )Nc                  >    t          g| R i |  g| R i |S r   )rs   )rg   rh   aten_oprf   r=   s     r   innerz@register_func.<locals>.wrapper.<locals>.get_inner.<locals>.inner   sC     TCDCCCFCCC49$999&999r   r0   )r   r   rf   r=   s   ` r   	get_innerz1register_func.<locals>.wrapper.<locals>.get_inner   s.    : : : : : : : r   r0   )rf   r   r   tableaten_opsr=   tabless   `   r   wrapperzregister_func.<locals>.wrapper   sa     
	4 
	4G        4 4!*7!3!3g4r   )r   r   )r   r   r=   r   s   ``` r   register_funcr      sa    h%% :fd##        Nr   c                    t                               | d           }||S t          j        j        | j        v rAddlm} |D ]'} ||          rt          d| j	         d|           (t          d |D                       }|dk    rg }| j        j        D ]]}t          |j        t          j                  r|                    |j         d            n|                    |j         d           ^|                    d	           d
                    |          }t'          || g|R i | t)          j        t,          |           S |dk    r+t'          d| g|R i | t)          j        t.          |           S d S )Nr   is_nested_intrH   z: invalid argument c              3   J   K   | ]}t          |t          j                  V  d S r   )r   rK   rL   rD   s     r   r   z lookup_jagged.<locals>.<genexpr>   s.      HHajEL99HHHHHHr   r   z: jt_allz: anyrG   r@      zlhs: any, rhs: any, ...)r   getrK   Tag	pointwisetags%torch.fx.experimental.symbolic_shapesr   r%   rb   sum_schema	argumentsr   r   
TensorTypeappendro   joinrs   	functoolspartialjagged_unary_pointwisejagged_binary_pointwise)	rf   rg   rh   dispatch_funcr   argnum_tensor_argsschema_partscheck_schema_strs	            r   lookup_jaggedr      s   $((t44M  ydi''GGGGGG
  	 	C}S!! "KDMKKcKK   HH4HHHHHa L|- < <ch(899 < ''38(=(=(=>>>E ''38(:(:(:;;;;&&&#yy66)4A$AAA&AAA$%;TBBB!!2DJ4JJJ6JJJ$%<dCCC4r   c                 p    |                                  |                                 | j        | j        d}|S )Noffsetslengths_metadata_cacherR   r   )r   rh   s     r   extract_kwargsr     s6    ;;==;;==.	 F Mr   c                     t          d |D                       t           | fd|D             i |fi t                    S )Nc              3   D   K   | ]}t          |t                    |V  d S r   rT   r   r   s     r   r   z)jagged_unary_pointwise.<locals>.<genexpr>  s1      DDsjl&C&CDsDDDDDDr   c              3   2   K   | ]}|u r|j         n|V  d S r   _values)r   r   njts     r   r   z)jagged_unary_pointwise.<locals>.<genexpr>  s1      BBccSjjs{{cBBBBBBr   nextr	   r   )rf   rg   rh   r   s      @r   r   r     sm    
DDdDDD
D
DCBBBBTBBBMfMM 


  r   c                 R   |d         |d         }}t          |t                    st          |t                    sJ d}t          |t                    rt          |t                    rt          ||j                  r7t           | |j        |j        g|dd          R i |fi t          |          S t          |                    | j        |j        |j                            t          |t                    }|rt          |          nt          |          }|r||fn||f\  }|	                                	                                k    rt          d          t          |          }		                                |		                                dz   k    r7|r	j        |	fn|	j        f\  }
}t           | |
|g|dd          R i |fi |S |	                                |	                                k    r|j        d         |j        d         k    r3t          |                    | j        |j        |j                            ddlm} j        }j        }|j        j        j        dz
           }||                    fdt'          |j                  D                       } ||j        j        |||          }|r|fn|f\  }
} | |
|g|dd          R i |S t          |                    | j        |j        |j                            )	Nr   r   zHcannot call binary pointwise function {} with inputs of shapes {} and {}r   z+NYI: broadcasting NT with T with larger dimnested_from_paddedc                 4    g | ]\  }}|j         k    r|nS r0   rR   )r   rm   rE   r   padded_max_Ss      r   r1   z+jagged_binary_pointwise.<locals>.<listcomp>^  s-    WWWDAq!r~%%QQ<WWWr   r   r3   sum_S
min_seqlen
max_seqlen)r   r	   r   rz   r   r   r%   formatrb   r!   NotImplementedErrorr   r   nested_tensorr   _maybe_min_seqlen_maybe_max_seqlenrR   expandrc   _offsets)rf   rg   rh   rt   rv   mismatch_error_msga_is_ntextracted_kwargsrV   
t_squeezedlhsrhsr   r   r   total_Lt_as_ntr   r   s                    @@r   r   r   $  s   7DGqAa&&E*Q*E*EEEE 	S  !\"" Wz!\'B'B W a)) 	QY	?DH????? CQRSCTCT   -44T]AGQWUUVVVL))G,3J~a(((9J9J )QFFAq6EBuuww!"OPPP%a((J	vvxx:>>##a'''/6TBJ
++Z<TSDDc?DH?????TTCSTTT 	uuww!%%'' 71:##"))$-!'JJ   	655555 )
)
!*"2>A#56"L HHWWWWWIagDVDVWWW
 
 %$K~!!
 
 
 %,>B=='2StC2tABBx2226222 )00QQ
R
RRr   c                 n   | t           j        j        j        u rt	          |i |S | j        dk    r' | |d         j        g|dd          R i | |d         S | j        dk    rdd}t          |||d          \  }}|                    d	          }t          |
                                |d
         |j        dd          }t          |
                                |d         |j        dd          }||k    r|S t          j        t          j        |j        ||dz                      }	g |j        d |         |	|j        |dz   d          R }
 |j        |
 S | j        dk    rdd}t          |||d          \  }}|                    d	          }|                    d          }|
                                |j        z
  dz
  }t%          |          |k    rt'          d          t           j                                        5   | |i |cd d d            S # 1 swxY w Y   t+          |           )Napply_r   r   flattenrF   c                     d S r   r0   )input	start_dimend_dims      r   _flatten_sigz+jagged_torch_function.<locals>._flatten_sig      Dr   Trg   rh   normalize_to_only_use_kwargsr   r   F)r'   r   rms_normc                     d S r   r0   )r   normalized_shapeweightepss       r   _rms_norm_sigz,jagged_torch_function.<locals>._rms_norm_sig  r   r   r   zNrms_norm(): Normalization over the ragged dim not supported for nested tensors)r   rF   NN)rK   _C_nnscaled_dot_product_attentionr   rb   r   r   popr-   r!   rR   r   reduceoperatormulr   reshaper`   ra   DisableTorchFunctionSubclassr   )rf   rg   rh   r   _
new_kwargsinpr   r   product	new_shaper   r   max_normalizables                 r   jagged_torch_functionr   r  s    ux|8882DCFCCC}  T!W_2tABBx2226222Aw }	!!	 	 	 	 +tFQU
 
 
: nnW%% %GGII{#O!&
 
 
	 #GGIIy!O!&
 
 
 J"8<9wQR{;R1STTPci

+PWPsy17OPP	s{I&& }
""	 	 	 	 +VRV
 
 
: nnW%%%>>*<== 7799s6:  #333`   X2244 	) 	)4(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) d
#
##s   HH!$H!zself: jt_allc                 L   | t           j        j        j        j        k    rdS | t           j        j        j        j        k    r|d         j        S | t           j        j        j        j        k    rt          |d         j                  S | t           j        j        j	        j        t           j        j        j
        j        fv r{|d         j        Ot          t          |d         j                  t          j        |d         j        dd                    z            S |d         j        
                                S | t           j        j        j        j        k    r|d         j        S | t           j        j        j        j        k    r|d         j                                        S d S )NFr   r   )rK   opsatenis_non_overlapping_and_densedefaultsym_sizerz   r!   r`   	sym_numelnumelrQ   intr   mathprodr   
sym_stride_stridessym_storage_offsetstorage_offsetrf   rg   rh   s      r   tensor_attr_supported_getterr    sC    uy~:BBBuuy~&...Aw}uy~!)))47=!!!	(0%).2F2NOOO7's47+,,tyaqrr9J/K/KKLLLAw$$&&&uy~(000Awuy~0888Aw--/// 98r   c                     t           j        S r   )rK   jaggedr  s      r   prim_layout_defaultr    s
    <r   c                 b    | t           j        j        j        j        k    rt          d          d S )NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)rK   r   r   r   r  r%   r  s      r   tensor_attr_unsupported_getterr    s8    
 uy~"***9
 
 	
 +*r   c                 "   ddl m} t          | ||d          \  }}|                    d          }|                                dS |                    dt          j                  |d<   |d         t          j        k    rdS  ||j	        fi |S )Nr   )is_contiguous_for_memory_formatTr   r   Fmemory_format)
r   r  r   r   r   r   rK   contiguous_formatpreserve_formatr   )rf   rg   rh   r  r   r   r   s          r   is_contiguous_generalr    s    CCCCCC&4T  MAz ..
!
!C {{}} u",..0# #J /"e&;;;t**3;EE*EEEr   z!self: jt_all, memory_format: any?c                 F   t          | ||d          \  }}|                    d          }|                                dS |                    dt          j                  |d<   |d         t          j        k    rdS t	          j        j        j	        j
        |j        fi |S )NTr   r   Fr  )r   r   r   r   rK   r  r  r   r   sym_is_contiguousr  r   rf   rg   rh   r   r   r   s         r   sym_is_contiguous_generalr    s     '4T  MAz ..
!
!C {{}} u",..0# #J /"e&;;;t9>+3CKNN:NNNr   z"input: jt_all, memory_format: any?c                 `   t          | ||d          \  }}|                    d          }t          |          }|j        U|d         t          j        k    r?ddlm} |j        dk    s
J d             ||	                                d           \  }}|S t           | |j        fi |fi |S )	NTr   r   r  r   )jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r   )r   r   r   rQ   rK   r  r   r  rR   unbindr	   r   )	rf   rg   rh   r   r   r   new_metar  contigs	            r   clone_defaultr"    s     '4T  MAz ..
!
!Cc""H
|o&%*AAA 877777 ?a'''M ('' )(tDDDIFAMS[77J77DD8DDDr   zinput: jt, weight: t, bias: t?c                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S NTr   r   r   r   r	   r   r   r  s         r   linear_defaultr&  :  h    &4T  MAz ..
!
!CS[77J77OO>#;N;NOOOr   z6self: jt, grad_output: jt, weight: t, output_mask: anyc                 v   t          | ||d          \  }}|                    d          }|                    d          }|                    d          }|                    d          }d\  }	}
}t          | |d|d           |d	         r2t          t	          j        |j        |          fi t          |          }	|d
         r|j                            d|	                    d	                    }|j                            d|	                    d
                    }t	          j        |
                                |          }
|d         ret          t          |j        j        d
z
                      }|dk    r|j                                        }nt	          j        |j        |d          }|	|
|fS )NTr   r   grad_outputr   output_mask)NNNselfr   r   rF   r   r0   F)keepdim)r   r   r{   r	   rK   matmulr   r   r   r   rV   r   ranger   cloner   )rf   rg   rh   r   r   r   r)  r   r*  dsdwdbgrad_2dinput_2dreduce_dimss                  r   linear_backward_defaultr6  E  s   
 '4T  MAz ..
!
!C..//K^^H%%F..//K!JBB$V[-HHH1~ 
L,f55
 
9G9T9T
 
 1~ 1 %--b&++a..AA;&&r6;;q>>::\'))++x001~ L E+"5":Q">??@@"$**,,BB;.UKKKBB<r   zinput: jt_all, dtype: anyc                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r$  r%  r  s         r   to_dtyper8  n  r'  r   c                 ^   ddl m} t          | ||d          \  }}|                    d          }|                    d            | |j        fi |}|j                            |j                  }d }	|j         |j                            |j                  }	dd	l	m
}
 dd
lm}m} |j        |j        n|j        }|	|n|	}t          ||
|f          r# ||          } ||          }|j        |_        n||         ||<   t!          |          }||d<   |	|d<   t#          |fi |}|S )Nr   _tensor_symint_registryTr   r   layoutdevicer   
FakeTensorFunctionalTensormb_unwrap_functional_tensorr   r   )r   r;  r   r   r   r   tor>  rQ   torch._subclasses.fake_tensorr@  #torch._subclasses.functional_tensorrB  rC  r   nested_int_memor   r	   )rf   rg   rh   r;  r   r   r   
new_valuesnew_offsetsnew_lengthsr@  rB  rC  ragged_source	new_thingtgtsrc
inp_kwargsr#   s                      r   to_copy_defaultrP  y  s   666666&4T  MAz ..
!
!CNN8ck00Z00J,//):/;;KK
|looZ->o??888888       
 %(L$8CLLclM*2I)j*:;<< T)))44))-88!1-D]-S	*$$J'Jy'Jy*33
33FMr   z-self: jt_all, src: jt_all, non_blocking: any?c                    t          | ||d          \  }}|                    d          }|                    d          }|j        |j        k    r|                                }d |D             }|                                }	d |	D             }
||
k    rt	          d|j         d|j                   t          ||	          D ]\  }}|                    |           |j                            |j                   |S )	NTr   r   rN  c                     g | ]	}|j         
S r0   r   r   cs     r   r1   z copy_default.<locals>.<listcomp>      666q17666r   c                     g | ]	}|j         
S r0   rS  rT  s     r   r1   z copy_default.<locals>.<listcomp>  rV  r   z<copy_(): expected compatible input and src shapes, but got: ry   )	r   r   rz   r  r%   r   r   copy_r   )rf   rg   rh   r   r   r   rN  	inp_compsinp_comp_shapes	src_compssrc_comp_shapesinp_compsrc_comps                r   copy_defaultr_    s.    '4T  MAz ..
!
!C
..

C
yCI JJLL	66I666JJLL	66I666o--/9/ /#&9/ /   #&i";"; 	% 	%HhNN8$$$$ Kck"""Jr   c                 ~   t          | ||d          \  }}|                    d          }t          j        |d<    | |j        fi |}|j                            |j                  }d }|j         |j                            |j                  }t          |          }	d|	v r||	d<   d|	v r||	d<   |j        |j        k    rodd	l
m}
 dd
lm}m} ddlm} |j        |j        n|j        }||n|}t#          ||
|f          r# ||          } ||          }|j        |_        n||         ||<   t'          |fi |	S )NTr   r   r<  r=  r   r   r   r?  rA  r   r:  )r   r   rK   stridedr   r   rD  r>  rQ   r   rE  r@  rF  rB  rC  r   r;  r   rG  r	   )rf   rg   rh   r   r   r   rH  rI  rJ  output_kwargsr@  rB  rC  r;  rK  rL  rM  rN  s                     r   like_factory_defaultrc    s    '4T  MAz ..
!
!C
 !=Jxck00Z00J,//):/;;KK
|looZ->o??"3''MM!!#.i M!!#.i 
zZ&&& 	=<<<<<	
 	
 	
 	
 	
 	
 	
 	

 	;:::::(+(<#,#.#6KKK	i*.>!?@@ 	X--i88C--m<<C"%"5C1H1W#I.
44m444r   zself: jt_all, fill_value: anyzself: jt_all, high: anyz!self: jt_all, low: any, high: anyc                 |    t          | ||d          \  }}|                    d          } | |j                   |S r$  r   r   r   r  s         r   zero__defaultrf    sM    &4T  MAz ..
!
!CDJr   z*self: jt_all, dim: any, half_to_float: anyc           
      (   t          | ||d          \  }}t          |d         t                    rt          d          |                    d          }t          |                                |d         fd|j                  \  |d<   }}}|rt          d          |r|j        dk    rt          d	          |r|j        t          d
          |d         d         |d<   |rt          j
        j                            t          j        j                            |j                            |j        j        d         d          |j        g|j        gt+          d                    |j                  }	t          j        j                            |	|j        g|j        j        d                   j        dg|j        j        dd          R  }
t/          |
fi t1          |          S t/           | |j        fi |fi t1          |          S )NTr   r!   zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorr   softmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorr   zesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorr   rF   z-inf)max_lengthspadding_valuer!   r   )r   r   r   r%   r   r<   r!   rR   rQ   rK   nn
functionalrh  r   r   _jagged_to_padded_dense_forwardr   r   r   r   _max_seqlenfloat_padded_dense_to_jagged_forwardr	   r   )rf   rg   rh   r   r   r   reduce_on_batchreduce_on_ragged_reduce_on_non_batchpadded_softmax_valuessoftmax_valuess              r   _softmax_defaultrx    s}    '4T  MAz *U#U++ 
V
 
 	
 ..
!
!C 				E			 	5  
`
 
 	
  
COa//s
 
 	
  
CL4I
 
 	

 #5)	Ju  C % 3 ; ;IN::##K%a("   _-#Fmm ;    !< 
!
 
!
 GG!\NK% H 
 
 
 "122&
 
 
 NBBnS.A.ABBBS[77J77OO>#;N;NOOOr   c                    t          | ||d          \  }}t          |d         t                    rt          d          |                    d          }t          |                                |d         fd|j                  \  |d<   }}}|rt          d          |rt          d          |d         d	         |d<   t           | |j	        fi |fi t          |          S )
NTr   r!   zLlog_softmax(): not supported for dimensions of type 'tuple' for NestedTensorr   log_softmaxzVlog_softmax(): not supported when reducing across the batch dimension for NestedTensorzVlog_softmax(): not supported when reducing along the ragged dimension for NestedTensorr   )r   r   r   r%   r   r<   r!   rR   r	   r   r   )	rf   rg   rh   r   r   r   rs  rt  ru  s	            r   _log_softmax_defaultr{  c  s,    '4T  MAz *U#U++ 
Z
 
 	
 ..
!
!C 			Ju%'	 	5
  
d
 
 	
  
d
 
 	

 #5)!,JuS[77J77OO>#;N;NOOOr   z7grad_output: jt, output: jt, dim: any, input_dtype: anyc                     t          | ||d          \  }}|                    d          }|                    d          }t           | |j        |j        fi |fi t	          |          S )NTr   r)  r#   r%  )rf   rg   rh   r   r   grad_outr#   s          r   _softmax_backwardr~    s    
 '4T  MAz ~~m,,H^^H%%FXv~<<<< @Nx@X@X  r   z!self: jt, float: any, train: any?c                     t          | ||d          \  }}|                    d          } | |j        fi |\  }}t          |fi t	          |          t          |fi t	          |          fS r$  r   r   r   r	   r   )rf   rg   rh   r   r   r   out1out2s           r   native_dropout_defaultr    s     '4T  MAz ..
!
!Cck00Z00JD$T11^C0011T11^C0011 r   z%grad_output: jt, mask: jt, scale: anyc                     t          | ||d          \  }}|                    d          }|                    d          }t           | |j        |j        fi |fi t	          |          S )NTr   r)  maskr%  )rf   rg   rh   r   r   r)  r  s          r   native_dropout_backward_defaultr    s    
 '4T  MAz ..//K>>&!!D[ $,==*== 

%
%  r   z2self: jt_all, dim: any, keepdim: any?, dtype: any?c                 &    t          | ddg|R i |S )Nr  r   _apply_reductionr  s      r   prod_dim_intr    s&    
 D&!=d===f===r   zself: jt_all, dtype: any?c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r$  re  r  s         r   prod_defaultr    P    &4T  MAz ..
!
!C4**z***r   z$self: jt, split_size: any, dim: any?c                    t          | ||d          \  }}|                    d          t                                          |d         j        d          |d<   t          fd | j        fi |D                       S )NTr   r   r!   r^   c              3   N   K   | ]}t          dd |it                    V   dS )valuesNr0   r	   r   r   rE   r   s     r   r   zsplit_tensor.<locals>.<genexpr>  sR         	55A5!4!455     r   )r   r   r-   r!   rR   r   r   r  s        @r   split_tensorr    s     '4T  MAz ..
!
!C(		:e$cow Ju     ck00Z00     r   z%self: jt, split_sizes: any, dim: any?c                     t          | ||d          \  }}|                    d          t                                          |d         j        d          |d<   fd | j        fi |D             S )NTr   r   r!   split_with_sizesc           	      F    g | ]}t          dd |it                    S r  r0   r  r  s     r   r1   z,split_with_sizes_default.<locals>.<listcomp>  sD        	55A5!4!455  r   )r   r   r-   r!   rR   r   r  s        @r   split_with_sizes_defaultr    s     '4T  MAz ..
!
!C(		:e$co7I Ju   ck00Z00   r   z+self: jt, dim: any, start: any, length: anyc                 *   t          | ||d          \  }}|                    d          }t          |                                |d         |j        d          } | |j        ||d         |d                   }t          |fi t          |          S )	NTr   r   r!   narrowstartlengthr!   r  r  )r   r   r-   r!   rR   r   r	   r   )rf   rg   rh   r   r   r   r!   r  s           r   r  r    s     '4T  MAz ..
!
!C
37799j&7(
S
SCT!(#	  F 66."5"5666r   z self: jt, chunks: any, dim: any?c                 \   t          | ||d          \  }}|                    d          t                                          |d         j        dd          \  |d<   }|r|d         }j                                        }|                    |          }d |D             }	d	 |	D             }	fd
|	D             d |D             }
j        	                    |
          fdt          dt                              D             S fd | j        fi |D             S )NTr   r   r!   chunkr)   chunksc                 :    g | ]}t          j        |d           S )r   rk  )rK   cumsumrD   s     r   r1   z!chunk_default.<locals>.<listcomp>  s'    KKKa5<q111KKKr   c                 <    g | ]}t          j        |d d          S )r   r   r   )value)FpadrD   s     r   r1   z!chunk_default.<locals>.<listcomp>  s)    NNN15F!444NNNr   c                 $    g | ]}|j         d S ))r   rR   r   )r   per_offsetsr   s     r   r1   z!chunk_default.<locals>.<listcomp>  s3     
 
 
 $CODD
 
 
r   c                 Z    g | ](}|                                                                 )S r0   )r   itemrD   s     r   r1   z!chunk_default.<locals>.<listcomp>"  s(    ???!quuww||~~???r   c                 D    g | ]}t          dd |         i|         S r  r   )r   rm   chunk_valuesnested_kwargss     r   r1   z!chunk_default.<locals>.<listcomp>'  sE     
 
 
 FFQFM!4DFF
 
 
r   r   c           	      F    g | ]}t          dd |it                    S r  r  r  s     r   r1   z!chunk_default.<locals>.<listcomp>,  sD     
 
 
 999^C%8%899
 
 
r   )r   r   r-   r!   rR   r   diffr  r   r^   r.  r`   )rf   rg   rh   r   r   r,   r  r   chunked_lengthschunked_offsetssplit_sizesr  r   r  s              @@@r   chunk_defaultr    s   &4T  MAz ..
!
!C,<		:e$cowPT- - -)Ju)  
H% ,##%%!--//KK?KKKNNoNNN
 
 
 
.
 
 
 @????{((55
 
 
 
 
1c,//00
 
 
 	


 
 
 
T#+4444
 
 
 	
r   zself: jt_all, dim: any?c                   
 t          | ||d          \  }}|d         }|dk    rt          d          |                    d          }|                                |                                }|                                }|j        ddt          t                   dt          t          t                            ffd	}	|K|
                                                                
 |	
           t          j        
d
z
            S dk    rt          d          |                                
|                                 |	
           
fdt          |j        d                   D             S )NTr   r!   r   z2unbind(): only supported for NestedTensor on dim=0r   rQ   r   c                 ,   d}j         dz
           }t          t          |                     D ]p}t          j        | |                    t          j        | |         |k               || |         z  }|)t          j        ||         | |         z   |k    d            qt          j        ||k               |Wt          t          |                    D ]<}t          j        ||                    t          j        ||         |k               ;d S d S )Nr   r   c                      dS )NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionr0   r0   r   r   rN   z2unbind_int.<locals>._torch_check.<locals>.<lambda>W  s    k r   )r   r.  r`   rK   _check_is_size_check)rQ   r   lengths_sumragged_dim_sizerm   r3   r  s        r   _torch_checkz unbind_int.<locals>._torch_checkC  s&     ,zA~6s8}}%% 		 		A !---L!78888A;&K#QK(1+-@kk   	[O34443x==)) = =$Xa[111Xa[O;<<<<  = =r   r   rk  zAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)c           	      ^    g | ])}t          j        d z
  |         |                   *S )r   r  )rK   r  )r   rm   lengths_scalarsoffsets_scalarsr3   r  s     r   r1   zunbind_int.<locals>.<listcomp>p  sV         	a!!$"1%		
 	
 	
  r   r   )r   r%   r   r  r   r   rR   r   r  r   r  tolistrK   r^   r.  r   )rf   rg   rh   r   r   r!   r   r   r   r  r  r  r3   r  s             @@@@r   
unbind_intr  2  s    '4T  MAz U
C
axxOPPP
..
!
!CZZ\\FkkmmGkkmmGJ= =tCy =HT#Y4G = = = = = = =: !,,..//11_%%%{6?aIIIIQO
 
 	
 nn&&Onn&&OL/222       w}Q'((   r   zself: jt, dim: anyc                    t          | ||d          \  }}|                    d          }|j        }t          t	          |j                  |d         |j        d          |d<   t           | |fi |fi t          |          S )NTr   r   r!   r   )	r   r   r   r-   r`   rz   rR   r	   r   )rf   rg   rh   r   r   r   r  s          r   squeeze_dimr  {  s    &4T  MAz ..
!
!C[F(CI
5)3?I Ju V22z22JJnS6I6IJJJr   zself: jt_all, dim: anyc                 f   t          | ||d          \  }}|                    d          }|j        }|d         }t          t	          |j                  dz   ||j        dd          |d<   t          |          }|d         |j        dz
  k    r|dxx         dz  cc<   t           | |fi |fi |S )	NTr   r   r!   r   	unsqueeze)r(   rR   )	r   r   r   r-   r`   rz   rR   r   r	   )	rf   rg   rh   r   r   r   r  r!   rb  s	            r   unsqueeze_defaultr    s    &4T  MAz ..
!
!C[F U
C(CIC+PT  Ju
 #3''M%COa///m$$$)$$$V22z22DDmDDDr   ztensors: any, dim: anyc                    t          | ||d          \  }}|                    d          }d |D             }t          |          dk    sJ |d         fd|D             }|d         }t          t          j                  |j        d          |d<   t           | d	 |D             fi |fi t          |d                   S )
NTr   tensorsc                      g | ]}|j         	|S r0   )	is_nestedr   rV   s     r   r1   zcat_default.<locals>.<listcomp>  s    000AAK0a000r   r   c                 L    g | ] }|j         r|n|                              !S r0   )r  	expand_as)r   rV   firsts     r   r1   zcat_default.<locals>.<listcomp>  s0    IIIAAK7qqQ[[%7%7IIIr   r!   catc                     g | ]	}|j         
S r0   r   r  s     r   r1   zcat_default.<locals>.<listcomp>      )))Aai)))r   )r   r   r`   r-   r   rR   r	   r   )	rf   rg   rh   r   r   r  nestedr!   r  s	           @r   cat_defaultr    s	   &4T  MAz nnY''G 10000Fv;;????1IEIIIIIIIG U
C(EK#u0% Ju )))))88Z88 <J7ST:<V<V  r   zself: any, other: anyc                     t           ||d          \  }}|                    d          }|                    d          } fd} fd}|j        r |j        s|                                dk    rV|                                |                                k    r,|j        |                                dz
  k     r |||          S |                                d	k    rq|                                |                                k    rG|j        |                                dz
  k     r't            |j        |fi |fi t          |          S n|j        s|j        r|                                dk    rA|                                |                                k    r|j        d	k    r |||          S |                                d	k    r\|                                |                                k    r2|j        d	k    r't            ||j        fi |fi t          |          S n|j        r|j        r|                                dk    r[|                                dk    rCt          ||j	                  r.t            |j        |j                  fi t          |          S |                                dk    r|                                dk    rj|j        d	k    r_|j        dk    rT|
                    |j                  |
                    |j                  k    rt          j         |||                    S t          d
|j	         d|j                   )NTr   r   otherc                     fdt          |                                 |                                          D             S )Nc                 .    g | ]\  }} ||          S r0   r0   )r   a_compb_comprf   s      r   r1   z8matmul_default.<locals>._unbind_impl.<locals>.<listcomp>  s6     
 
 
%5ffDD  
 
 
r   )r   r  )rt   rv   rf   s     r   _unbind_implz$matmul_default.<locals>._unbind_impl  sH    
 
 
 
9<QXXZZ9T9T
 
 
 	
r   c                    | j         r| }n|}ddlm} |j        }|j        }|}|j        j        |j        dz
           }||}g |j        d |j                 ||j        |j        dz   d          R }|                    d|          }	| j         r |	|          }
n | |	          }
 ||
|j	        |j        |||          S )Nr   r           output_sizer   )
r  r   r   r   r   r   r   rR   to_padded_tensorr   )rt   rv   r   r   r   r   r   r   padded_shape	padded_ntpadded_trf   s              r   _padded_implz$matmul_default.<locals>._padded_impl  s"   ; 	BBB555555)
)
!*"2>A#56"L
X&&'

 Xbnq(**+
 

 '''FF	; 	*tIq))HHtAy))H!!K~!!
 
 
 	
r      r   r   z1matmul(): not supported between inputs of shapes ry   )r   r   r  r!   rR   r	   r   r   r   rz   r   rK   stackr%   r   )	rf   rg   rh   r   r   r   r  r  r  s	   `        r   matmul_defaultr    s{   &4T  MAz ..
!
!CNN7##E
 
 
 
 

!
 !
 !
 !
 !
J } 49U_ 49 GGIINN		UYY[[((#''))a-//  <U+++ IIKK1		EIIKK''#''))a-//S[%66:66 :H:M:M   ] 9u 999;;!		swwyy 8 8U=NRS=S=S<U+++
 WWYY!^^		cggii 7 7E<MQR<R<RS%-66:66 :H:O:O  
 
 95? 9 7799q==UYY[[1__1CC1U1U_S[%- @ @XXNSVDWDWXXX GGIINN		q  1$$!Q&&))UZZ8I-J-JJJ ;||C77888
YCIYYEKYY  r   zself: jt_all, mat2: anyc                 r   t          | ||d          \  }}|                    d          }|                    d          }|                                dk    rt          d          |                                dk    rt          d          t	          t
          j        j        j        j	        ||          S )NTr   r   mat2r  zbmm(): input must be 3Dzbmm(): mat2 must be 3D)
r   r   r!   ra   r  rK   r   r   r-  r  rf   rg   rh   r   r   r   r  s          r   bmm_defaultr  (  s    &4T  MAz ..
!
!CNN6""E
wwyyA~~2333yy{{a1222%)./7eDDDr   z'self: jt_all, size: any, implicit: any?c                    t          | ||d          \  }}|                    d          |d         d|vs|                    d          rJ t                    st          dj         d           fdt          d	                                          D             }t           | j        |          fi t                    S )
NTr   r   r   implicitzexpand(): cannot expand shape z -> c                 :    g | ]}|j         k    rd n|         S )rF   r   )r   r   r   r   s     r   r1   z"expand_default.<locals>.<listcomp>H  s-    WWWaS_,,""$q'WWWr   r   )
r   r   r   r%   rz   r.  r!   r	   r   r   )rf   rg   rh   r   r   
expand_argr   r   s         @@r   expand_defaultr  9  s     '4T  MAz ..
!
!CfDj((*..2L2L((Mc4(( SQCIQQ4QQRRRWWWWW5CGGIICVCVWWWJS[*55MM9L9LMMMr   zself: t, other: jtc                     t          | ||d          \  }}|                    d          }|                    d          }t           | ||j                  fi t	          |          S )NTr   r   r  r%  r  s          r   expand_as_defaultr  L  ss    &4T  MAz ..
!
!CNN7##ES%-00JJN54I4IJJJr   zself: jt_all, size: anyc           
         t          | ||d          \  }}|                    d          }|                    d          }t          |          |                                k    rR|                    g d t          |                                t          |          z
            D             |          S t          d          )NTr   r   r   c              3      K   | ]}d V  dS )r   Nr0   )r   r   s     r   r   zbroadcast_to.<locals>.<genexpr>b  s"      EE1QEEEEEEr   zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)r   r   r`   r!   r   r.  ra   )rf   rg   rh   r   r   r   r   s          r   broadcast_tor  X  s    &4T  MAz ..
!
!C>>&!!D
4yyCGGIIzzMEEcggii#d)).C(D(DEEEMMNNN
	4  r   ztensors: anyc           
         t          | ||d          \  }}|                    d          }t          |          dk    rt          d          t          |          dk    r|d         S g }t	          j        d |D              }t          d |D                       }|D ]}	|	j        r)|                    |		                    |                     2|	
                                t          |          k     rK|                    t          |		                    |j        j                  fi t          |                     t          d	          t          |          S )
NTr   r  r   z7broadcast_tensors(): expected at least one tensor inputr   c              3   $   K   | ]}|j         V  d S r   rS  r  s     r   r   z$broadcast_tensors.<locals>.<genexpr>w  s$      .H.H1qw.H.H.H.H.H.Hr   c              3   D   K   | ]}t          |t                    |V  d S r   rT   r  s     r   r   z$broadcast_tensors.<locals>.<genexpr>y  s1      AAQZ<%@%@AqAAAAAAr   zubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)r   r   r`   ra   rK   broadcast_shapesr   r  r   r  r!   r	   r   r   r   r   )
rf   rg   rh   r   r   r  outsbroadcast_shaper   rV   s
             r   broadcast_tensorsr  j  sr   &4T  MAz nnY''G
7||qRSSS
7||qqzD,.H.H.H.H.HIO
AA'AAA
A
AC  ; 
	KK778888UUWWs?++++KKQ^^CK,=>>VV.QTBUBUVV    ;  
 ;;r   z(condition: jt_all, self: any, other: anyc                 D   t          | ||d          \  }}|                    d          }|                    d          }|                    d          }t          j        |||          \  }}}t	           | |j        |j        |j        fi |fi t          |          S )NTr   	conditionr   r  )r   r   rK   r  r	   r   r   )rf   rg   rh   r   r   r  r   r  s           r   
where_selfr    s     '4T  MAz {++I
..
!
!CNN7##E "3IsEJJIsEYU]IIjII 

#
#  r   zself: jt, device: any?c                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r$  r%  r  s         r   _pin_memory_defaultr    r'  r   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r$  re  r  s         r   is_pinned_defaultr    r  r   zself: jt_all, other: jt_allc                 :    |d         j         |d         j         k    S r   )rz   r  s      r   is_same_size_defaultr    s     7=DGM))r   c                 ~   t          | ||d          \  }}|                    d          |d         d u p:t          |d         t          t          f          ot          |d                   dk    }|r | j        fi |}|                    dd          r[t          |t          t          f          r% t          |          fd|D                       }n|	                    j
                  }|S |d         }	t          |d         t          t          f          }
|
s|	g}	t                                          |	| j
                  \  }}}}|
s|d         }||d<   |rj        t          | d	          dd
lm} |r|r3 | j        fi |}|                    dd          r |d |          }|S |rt          | d          |                    d           |
rj
        gnj
        } |                     |          fd|i|S |rt          | d           | j        fi |}t%                    |                    dd          sVt          |d         t          t          f          r|d         n|d         g}|D ] }|j
        dz
  k     rdxx         dz  cc<   ! |fd|          S )NTr   r   r!   r   r,  Fc              3   L   K   | ]}|                     j                  V  d S r   )r  rR   )r   or   s     r   r   z#_apply_reduction.<locals>.<genexpr>  s1      JJCO < <JJJJJJr   zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapc                 ,    |                      d          S )Nr   )r  )r  s    r   rN   z"_apply_reduction.<locals>.<lambda>  s    Q r   zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsr   rR   c                     t          | fi S r   r   )r  
out_kwargss    r   rN   z"_apply_reduction.<locals>.<lambda>#  s    ,q"?"?J"?"? r   )r   r   r   r   r   r`   r   r   r   r  rR   r<   r!   rQ   r%   torch.utils._pytreer	  r  r   )rf   	func_nameidentity_elementrg   rh   r   r   full_reductionoutdim_to_convert
is_dimlistconverted_dimrs  rt  reduce_on_non_batchr	  dim_to_passdimlistr   r   r  s                      @@r   r  r    s   &4T  MAz ..
!
!C  &$. :e$udm44TZ=N9O9OST9T   d3;--*-->>)U++ 	5#t}-- 5d3iiJJJJcJJJJJmmCO44
  &NJu-t}==J *() 					 	  )%a(%Ju 
CL4 ; ; ;
 
 	

 -,,,,,  2F 	$s{11j11C~~i// >h77==J # "  : : :   NN5!!!/9N3?++sK4$$%566 <GKU    	 K K K   d3;--*--#C((
~~i// 
	3 j/%??)
5!! '( 
  3 3s***}---2--- x????EEEr   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r$  re  r  s         r   sum_defaultr  &  r  r   z3self: jt_all, dim: any?, keepdim: any?, dtype: any?c                 &    t          | ddg|R i |S )Nr   r   r  r  s      r   sum_dim_IntListr  1  s&    
 D%<T<<<V<<<r   z"self: jt_all, dim0: any, dim1: anyc           
      ~   t          | ||d          \  }}ddlm} |                    d          } ||                                |d         |d         f          \  }}||j        k    s||j        k    r|dk    s|dk    rt          d          ||j        k    r|}	n|}	t          |          }
|	|
d	<   t          |	                                
                    t          t          |j                  ||j                  t          t          |j                  ||j                            fi |
S t          |                                |d         |j        d
          |d<   t          |                                |d         |j        d
          |d<   t           | |j        fi |fi t          |          S )NTr   r   r   r   dim0dim1z?Transpose is not supported on the batch dimension for jagged NTrR   	transpose)r   r   r   r   r!   rR   ra   r   r	   r  r  r   r`   rz   r-   r   )rf   rg   rh   r   r   r   r   r  r  to_dimrO  s              r   transpose_intr!  9  s    '4T  MAz 655555
..
!
!C""37799z&/A:fCU.VWWJD$
 s$#/"9"9199		Q   3?""FFF#C((
$*
=!JJLL""#C	NND#/JJ#C	NND#/JJ 
 

 
 
 	
 *		:f%s Jv *		:f%s Jv S[77J77OO>#;N;NOOOr   zself: jt_all, dims: anyc                   
 t          | ||d          \  }}|                    d          
|                    d          }t          
          }t          
j                  t          |          k    r't          d ddt          |           dz             d	d
lm}  ||          }t          |          t          t          |                    k    rt          d          
j	        t          d          |d	         d	k    rt          d          |
                    
j                  |d<   
fd|dd          D             }	|	|d<   t           | 
j        fi |fi |S )NTr   r   r6   z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().r   r   z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrR   c                 <    g | ]}t          |j                  S r0   )r   rR   )r   r!   r   inp_dims     r   r1   z#permute_default.<locals>.<listcomp>  s7        	GS#/::  r   r   )r   r   r   r`   rz   ra   r   r   setrQ   indexrR   r	   r   )rf   rg   rh   r   r   r6   rO  r   canonicalized_dims
inner_dimsr   r%  s             @@r   permute_defaultr*  f  s   &4T  MAz ..
!
!C>>&!!D$$J#)nnG #d))OGOOO_RUVZR[R[___`
 
 	

 655555**7D99
#c*<&=&=">">>>EFFF
|P
 
 	
 !!!K
 
 	
 !3 8 8 I IJ}    %abb)  J $JvS[77J77FF:FFFr   c                    t          | ||d          \  }}|                    d          |                    d          j        dk    r@t          j                  t                    k    rt          dj         d d          t                    d	k     st                    st          d
j         d           fdfdt          t                    dz
            D             }t          j
                                                  5  t           | j        |          fi t                    cd d d            S # 1 swxY w Y   d S )NTr   r   r   r   zVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (r#  r  zview(): cannot view shape z as c                 j    | j         dz
  k    rj                            |           S | dz            S rP   )rR   r   r   )	inner_idxr   r   s    r   get_inner_sizez$view_default.<locals>.get_inner_size  s:    !+++;##I...	A&&r   c                 &    g | ]} |          S r0   r0   )r   rm   r.  s     r   r1   z view_default.<locals>.<listcomp>  s#    BBB..##BBBr   )r   r   rR   r   rz   r%   r`   r   r.  rK   inference_modeis_inferencer	   r   r   )	rf   rg   rh   r   r   
inner_sizer.  r   r   s	         @@@r   view_defaultr3    s   
 '4T  MAz ..
!
!C>>&!!D
!ci 0 0E$KK ? ?@ Y@ @7;@ @ @
 
 	
 4yy1}}.sD99}M	MMtMMNNN ' ' ' ' ' ' CBBBU3t99q=-A-ABBBJ 
	c..00	1	1 R RDDj99QQ^C=P=PQQR R R R R R R R R R R R R R R R R Rs   #)EE EzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anyc                    t          | ||d          \  }}|                    d          }|                                dk    rt          d          |d         }|j        |j                 }|                                t          |          z
  }|dk    rt          d          ||v r|j        t          d	          ||v rt          j	        j
                            |j                            |j        
          |j        g|j        g          }	t          j	        j
                            t          j        |j        j        d         df|j        |j                  |j        g|j        g                              |	j                  }
|j                                                            d                              d          |	j        d         z  }t          j        |	dd          |z  }|	|z
  |
z  }t          j        t          j        |          dd          |z  }t          j        ||d         z             }||z  }t          j	        j
                            ||j        g|j        j        d                                       d|j        |j        dz   d                    }t7          |fi t9          |          ||fS  | |j        fi |\  }}}t7          |fi t9          |          ||fS )NTr   r   r   zOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsr   r   zVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr   )ri  r   )r>  dtype)r   r   )r!   r,  r   rl  rF   )r   r   r!   r%   r   rR   r`   rQ   rK   r   r   ro  r   r   r   rp  onesr>  r6  r   r  r  r   squaresqrtrr  	unflattenr	   r   )rf   rg   rh   r   r   r   r   ragged_sizenum_dims_not_normalizedpadded_inputpadded_maskragged_lengthsmeanpadded_normalizedvariancestdpadded_layer_normjagged_layer_norm_valuesr#   s                      r   native_layer_norm_defaultrF    sm   
 '4T  MAz ..
!
!C
wwyyA~~]
 
 	
 ""45)CO,K!ggii#.>*?*?? 	 1$$d
 
 	
 &&&3<+Cy
 
 	

 	'''y~EEK/     \N) F 
 
 inDDJ)!,a039UUU\N) E 
 
 &
 
	 	 L))!,,66q99L<Nq<QQ 	
 I  
  	 D K/ 	
 I.//  
  	 jJu$5566-3#(9>#Q#Q\NK% $R $
 $
 )	#/A-//0
 
 	! 1II^C5H5HII
 	
 S[77J77FD#77>##6#677sCCr   zpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc                     t          | ||d          \  }}|                    d          }|                    d          } | |j        |j        fi |\  }}}	|d ||	fS t          |fi t	          |          ||	fS )NTr   r}  r   r  )
rf   rg   rh   r   r   r}  r   d_inputd_gammad_betas
             r   "native_layer_norm_backward_defaultrK  '  s    
 '4T  MAz ~~j))H
..
!
!C#tH$4ckPPZPPGWfgv&&88N3$7$788'6JJr   z"self: jt_all, dim: any, index: anyc                    t          | ||d          \  }}|                    d          }t          |                                |d         |j        dd          \  |d<   }|r |                                |d                  S |j        t          d          t          |          }|d         |j        d	z
  k     r|d
xx         d	z  cc<   t           | |j
        fi |fi |S )NTr   r   r!   selectr  r'  zSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesr   rR   )r   r   r-   r!   rR   r  rQ   ra   r   r	   r   )rf   rg   rh   r   r   r   r,   r  s           r   
select_intrN  8  s   &4T  MAz ..
!
!C,<		:e$coxQU- - -)Ju)  1zz||Jw/00
|a
 
 	

  $$J%3?Q...=!!!Q&!!!S[77J77FF:FFFr   z7self: jt, dim: any?, start: any?, end: any?, step: any?c                    t          | ||d          \  }}|                    d          }t          |                                |d         |j        d          |d<   t           | |j        fi |fi t          |          S )NTr   r   r!   slice)r   r   r-   r!   rR   r	   r   r   r  s         r   slice_tensorrQ  U  s    
 '4T  MAz ..
!
!C(		:e$cow Ju S[77J77OO>#;N;NOOOr   z8input: jt_all, indices: any, values: t, accumulate: any?c                    t          | ||d          \  }}|                    d          }|                    d          }t          |          |                                k    sJ t          |          |j        dz   k     r|                                st          d          ddlm} |j	        }|j
        }	|	}
|j        j        |j        dz
           }|
|}
g |j        d |j                 |
|j        |j        dz   d          R }|                    d|	          } | | ||fi ||j        |j        |||	
          }| t          j        j        j        j        k    r.|j                            |                                           |S |S |                                '|                                                                }n|                                }t          j        t          j        ||j                 |k               d           |                                |d                  ||j                 z   }|d|j                 |gz   ||j        dz   d          z   }| t          j        j        j        j        k    r | |j        |fi ||_        |S t7           | |j        |fi |fi t9          |          S )NTr   r   indicesr   z[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsr   r  r  r   z7Some indices in the ragged dimension are out of bounds!r   )r   r   r`   r!   rR   is_contiguousr%   r   r   r   r   r   r   r  r   rK   r   r   
index_put_r  rX  r  r   r   r  _assert_asyncr   r	   r   )rf   rg   rh   r   r   r   rS  r   r   r   r   r   r  
padded_inpnew_njtr   ragged_indicesfunc_indicess                     r   rU  rU  f  s    '4T  MAz #w//C nnY''Gw<<37799$$$$
7||co)))  "" 	m   	655555*
*
!+#COa$78"L
Y(()

 Ys*,,-
 

 ))#<)HH
$$DW33
33L!!
 
 
 59>,444Kgnn..///J
 {{}}++--$$&&++--		'#/*W455A   [[]]71:.1IIN 	CO#$
	 #/A%''
(		)  uy~(000d3;CC
CC
S[,55*55 


  r   z{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anyc                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r$  r%  r  s         r   convolution_defaultr\    sj     '4T  MAz ..
!
!CS[77J77OO>#;N;NOOOr   c                 N   t          | ||d          \  }}|d         }t          |                                |d         d|j                  \  }}}}|r|s|rJ |d         }	d|d<   t	          t
          j        j        j        j	        ddfi |}
|j
        |j
        n|j                                        }t          |
                                dz
            D ]}|                    d	          }|
|z  }|	s|                    |j                  }|S d
}t	          | d|fi |S )NTr   r   r!   r@  r,  r   r   rF   gzG?)r   r<   r!   rR   r  rK   r   r   r   dim_IntListrQ   r   r  r.  r  r   )rf   rg   rh   r   r   r   rs  rt  r  r,  intermediate_sumr   r  intermediate_values                 r   mean_dimra    st    '4T  MAz W
CBS		5	C C?Q)+>   &&&&Y' $
9+IN*FA
 
9C
 

 #&,":#,,@Q@Q@S@S'++--122 	, 	,A''++GG( 	/++co..C

 D&*<KK
KKKr   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r$  re  r  s         r   mean_defaultrc    r  r   z&self: jt_all, dim: any?, keepdim: any?c                 &    t          | ddg|R i |S )Nr5   Fr  r  s      r   any_dimsre    s$    D%@@@@@@@r   z%self: jt_all, dim: any, keepdim: any?c                     t          | ||d          \  }}|d         g|d<   t          t          j        j        j        j        fi |S NTr   r!   )r   re  rK   r   r   r5   r6   rf   rg   rh   r   r   s        r   any_dimri  
  W    &4T  MAz
 $E*+JuEIN&+::z:::r   c                 &    t          | ddg|R i |S )Nr   Tr  r  s      r   all_dimsrl    s$    D%???????r   c                     t          | ||d          \  }}|d         g|d<   t          t          j        j        j        j        fi |S rg  )r   rl  rK   r   r   r   r6   rh  s        r   all_dimrn    rj  r   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r$  re  r  s         r   all_any_max_min_defaultrp  %  sR     '4T  MAz ..
!
!C4**z***r   c                     t          | ||d          \  }}t          j        |d         j                  j        }t          | d|g|R i |S )NTr   r   minr   rK   finfor6  maxr  rf   rg   rh   r   r   	dtype_maxs         r   min_dimrx  8  a    &4T  MAz Jw/566:ID%DTDDDVDDDr   c                     t          | ||d          \  }}t          j        |d         j                  j        }t          | d|g|R i |S )NTr   r   ru  r   rK   rt  r6  rr  r  rf   rg   rh   r   r   	dtype_mins         r   max_dimr~  B  ry  r   c                     t          | ||d          \  }}t          j        |d         j                  j        }t          | d|g|R i |S )NTr   r   aminrs  rv  s         r   amin_defaultr  L  c     '4T  MAz Jw/566:ID&)EdEEEfEEEr   c                     t          | ||d          \  }}t          j        |d         j                  j        }t          | d|g|R i |S )NTr   r   amaxr{  r|  s         r   amax_defaultr  X  r  r   c                     t          | ||d          \  }}t          j        |d         j                  j        }t          | d|g|R i |S )NTr   r   argminrs  rv  s         r   argmin_defaultr  d  c     '4T  MAz Jw/566:ID(IGGGGGGGr   c                     t          | ||d          \  }}t          j        |d         j                  j        }t          | d|g|R i |S )NTr   r   argmaxr{  r|  s         r   argmax_defaultr  p  r  r   zAgrad: jt_all, dim: any, indices: jt_all, sizes: any, keepdim: anyc                 P  
 ddl m
 t          | ||d          \  }}|                    d          }|j        |d<   |                    d          }|j        |d<   t          
fdt          |d                   D                       }t          t          |d                   |d	         |d
          |d	<   |                    d          }|j        	                    |j
        dz
            ||<   |dd          }||d<   t          |          }	||	d<   t           | di |fi |	S )Nr   r   Tr   gradrS  c              3   8   K   | ]\  }} |          |V  d S r   r0   )r   rm   r   r   s      r   r   z=value_selecting_reduction_backward_default.<locals>.<genexpr>  s7      WWDAqmmTUFVFVWaWWWWWWr   sizesr!   "value_selecting_reduction_backwardr   rR   r0   )r   r   r   r   r   r   rc   r-   r`   r   rR   r   r	   )rf   rg   rh   r   r   r  rS  r3   r  rb  r   s             @r   *value_selecting_reduction_backward_defaultr  |  s_   
 DCCCCC&4T  MAz >>&!!DJvnnY''G#OJyWWWWIj.A$B$BWWWWWJ(Jw  5,	 Ju NN7##E,,W-@1-DEEE*!""IEJw"7++M#-M- **z**<<m<<<r   c                 `   t          | ||d          \  }}|                    d          }|D ]}t          |t                    st	          d          |                                |d                                         k    rt	          d          t          ||d         j                  st	          d          t          |d                                         dz   |d	         |d         j	        d
          |d	<   t           | d |D             fi |fi t          |d                   S )NTr   r  z+stack(): expected all nested tensors inputsr   z9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurer   r!   r  c                     g | ]	}|j         
S r0   r   r  s     r   r1   z!stack_default.<locals>.<listcomp>  r  r   )r   r   r   r	   r%   r!   r   r   r-   rR   r   )rf   rg   rh   r   r   r  rV   s          r   stack_defaultr    sb   &4T  MAz
 nnY''G  !\** 	NLMMM5577gajnn&&&&K   "!WQZ%566 	X  	
 )
1j/1G Ju )))))88Z88 <J7ST:<V<V  r   zQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?c                     t          | ||d          \  }}|                    d          }|                    d          }t           | ||j        fi |fi t	          |          S )NTr   rS  r   r%  )rf   rg   rh   r   r   rS  r   s          r   embedding_defaultr    s    
 '4T  MAz
 nnY''G^^H%%FVW_33
33 7Eg7N7N  r   zYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anyc                     t          | ||d          \  }}|                    d          }|                    d          } | |j        |j        fi |S )NTr   rS  r)  re  )rf   rg   rh   r   r   rS  r)  s          r    embedding_dense_backward_defaultr    sh    
 '4T  MAz nnY''G..//K4#W_CC
CCCr   c                     t          | ||d          \  }}|                    d          }|j                                        S r$  )r   r   r   detachr  s         r   values_defaultr    sN     '4T  MAz ..
!
!C ;r   c                 x    t          | ||d          \  }}|                    d          } | |j                  S r$  re  r  s         r   all_defaultr    sH    &4T  MAz ..
!
!C4r   z-self: jt_all, padding: any, output_size: any?c                 v   t          | ||d          \  }}|                    d          }|j        t          d          |d         }|||j                 }n(|j        |j        n|j                            d          }|	                                }|j        dk    r|
                    |j        dz
  d          }|j        }	|                                dk    r|                    d	          }n-|                                dk    r|                    d
          }|j        t           j        u }
|
r&|j        r|                    t           j                  }t           j        j                            ||j        g|g|d                   }|
r&|j        r|                    t           j                  }t3          |	          dk    r|                    d
|	dd                    }n(t3          |	          dk    r|                    d
          }|j        dk    r|
                    |j        d          }|S )NTr   r   z?to_padded_tensor(): not supported for nested tensors with holesr  r   r   r   r5  rF   padding)r   r   rQ   r%   rR   _max_seqlen_tensorrp  r   r   r  r  r   r!   r   r  r6  rK   boolis_cudarD  halfr   r   ro  r   r`   r:  r   )rf   rg   rh   r   r   r   r  max_seq_lenr  values_shapeis_bool
padded_outs               r   to_padded_tensor_defaultr    s>   
 '4T  MAz ..
!
!C
|M
 
 	

 ]+K!#/2 %1 OO!!!$$ 	 ZZ\\F
!!#/A"5q99<Lzz||a!,,			!!"%% lej(G '6> '5:&&??		9	 J  /:% /]]5:..
 <1))"l122.>??

	\		a		''++

))#/1==
r   zcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?c                    t          | ||d          \  }}|d         |d         }}|                    dd          }|dk    r|                    |d          }|j        }|                                dk    r|                    d	          }n-|                                dk     r|                    d
          }|j        t          j	        u }	|	r&|j
        r|                    t          j                  }t          j        j                            ||g|d                   }
|	r&|
j
        r|
                    t          j	                  }
t!          |          dk    r|
                    d
|dd                    }
n(t!          |          dk     r|
                    d
          }
|dk    r|
                    |dz
  d          }
|d         }|d         }i }|||d<   |||d<   t'          |
|||          S )NTr   paddedr   r3   r   r  r   r5  rF   r   r   r   r   )rR   r   )r   r   r  r   r!   r   r  r6  rK   r  r  rD  r  r   r   rr  r`   r:  r   r	   )rf   rg   rh   r   r   r  r   r3   padded_ragged_dim1_shaper  r  r   r   metadata_caches                 r   "_nested_from_padded_tensor_defaultr  <	  s   
 '4T  MAz !*Jy,AGFa00J A~~!!*a00%|zz||a!,,			!!"%% lej(G '6> '5:&&Y^;;	:g. F  '6> '5:&& #$$q((!!"&>qrr&BCC	%	&	&	*	*##A~~!!*q.!44L)JL)JN'1|$'1|$&	   r   zcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c                     t          | ||d          \  }}|d         |d         |d         }}}|d         }|d         }	|d         }
i }|	|	|d<   |
|
|d<   t          |||||	          S )
NTr   r   r   r   r3   r   r   )r   rR   r   )r   r	   )rf   rg   rh   r   r   r  r   r   r3   r   r   r  s               r    _nested_view_from_jagged_defaultr  u	  s    
 '4T  MAz
 	799 GF
 L)JL)JL)JN'1|$'1|$&   r   c                 f    t          | ||d          \  }}|                    d          }|j        S r$  )r   r   r   r  s         r   _nested_get_offsetsr  	  >    &4T  MAz ..
!
!C<r   c                 f    t          | ||d          \  }}|                    d          }|j        S r$  )r   r   rQ   r  s         r   _nested_get_lengthsr  	  r  r   c                 f    t          | ||d          \  }}|                    d          }|j        S r$  )r   r   rR   r  s         r   _nested_get_ragged_idxr  	  s>    &4T  MAz ..
!
!C?r   c                     t          | ||d          \  }}|                    d          }|j                            dd           S )NTr   r   r   r   r   r   r   r  s         r   _nested_get_min_seqlenr  	  O    &4T  MAz ..
!
!C""<666r   c                     t          | ||d          \  }}|                    d          }|j                            dd           S )NTr   r   r   r  r  s         r   _nested_get_max_seqlenr  	  r  r   zself: jt, mask: anyc                 <   t          | ||d          \  }}|                    d          }|                    d          }|j        dk    rt          d          |j        |j        k    rt          d|j         d|j                   |j                            |                                          }t          j	        |                                
                    d	
          d          }t          |          }||j                 |d<   t          dd|i|S )NTr   r   r  r   z3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape r   rk  r  r   r  r0   )r   r   r   r%   r   r   masked_selectr  r  r  r  r   r   r	   )	rf   rg   rh   r   r   r   r  
res_valuesmask_cumsums	            r   masked_select_defaultr  	  s(   &4T  MAz ..
!
!C>>&!!D
x!||PQQQ	dj	 	 \tz\\QTQZ\\
 
 	
 **4;;==99J%,,,33V<<K#D!#,/DO  
  r   z2grad_output: t, self: jt_all, dim: any, index: anyc                 &   t          | ||d          \  }}|                    d          }|                    d          }t          j        ||j                  }|                    |d         |d                                       |           |S )NTr   r   r)  )r6  r!   r'  )r   r   rK   
zeros_liker6  rM  rX  )rf   rg   rh   r   r   r   r)  
grad_inputs           r   _nested_select_backward_defaultr  	  s    
 '4T  MAz ..
!
!C..//K!#[->???Jj'G)<==CCKPPPr   zself: jt_all, s: anyc                     |d         }|d         } | |j         |            | |j        |           |j         | |j        |           d S d S r   )r   r   rQ   )rf   rg   rh   r   streams        r   record_stream_defaultr  	  sf    
q'C!WFDfDv
|S\6"""""  r   zRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?c                     t          | ||d          \  }}|                    d          }t          |d                   dk    r | |j        fi |S t	          d          )NTr   r   r   r   z2new_empty() not supported for NJT with shape != ())r   r   r`   r   r%   r  s         r   new_empty_defaultr  	  sy     '4T  MAz ..
!
!C
:f!##tCK..:...
K
L
LLr   zself: jt_all, ...c                     t          d |D                       }t           | d |D             i |fi t          |          S )Nc              3   D   K   | ]}t          |t                    |V  d S r   rT   r   s     r   r   z&activation_backward.<locals>.<genexpr> 
  s1      LLsjl.K.KLsLLLLLLr   c              3   R   K   | ]"}t          |t                    r|j        n|V  #d S r   )r   r	   r   r   s     r   r   z&activation_backward.<locals>.<genexpr>#
  s7      TTZ\::CckkTTTTTTr   r   )rf   rg   rh   r)  s       r   activation_backwardr  
  sq     LLdLLLLLKTTtTTT	
	
 	
 
 
%
%  r   zself: jt_all, value: anyc                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r$  r%  r  s         r   fill_Scalarr  *
  r'  r   c                 x    t          | ||d          \  }}|                    d          } | |j        fi | |S r$  re  r  s         r   fill__Scalarr  5
  sU    &4T  MAz ..
!
!CD##
###Jr   c                     t          | ||d          \  }}|                    d          }t          |          } | |j                  \  }}t	          |fi |t	          |fi |fS r$  )r   r   r   r   r	   )	rf   rg   rh   r   r   r   rb  mantissaexponents	            r   frexp_Tensorr  A
  s    &4T  MAz ..
!
!C"3''Mck**Hh22M22L5 5!5 5  r   z+grad: any, self: any, other: any, mask: anyc                    t          | ||d          \  }}|                    d          }|                    d          }|                    d          }|                    d          }|dS d }	|d         r)t          j        ||                    d	d
                    }	d }
|d         r)t          j        |                    d	d
          |          }
|	|
fS )NTr   r  r   r  r  r   r   rF   r   )r   r   rK   r-  r  )rf   rg   rh   r   r   r  r   r  grad_input_mask	grad_self
grad_others              r   matmul_backward_defaultr  P
  s    
 '4T  MAz >>&!!D
..
!
!CNN7##E nnV,,O||Iq @Lur2'>'>??	Jq ?\#--B"7"7>>
z""r   )flex_attentionflex_attention_backward)GraphModuler0   querykeyr  	score_mod
block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersc	                    |                                  dk    r0|                                 dk    r|                                 dk    sJ t          d ||z   D                       rt          d          i |ddd}t          |                                                     d          |                                                    d          |                                                    d          ||||||	  	        }	t          j                            |	d         	                    dd	          
                    d          | j        | j        | j        | j        
          	                    dd	          }
t          j                            |	d         	                    dd	          
                    d          | j        | j        | j        | j        
          	                    dd	          }t          j                            |	d	         	                    dd	          
                    d          | j        | j        | j        | j        
          	                    dd	          }|
||fS )N   c              3   X   K   | ]%}t          |t          j                  o|j        V  &d S r   )r   rK   rL   r  )r   bufs     r   r   zflex_njt.<locals>.<genexpr>
  sI         	3%%7#-     r   zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.T)
OUTPUT_MAXOUTPUT_LOGSUMEXPr   )r  r  r  r  r  r  r   r   r   r   )r!   r5   r%   flex_attention_hopr  r  rK   r  nested_tensor_from_jaggedr  r   r   rQ   r   r   )r  r  r  r  r  r  r  r  r  r#   
output_njtlogsumexp_njtmax_scores_njts                r   flex_njtr  s
  sm    99;;!		Q599;;!3C3C3CC   *-CC     
 U
 
 	
 VUdPTUUUN    ##

q!!  ##% 75
 
 
F 77q	Aq!!))!,,** 8   i1oo  L::q	Aq!!))!,,** ;   i1oo  \;;q	Aq!!))!,,** <   i1oo  ~66r   r  	logsumexpr}  grad_logsumexpfw_graphjoint_graph.c                    t          |                                                     d          |                                                    d          |                                                    d          |                                                    d          |                                                    d          |                                                    d          |                                                    d          |||	|
|||          }|\  }}}}t          j                            |                    dd                              d          | j        | j	        | j
        | j                                      dd          }t          j                            |                    dd                              d          |j        |j	        |j
        |j                                      dd          }t          j                            |                    dd                              d          |j        |j	        |j
        |j                                      dd          }||||fS )Nr   )r  r  r}  r  r  r  r  r  r  r  r  r   r   r  )flex_attention_backward_hopr  r  rK   r  r  r  r   r   rQ   r   r   )r  r  r  r  r  r}  r  r  r  r  r  r  r  r  r#   dense_q_graddense_k_graddense_v_gradscore_mod_other_buffer_grads
njt_q_grad
njt_k_grad
njt_v_grads                         r   flex_njt_backwardr  
  s>   & )  ##

q!!  ##JJLL""1%%""$$..q11"",,Q//%,,..88;;% 75  F$ NTJL,.J77q!$$,,Q//** 8   i1oo  77q!$$,,Q//(( 8   i1oo  77q!$$,,Q//** 8   i1oo  
J0LMMr   z	self: anyc                 "    ddl m}  |            S )Nr   )_nt_view_dummy)$torch.nested._internal.nested_tensorr  )rf   rg   rh   r  s       r   _nested_get_jagged_dummyr  
  s"    CCCCCC>r   r   IMPLr  CPUCUDAMeta)F)TFF)r   )r0   r0   )r   r  r   typingr   rK   torch.nn.functionalrm  rn  r  torch.fx.operator_schemasr   torch.nested._internal.sdpar   r   r	   r
   r   Any__annotations__r   Dictr   r-   r<   strrs   r{   r   r   r   r   register_jagged_funcCallabler   r   r   r   r   r   r   r   r  r  r!   r  r  r  r
  r  primr<  r  r   r  rT  r  r  r  r  r/  r"  linearr&  linear_backwardr6  rD  r6  r8  _to_copyrP  rX  r_  r  
empty_like	ones_liker  	rand_like
randn_likerc  	full_likerandint_like	low_dtypezero_rf  _softmaxrx  _log_softmaxr{  _softmax_backward_datar~  native_dropoutr  native_dropout_backwardr  r  dim_intr  r  r^   rL   r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r-  r  bmmr  r   r  r  r  r  r  wherer+  r  _pin_memoryr  	is_pinnedr  is_same_sizer  r  r   r  r^  r  r  r!  permuter*  view_unsafe_viewr3  native_layer_normrF  native_layer_norm_backwardrK  rM  rN  rP  rQ  	index_putrU  convolutionr\  r@  ra  rc  r5   r6   re  ri  r   rl  rn  ru  rr  rp  rx  r~  r  r  r  r  r  r  r  r  r  r  r  r  	embeddingr  embedding_dense_backwardr  r  _nested_get_valuesr  r  r  r  _nested_from_padded_tensorr  _nested_view_from_jaggedr  r  r  r  r  r  r  r  _nested_select_backwardr  record_streamr  	new_empty	new_zerosnew_onesr  elu_backwardhardshrink_backwardhardsigmoid_backwardhardtanh_backwardsoftplus_backwardsoftshrink_backwardr  fillScalarr  fill_r  frexpr  matmul_backwardr  &torch._higher_order_ops.flex_attentionr  r  r  r  torch.fx.graph_moduler  py_implTuplerq  r  Unionr  r  library_scoped_libraryimplr0   r   r   <module>rQ     s                              8 8 8 8 8 8 K K K K K K ' ' ' ' ' ' c   #% $sCx. % % %3 3 3 3.    :Y Y Y Y8=S =D = = = =@

#&
+7
AD
	
 
 
 
    0  . )y(8HII &HX,> & & & &R    KS KS KS\E$ E$ E$P 	3;	'	"	$	 (	!)	)1  0 0 0, ein+3^DD  ED 
Y^ ! 
 
	 
 ein2:NKKF F LKF(  	IN .0S   
 	IN$,.Q O O O( 	IN "F E E E2 ein+35UVVP P WVP 	IN"*< " "	 "J ein'-/JKKP P LKP ein-5~FF% % GF%P 	IN "Q   : D  UY^*2N C C  
 	!)	 (	!)	 (	!) 	 	,5 ,5	 	,5^ X  UY^-57V W W   U  UY^08:S T T    	IN)+N   
 ein*2NCC  DC 	IN#%Q EP EP EPP 	IN')U "P "P "PJ 	IN)1=  	  	IN!)+N 
 
 
 	IN*2+ 	 		 	 	IN8 > >	 > ein)13NOO+ + PO+ 	IN!G   " 	IN#+-T   " 	IN!#P 7 7 7  ein*24VWW&
 &
 XW&
R ein+/1JKKE E LKEP ein,02FGGK K HGK ein.68PQQE E RQE, ein(02JKK  LK0 ein+35LMMj j NMjZ ein(02KLLE E MLE  	IN!#L N N N  ein.68LMMK K NMK ein19;TUU  VU" ein6>OO  PO> 	INI   $ ein08:RSSP P TSP ein.68PQQ+ + RQ+ 	IN')F * * *gF gF gFT ein(02MNN+ + ON+ 	IN"9 = =	 = 	IN "F 'P 'P 'PT ein,46OPP%G %G QP%GP 
Y^ %)."="EF ,R ,R	 ,R^ 	IN$,N ^D ^D	 ^DB 	IN-5v 
K 
K	 
K ein+/1UVVG G WVG8 	IN= 
P 
P	 
P 	IN$>  	IN%> Q Q	 	 Qh 	IN&G 
P P 
P 	INR #L #L #LL ein)13NOO+ + PO+ ein(-/WXXA A YXA ein(,.UVV; ; WV; ein(-/WXX@ @ YX@ ein(,.UVV; ; WV; 	"	"	"	"	  + + + ein(,.UVVE E WVE ein(,.UVVE E WVE 	IN!I F F F 	IN!I F F F 	IN!#K H H H 	IN!#K H H H 	IN5=G = =	 =@ ein*24LMM  NM< 	IN$W  	  	IN+3_ D D	 D 	%	)1  	  	  	  ein(0.AA  BA 	IN#+3 8 8	 8v 	IN-5i 2 2	 2j 	IN+3i  	 8 ein8@.QQ  RQ ein8@.QQ  RQ ein;C^TT  UT ein;C^TT7 7 UT7 ein;C^TT7 7 UT7 ein2:<QRR  SR0 	IN*28  	  ein2:<RSS# # TS# 	 (	 (	'
 Y 
M 
M 
M 	#+	*2	+3	(0	(0	*2 
 
	 	
 
	 ein)02LMMP P NMP ein*13MNN  ON ein*1>BB  CB 	IN"*1 # #	 #0        . - - - - - L)) &($&@7 @7<@7	@7 <@7 	@7
 @7 @7 cN@7 #@7 "@7 5<u|34@7 @7 @7 *)@7F %$\22 &($&;N ;N<;N	;N <;N 
	;N
 |;N l;N L;N Hk)*;N ;N ;N ;N cN;N #;N ";N 	L%,eHU\4JC4O.PP;N ;N ;N 32;N~ ein=E{SS  TS ]""6622 LdII(*BEJJJII(*BFKKKII(*BFKKKL L L L L L L L L L L L L L L L L Ls   IAAJ2J2AJ6J9AJ6