`i dZddlmZmZddlmZmZmZmZm Z m Z m Z m Z ddl mZddlmZmZmZddlmZddlmZmZddlmZddlZd ddd d eeefd eeefd edefdZ d&ddddeded efdZd ddddededefdZddddedefdZ d'd eeefdedeeeffdZ!d'd eeefdedeeeffdZ"GddZ#Gd d!Z$Gd"d#Z%Gd$d%e%Z&dS)(z Shared utilities for grouped GEMM kernels. This module contains the tile scheduler classes and helper functions used by both the forward (grouped_gemm_swiglu) and backward (grouped_gemm_dswiglu) kernels. )TupleUnion)BooleanIntegerInt32minextract_mlir_valuesnew_from_mlir_values dsl_user_op const_expr)ir)scfllvmnvvm)T)Float32rNT)nanlocipabrreturnc ,|rd}nd}ttjtjt|||t|||g|dddtjjS)zCompute the minimum of two float32 values with NaN handling. :param a: First operand :param b: Second operand :param nan: If True, propagate NaN values :return: Minimum value zmin.NaN.f32 $0, $1, $2;zmin.f32 $0, $1, $2;rrz=f,f,fTFhas_side_effectsis_align_stack asm_dialect)rr inline_asmrf32ir_value AsmDialectAD_ATT)rrrrr ptx_instrs l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/cudnn/grouped_gemm/utils.pyfminr&<s *- )   EGG QZZ SR 0 0'!**2E2E#RT2E2U2U V  ! .      Frmask_and_clampabsc &t|}|||}t|||} d} |tjt j|| g| dddtjjS)alPerform a warp-level reduction synchronization for max with abs and NaN. :param value: Value to reduce :param kind: Reduction kind (unused, kept for API compatibility) :param mask_and_clamp: Warp mask and clamp value :param abs: Whether to use absolute value :param nan: Whether to handle NaN values :return: Reduced value across warp rz&redux.sync.max.abs.NaN.f32 $0, $1, $2;z=f,f,iTFr) typer!rrrrr r"r#) valuekindr)r*rrr value_typevalue_irmask_irr$s r%warp_redux_syncr2\s&eJ~~#"~--HN##,,,<= 0) using direct bitcast. :param ptr: Pointer to the memory location :param value: The float32 value to compare and potentially store (should be >= 0) :return: The old value at the memory location rresopptrrrr)rbitcastri32r!r atomicrmwcutlass_mlirdialects AtomicOpKindMAXrr )r8r-r3rr value_int old_value_ints r%atomic_max_float32rCs  QUWWennn&D&D#RTUUUIN EGG = ! & 3 7    M 4<CBGGG H HHr'c tjtjtjjjjj|| ||||}t|S)zPerform atomic add operation on a float32 value in global memory. :param ptr: Pointer to the memory location :param value: The float32 value to add :return: The old value at the memory location rr5) rr;rr r<r=r>r?FADDr!r)r8r-rr old_values r%atomic_add_float32rGs` EGG = ! & 3 8  ..SR. ( (  I 9  r'fastmathctjdtj| |zS)zCompute the sigmoid function: 1 / (1 + exp(-a)). :param a: Input value :param fastmath: Whether to use fast math approximations :return: Sigmoid of input g?rH)cutearch rcp_approxmathexprrHs r% sigmoid_f32rQs3 9  dimmQBm&J&J J K KKr'c*|t||zS)zCompute the SiLU (Swish) activation: a * sigmoid(a). :param a: Input value :param fastmath: Whether to use fast math approximations :return: SiLU of input rJ)rQrPs r%silu_f32rSs {1x000 00r'ceZdZdZdejdefdZdee j fdZ dee j ddfdZ e defd Ze dejfd Zd S) WorkTileInfozA class to represent information about a work tile. :ivar tile_idx: The index of the tile. :type tile_idx: cute.Coord :ivar is_valid_tile: Whether the tile is valid. :type is_valid_tile: Boolean tile_idx is_valid_tilec<||_t||_dSN) _tile_idxr_is_valid_tile)selfrVrWs r%__init__zWorkTileInfo.__init__s!%m44r'rc|t|j}|t|j|SrY)r rVextendrWr\valuess r%__extract_mlir_values__z$WorkTileInfo.__extract_mlir_values__s4$T]33 )$*<==>>> r'ract|dksJt|j|dd}t|j|dg}t ||S)N)lenr rZr[rU)r\ra new_tile_idxnew_is_valid_tiles r%__new_from_mlir_values__z%WorkTileInfo.__new_from_mlir_values__s\6{{a+DNF3B3KHH 01DvbzlSSL*;<< 1. True means along M, false means along N. :type raster_along_m: bool :raises ValueError: If cluster_shape_k is not 1. rxzunsupported cluster_shape_k z"expect swizzle_size >= 1, but get Nrr)striderr) ValueErrorry_cluster_shape_mnkcluster_shape_mnr|_raster_along_m_locrK make_layoutceil_divproblem_layout_ncluster_mnlround_upshapesizefast_divmod_create_divisor batch_fddcluster_shape_m_fddcluster_shape_n_fdd) r\ryrzr{r|rrproblem_shape_ncluster_mnlproblem_layout_sizecluster_count_mcluster_count_ns r%r]z&PersistentTileSchedulerParams.__init__ s6 Q 1 $ $R$"3 1"1" 5(- ,0+; M$68I"1"8MSV[] ^ ^ ^, , , ( !  )-06(6PL!$$\1aDQGO">DQGO"<=PVY^`aaaDN(,'F\_df'g'g'gD $(,'F\_df'g'g'gD $ $ $"DN'+D $'+D $ $ $r'cggc}|_|j|j|j|jfD]=}t |}||z }|jt|>g}g}td|j fd|j fd|j fgD]C\}\}}|9t |} | | ||D||z }|jt|||_ |S)Nrrr) _values_posryrrr|r appendrf enumeraterrrr__fastdivmod_indices) r\raobj obj_valuesfastdivmod_valuesfastdivmod_indicesifdd_namefdd_obj fdd_valuess r%rbz5PersistentTileSchedulerParams.__extract_mlir_values__us?#%r   (  #      5 5C -S11J j F   # #C OO 4 4 4 4&/dn-&(@A&(@A ' ' - - "A"'"099 !((444"))!,,,## $6 7 7888#5  r'c zg}t|}t|j|j|j|jg|jddD]:\}}|t||d|||d};tt|d|j i}gd}t|drt|jdkrot|jD]Z\}} || } t!|| } | ;|t|kr(t| ||g} t#|| | [|S)Nrer)rrrrr)rrzipryrrr|rrr rwtuplerhasattrrfrrgetattrsetattr) r\raobj_list values_copyrn_items new_params fdd_namesjoriginal_indexr original_fddreconstructed_fdds r%riz6PersistentTileSchedulerParams.__new_from_mlir_values__so6ll  ,'$!    SbS !  0 0LC OO0k(7(6KLL M M M%ghh/KK3U8__U49UU POO 4. / / EC8P4Q4QTU4U4U%.t/G%H%H E E!>$^4&tX66 +C 4D4D0D0D(<\KXYNK[(\(\%J2CDDDr'max_active_clustersrcHtdt|jj|jD|jjdfz}t j|||}t j|j||}||z}t||}||z} g|j| RS)aComputes the grid shape based on the maximum active clusters allowed. :param max_active_clusters: The maximum number of active clusters that can run in one wave. :type max_active_clusters: Int32 :return: A tuple containing the grid shape in (m, n, persistent_clusters). - m: self.cluster_shape_m. - n: self.cluster_shape_n. - persistent_clusters: Number of persistent clusters that can run. c3JK|]\}}tj||zVdSrY)rKr).0xys r% z?PersistentTileSchedulerParams.get_grid_shape..s3}}$!QTYq\\A-}}}}}}r'r~r)rrrrrrKrr) r\rrr num_ctas_mnlnum_ctas_in_problemnum_ctas_per_clusternum_ctas_per_wavenum_persistent_ctasnum_persistent_clusterss r%get_grid_shapez,PersistentTileSchedulerParams.get_grid_shapes}}#d>^>dfjf{:|:|}}}}}  , 21 5A  #i #"EEE#y)>CBOOO/2FF!"57HII"59M"M@&@(?@@@r')Trx)rmrnrorpr rKShapeboolintr]rbrirrrrrur'r%rwrws    $ g,  g,g,g,!%g, :g, g,  g,g,g,[g,R"""H###J@DAAA%ARWX_ahjqXqRrAAA[AAAr'rwc eZdZdZdedededejdef dZde e j fd Z d e e j ddfd Z eed d d dedeeeefdeeeeffdZed d d dededeeeeffdZd d d dedefdZd d d dedeeeeffdZed d d defdZed d d defdZedd d ddefdZedefdZd S)StaticPersistentTileScheduleraA scheduler for static persistent tile execution in CUTLASS/CuTe kernels. :ivar params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl :type params: PersistentTileSchedulerParams :ivar num_persistent_clusters: Number of persistent clusters that can be launched :type num_persistent_clusters: Int32 :ivar cta_id_in_cluster: ID of the CTA within its cluster :type cta_id_in_cluster: cute.Coord :ivar _num_tiles_executed: Counter for executed tiles :type _num_tiles_executed: Int32 :ivar _current_work_linear_idx: Current cluster index :type _current_work_linear_idx: Int32 paramsrcurrent_work_linear_idxcta_id_in_clusternum_tiles_executedcL||_||_||_||_||_dS)aInitializes the StaticPersistentTileScheduler with the given parameters. :param params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl. :type params: PersistentTileSchedulerParams :param num_persistent_clusters: Number of persistent clusters that can be launched. :type num_persistent_clusters: Int32 :param current_work_linear_idx: Current cluster index. :type current_work_linear_idx: Int32 :param cta_id_in_cluster: ID of the CTA within its cluster. :type cta_id_in_cluster: cute.Coord :param num_tiles_executed: Counter for executed tiles. :type num_tiles_executed: Int32 N)rr_current_work_linear_idxr_num_tiles_executed)r\rrrrrs r%r]z&StaticPersistentTileScheduler.__init__s1* '>$(?%!2#5   r'rcft|j}|t|j|t|j|t|j|t|j|SrY)r rr_rrrrr`s r%rbz5StaticPersistentTileScheduler.__extract_mlir_values__s$T%ABB )$*GHHIII )$*@AABBB )$*BCCDDD  )$+66777 r'racrt|dksJt|j|dg}t|j|dg}t|j|dd}t|j|dg}|dd}t|j|}t|||||SNrrxr~)rfr rrrrrrr\ranew_num_persistent_clustersnew_current_work_linear_idxnew_cta_id_in_clusternew_num_tiles_executed params_valuesrs r%riz6StaticPersistentTileScheduler.__new_from_mlir_values__s6{{a&:4;WZ`abZcYd&e&e#&:4;X[abc[dZe&f&f# 4T5KVTUVWTW[ Y Y!5d6NQWXYQZP[!\!\qrr )$+}EE ,  ' ' ! "    r'Nr block_idxgrid_dimchtj|||tj|j||z}|\}}}t|} t||jdzt||jdztdf} td} t ||| | | S)a Initialize the static persistent tile scheduler. :param params: Parameters for the persistent tile scheduler. :type params: PersistentTileSchedulerParams :param block_idx: The 3d block index in the format (bidx, bidy, bidz). :type block_idx: Tuple[Integer, Integer, Integer] :param grid_dim: The 3d grid dimensions for kernel launch. :type grid_dim: Tuple[Integer, Integer, Integer] :return: A StaticPersistentTileScheduler object. :rtype: StaticPersistentTileScheduler rrrx)rKrrrr) rrrrrrbidxbidybidzrrrs r%createz$StaticPersistentTileScheduler.create%s2#')H#""E"E"ESYSjpsxzI{I{I{"{$dD#(++ $033 4 4 $033 4 4 !HH  #1XX,  # #      r'rc2||||S)aCalculates the grid shape to be launched on GPU using problem shape, threadblock shape, and active cluster size. :param params: Parameters for grid shape calculation. :type params: PersistentTileSchedulerParams :param max_active_clusters: Maximum active clusters allowed. :type max_active_clusters: Int32 :return: The calculated 3d grid shape. :rtype: Tuple[Integer, Integer, Integer] r)r)rrrrs r%rz,StaticPersistentTileScheduler.get_grid_shapeVs ($$%8cb$IIIr'c |tj|jj||k}|jjdkr||||}n"|jj|||}tdt||j g|jj tdRD}t||S)hCompute current tile coord given current_work_linear_idx and cta_id_in_cluster. :param current_work_linear_idx: The linear index of the current work. :type current_work_linear_idx: Int32 :return: An object containing information about the current tile coordinates and validity status. :rtype: WorkTileInfo rrxc3|K|]7\}}}t|t|zt|zV8dSrY)r)rrrzs r%rzQStaticPersistentTileScheduler._get_current_work_for_linear_idx..sX  1a !HHuQxx %(( *      r') rKrrrr|%_get_cluster_work_idx_with_fastdivmodget_flat_coordrrrrrrU)r\rrris_validcur_cluster_coordcur_tile_coords r% _get_current_work_for_linear_idxz>StaticPersistentTileScheduler._get_current_work_for_linear_idxms+TYt{7^dgln-o-o-oo ; #q ( ( $ J JKbhkpr J s s  !% G V VWntw|~ V     !&9$+.9a99     NH555r'ct||jj\}}t||jj\}}t||jj\}} || |fS)aFastDivmod optimized CLUSTER coordinate calculation. CRITICAL: This should mimic problem_layout_ncluster_mnl.get_hier_coord() which returns CLUSTER coordinates, not tile coordinates! :param current_work_linear_idx: Linear index in the work space :type current_work_linear_idx: Int32 :return: Cluster coordinates (m, n, l) or None if FastDivmod not available :rtype: Tuple[Int32, Int32, Int32] or None )divmodrrrr) r\rrrwork_iteration work_unit_idcluster_n_batch cluster_mbatch_l cluster_ns r%rzCStaticPersistentTileScheduler._get_cluster_work_idx_with_fastdivmodsa(..Et{G\']']$ &,L$+:Y%Z%Z"$OT[5TUU9g..r'c<||j||SNr)rrr\rrs r%get_current_workz.StaticPersistentTileScheduler.get_current_works"44T5RX[`b4cccr'c0|||Sr)rrs r%initial_work_tile_infoz4StaticPersistentTileScheduler.initial_work_tile_infos$$$444r'rx) advance_countrrrc|xjt|t|jzz c_|xjtdz c_dS)Nrx)rrrr)r\rrrs r%advance_to_next_workz2StaticPersistentTileScheduler.advance_to_next_worksN %%})=)=dFb@c@c)cc%%   E!HH,    r'c|jSrY)rrks r%rz0StaticPersistentTileScheduler.num_tiles_executeds ''r')rmrnrorprwrrKrqr]rrr rsrbri staticmethodr rrrrrUrrrrrrrtrrur'r%rrs  6-6"'6"' 6  : 6 " 66666 bh     tBH~ Ba    &   , , , -, '723, '12, , , [\, ^   JJJ-J"J w( ) JJJ\J,W[_c6666ht6666B\`dh///U/mrsxzBGtGnH////6&*tddd ddd[d,0T555l555[5;<$4---S---[-(E(((X(((r'rc eZdZdZ ddedededejdedef fd Z d e e j d dfd Z ee dd d ddedeeeefdeeeefdefdZd d dded efdZxZS)$StaticPersistentRuntimeTileScheduleraA scheduler for static persistent runtime tile execution in CUTLASS/CuTe kernels. This scheduler will always launch all the SMs and the scheduler will generate the real tile info for each SM. :ivar params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl :type params: PersistentTileSchedulerParams :ivar num_persistent_clusters: Number of persistent clusters that can be launched :type num_persistent_clusters: Int32 :ivar cta_id_in_cluster: ID of the CTA within its cluster :type cta_id_in_cluster: cute.Coord :ivar _num_tiles_executed: Counter for executed tiles :type _num_tiles_executed: Int32 :ivar _current_work_linear_idx: Current cluster index :type _current_work_linear_idx: Int32 rxrrrrr inner_modect||||||dvrtd|||_dS)aFInitializes the StaticPersistentRuntimeTileScheduler with the given parameters. :param params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl. :type params: PersistentTileSchedulerParams :param num_persistent_clusters: Number of persistent clusters that can be launched. :type num_persistent_clusters: Int32 :param current_work_linear_idx: Current cluster index. :type current_work_linear_idx: Int32 :param cta_id_in_cluster: ID of the CTA within its cluster. :type cta_id_in_cluster: cute.Coord :param num_tiles_executed: Counter for executed tiles. :type num_tiles_executed: Int32 :param inner_mode: The inner mode along which the linear index will be decomposed first. :type inner_mode: int )rrxz;inner_mode must be 0(for M mode) or 1(for N mode), but got N)superr]rr)r\rrrrrr __class__s r%r]z-StaticPersistentRuntimeTileScheduler.__init__s`0   # #       V # #g[egghh h$r'rarc~t|dksJt|j|dg}t|j|dg}t|j|dd}t|j|dg}|dd}t|j|}t||||||jSr) rfr rrrrrrrrs r%riz=StaticPersistentRuntimeTileScheduler.__new_from_mlir_values__s6{{a&:4;WZ`abZcYd&e&e#&:4;X[abc[dZe&f&f# 4T5KVTUVWTW[ Y Y!5d6NQWXYQZP[!\!\qrr )$+}EE 3  ' ' ! " O    r'Nrrrcjtj|||tj|j||z}|\}}} t| } t||jdzt||jdztdf} td} t ||| | | |S)aInitialize the static persistent tile scheduler. :param params: Parameters for the persistent tile scheduler. :type params: PersistentTileSchedulerParams :param block_idx: The 3d block index in the format (bidx, bidy, bidz). :type block_idx: Tuple[Integer, Integer, Integer] :param grid_dim: The 3d grid dimensions for kernel launch. :type grid_dim: Tuple[Integer, Integer, Integer] :param inner_mode: The inner mode along which the linear index will be decomposed first. :type inner_mode: int :return: A StaticPersistentRuntimeTileScheduler object. :rtype: StaticPersistentRuntimeTileScheduler rrrx)rKrrrr) rrrrrrrrrrrrrs r%rz+StaticPersistentRuntimeTileScheduler.creates8#')H#""E"E"ESYSjpsxzI{I{I{"{$dD#(++ $033 4 4 $033 4 4 !HH  #1XX3  # #       r'c||jjj}d}t|jdkr't j||df|ddf}n&t j|d|fd|df}||}|d|dtdf}d} t|| S)rirx)rrT) rrrr rrKrget_hier_coordrrU) r\rrr ntile_shapeint_max ntile_layoutcluster_tile_coord_mnrrs r%rzEStaticPersistentRuntimeTileScheduler._get_current_work_for_linear_idx9sk=C  do* + + c+Wk!n,E{[\~_`NabbbLL+[^W,EqR]^_R`NabbbL , ; ;rs:                     2222222222!!!!!!;;;;;;;;   UG^ UG^   D $ "  """"  "  """"R  III I I IIIID     2LL5(LDLU5RY>EZLLLL11eWn%11%wBW1111********Z\A\A\A\A\A\A\A\A~[([([([([([([([(|X6X6X6X6X6+HX6X6X6X6X6r'