,`i!UddlZddlmZddlZddlmZddlmZddlm Z ddl m Z m Z ddl mZmZe jZdefdZeZdefd Zd edefd Zer dd lmZejZ dvdejdejdejdejdejdejdzdedededejdzdejdzdejdzdejdzdejdzdejdzdejf dZ dvdejdejdejdejdejdejdzdedededejdzdejdzdejdzdejdzdejdzdejdzdejf dZ dwdejdejdejdejdejd ejdzd!ejdzd"ejdzd#ejdzd$ed%ejdzdejdzdedejfd&Z dwdejdejdejdejdejd ejdzd!ejdzd"ejdzd#ejdzd$ed%ejdzdejdzdedejfd'Z!dejd(ejd)ejd*ejd+eddf d,Z"dejd(ejd)ejd*ejd+eddf d-Z#dejd(ejd*ejddfd.Z$dejd(ejd*ejddfd/Z% dxd*ejd1ejdejdejd2ed3ed4ed5e&ddfd6Z' dxd*ejd1ejdejdejd2ed3ed4ed5e&ddfd7Z( dyd*ejdejdejd2ed3ed4ed9e)d5e&ddfd:Z* dyd*ejdejdejd2ed3ed4ed9e)d5e&ddfd;Z+da,edze-d<<defd=Z. dzd?ejd@ejdAejdBejdCedDejdzdEejdzdFejdzdGe&dHe&dIejdzdJejdzddfdKZ/ dzd?ejd@ejdAejdBejdCedDejdzdEejdzdFejdzdGe&dHe&dIejdzdJejdzddfdLZ0dej1fdMejdNejdOejdPejdQejdzdejdejfdRZ2dej1fdMejdNejdOejdPejdQejdzdejdejfdSZ3ej1fdMejdNejdOejdPejdejdejf dTZ4ej1fdMejdNejdOejdPejdejdejf dUZ5ej1fdMejdNejdOejdPejdejdejf dVZ6ej1fdMejdNejdOejdPejdejdejf dWZ7dXejdYejdZe&dejfd[Z8dXejdYejdZe&dejfd\Z9dXejd]ejdYejdZe&de:ejejff d^Z;dXejd]ejdYejdZe&de:ejejff d_ZdXejdYejd`e&daejde:ejejff ddZ?dXejdYejd`e&daejde:ejejff deZ@ d{dXejdaejdfejdzde:ejejffdgZA d{dXejdaejdfejdzde:ejejffdhZB d{dXejdaejdfejdzde:ejejffdiZC d{dXejdaejdfejdzde:ejejffdjZDdXejd]ejdYejdZe&dkede:ejejejff dlZEdXejd]ejdYejdZe&dkede:ejejejff dmZFdXejdYejdZe&dkede:ejejff dnZGdXejdYejdZe&dkede:ejejff doZHdXejdkede:ejejffdpZIdXejdkede:ejejffdqZJdXejdkede:ejejffdrZKdXejdkede:ejejffdsZLd aMGdtduZNeNOdS)|N)Callable) OpOverload)current_platform)direct_register_custom_opis_torch_equal_or_newer)rocm_aiter_sparse_attn_indexer#rocm_aiter_sparse_attn_indexer_fakereturnc(ddlm}|dduS)Nr find_specaiter)importlib.utilr r s c/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/_aiter_ops.pyis_aiter_foundrs)(((((( 9W  T ))cZtjrtrddlm}|SdS)Nron_gfx9F)ris_rocmIS_AITER_FOUNDvllm.platforms.rocmrrs ris_aiter_found_and_supportedr!s=!!n//////wyy 5rfunccFtjfd}|S)zdDecorator that only executes the function if ROCm AITER package is supported on gfx9 archs. c4tr|i|SdSN)r)argskwargsrs rwrapperz#if_aiter_supported..wrapper.s- ( ) ) )4((( (tr) functoolswraps)rr s` rif_aiter_supportedr#)s: _T Nr)dtypesF hidden_statesw1w2 topk_weighttopk_ids expert_maskactivation_method quant_methoddoweight_stage1w1_scalew2_scalea1_scalea2_scalenum_local_tokens output_dtypec~ddlm}m}ddlm}||}||}||||||||||| | | | | |S)Nr)ActivationType QuantType) fused_moe)r2dtype)rr5r6aiter.fused_moer7)r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r5r6r7 activation quant_types r_rocm_aiter_fused_moe_implr<Ds"0///////)))))) 122J<((J 9  )   rcZ|tj||Stj|S)Nr8torch empty_liker%r&r'r(r)r*r+r,r-r.r/r0r1r2r3s r_rocm_aiter_fused_moe_fakerCns1" \BBBB  M * **r topk_weights fc1_scale fc2_scalefc1_smooth_scalefc2_smooth_scalea16per_tensor_quant_scalec `ddlm} ddlm}| | }||||||||||| | | | S)Nr)r5) asm_moe_tkw1)rErFrGrHrIrJr*r:)rr5aiter.fused_moe_bf16_asmrL)r%r&r'rDr)rErFrGrHrIrJr*r+r5rLr:s r_rocm_aiter_asm_moe_tkw1_implrNsx%$$$$$555555 122J <  )) 5   rc *tj|Srr? r%r&r'rDr)rErFrGrHrIrJr*r+s r_rocm_aiter_asm_moe_tkw1_fakerQs  M * **r topk_indicestoken_expert_indices gating_output renormalizec0ddlm}||||||dS)Nr) topk_softmax)rrW)rDrRrSrTrUrWs r_rocm_aiter_topk_softmax_implrXs@#"""""Ll$8-rcdSrrDrRrSrTrUs r_rocm_aiter_topk_softmax_faker\s  Drc,ddlm}||||dS)Nr) topk_sigmoid)rr^)rDrRrTr^s r_rocm_aiter_topk_sigmoid_implr_s0 #"""""L|];;;;;rcdSrrZ)rDrRrTs r_rocm_aiter_topk_sigmoid_fakeras  Dr?correction_biasnum_expert_group topk_group need_renormrouted_scaling_factorc 6ddlm}|||||||||dS)Nr)biased_grouped_topk)rri) rTrcrDr)rdrerfrgris r$_rocm_aiter_biased_grouped_topk_implrjsM*)))))     rcdSrrZrTrcrDr)rdrerfrgs r$_rocm_aiter_biased_grouped_topk_fakerm  Drsoftmax scoring_funcc B|dk}ddlm} | ||||||||dS)Nror) grouped_topk)rrr) rTrDr)rdrerfrprg is_softmaxrrs r_rocm_aiter_grouped_topk_implrt sV*J""""""L     rcdSrrZrTrDr)rdrerfrprgs r_rocm_aiter_grouped_topk_fakerw#rnr_AITER_MLA_SUPPORTS_FP8ct^ ddl}ddlm}||}d|jvod|jvan+#t ttttf$rdaYnwxYwtS)zKCheck if aiter.mla.mla_decode_fwd supports q_scale and kv_scale parameters.Nrmla_decode_fwdq_scalekv_scaleF) rxinspect aiter.mlar{ signature parameters ImportErrorModuleNotFoundErrorAttributeError ValueError TypeError)r~r{sigs r_check_aiter_mla_fp8_supportr4s& , NNN 0 0 0 0 0 0##N33CS^+L cn0L $ #       , , ,', # # # , #"s1;%A#"A#q kv_buffero qo_indptr max_seqlen_qo kv_indptr kv_indiceskv_last_page_lenssm_scale logit_capr|r}c ddlm} || d} tr | | d<| | d<| ||ddd|jd||||||fi| dS)Nrrz)rrr|r})rr{rviewshape)rrrrrrrrrrr|r}r{rs r_rocm_aiter_mla_decode_fwd_implrPs)(((((F $%%&#y%zN r1a--         rc dSrrZ) rrrrrrrrrrr|r}s r_rocm_aiter_mla_decode_fwd_fakerws  DrABAsBsbiasc.ddlm}|||||||S)Nr) gemm_a8w8_CK)rr)rrrrrr3rs r_rocm_aiter_gemm_a8w8_implrs2#""""" <1b"dL 9 99rct|jd}|jd}tj||||j}|SNrr8devicerr@emptyr) rrrrrr3mnYs r_rocm_aiter_gemm_a8w8_fakers9  A  A Aq QX>>>A Hrc.ddlm}||||||SNr)gemm_a8w8_blockscaler>)%aiter.ops.triton.gemm_a8w8_blockscalerrrrrr3rs r,_rocm_aiter_triton_gemm_a8w8_blockscale_implrs3KJJJJJ  1b"L A A AArct|jd}|jd}tj||||j}|Srrrrrrr3rrrs r,_rocm_aiter_triton_gemm_a8w8_blockscale_faker9  A  A Aq QX>>>A Hrc.ddlm}||||||Sr)rrrs r%_rocm_aiter_gemm_a8w8_blockscale_implrs2+*****  1b"L A A AArct|jd}|jd}tj||||j}|Srrrs r%_rocm_aiter_gemm_a8w8_blockscale_fakerrrxweightvariance_epsiloncddlm}|dkrE|j}|d|d}||||}||S||||S)Nr)rms_normr)rrdimrreshape)rrrrx_original_shapes r_rocm_aiter_rms_norm_implrsuuww{{7 IIb*2. / / HQ 0 1 1yy)*** 8Av/ 0 00rc*tj|Srr?rrrs r_rocm_aiter_rms_norm_fakers  A  rresidualcddlm}tj|}tj|}|||||||||fS)Nr)rmsnorm2d_fwd_with_add)rrr@rA)rrrrr residual_outouts r'_rocm_aiter_rmsnorm2d_fwd_with_add_implrsj -,,,,,#H--L  1  C     rcZtj|}tj|}||fSrr?)rrrrrrs r'_rocm_aiter_rmsnorm2d_fwd_with_add_fakers/ #H--L  1  C  repsilon quant_dtypec Jddl}|tjtfvsJtj|jddtj|j}tj|j||j}tj|}| |||||||d|||fSNrrr)use_model_sensitive_rmsnorm) rr@int8 _FP8_DTYPErrfloat32rrA#rmsnorm2d_fwd_with_add_dynamicquant) rrrrr rocm_aitery_scalerrs r0_rocm_aiter_rmsnorm_fused_add_dynamic_quant_implr s 5:z2 2 2 2 2k!'!*au}QXNNNG +ag[ B B BC#A&&L22  $%3     g %%rctj|jddtj|j}tj|j||j}tj|}|||fSNrrr)r@rrrrrA)rrrrrrrrs r0_rocm_aiter_rmsnorm_fused_add_dynamic_quant_faker*s]k!'!*au}QXNNNG +ag[ B B BC#A&&L  g %%rcddl}|tjtfvsJtj|jddtj|j}tj|j||j}||||||d||fSr) rr@rrrrrrrmsnorm2d_fwd_with_dynamicquant)rrrrrrrs r,_rocm_aiter_rmsnorm_fused_dynamic_quant_implr8s  5:z2 2 2 2 2k!'!*au}QXNNNG +ag[ B B BC.. Qa/ <rctj|jddtj|j}tj|j||j}||fSr)r@rrrr)rrrrrrs r,_rocm_aiter_rmsnorm_fused_dynamic_quant_fakerLsJ k!'!*au}QXNNNG +ag[ B B BC <rscalec(ddlm}||||S)Nr)per_tensor_quant_hip)aiter.ops.quantr)rrrrs r!_rocm_aiter_per_tensor_quant_implrXs+ 544444  5+ 6 66rcztj||tjdtj|jfS)Nr>rr)r@rArrrrrrs r!_rocm_aiter_per_tensor_quant_fakerbs?  A[ 1 1 15; qx444 rc ,ddlm}|tjtfvsJ|j}tj|jt|j}|3tjg|dddRtj|j}||||dddd||fS)Nr)dynamic_per_token_scaled_quantrrrF)scale_ub shuffle_scalenum_rowsnum_rows_factor) rrr@rrrrrr)rrrr out_shapers r _rocm_aiter_per_token_quant_implrls?>>>>> 5:z2 2 2 2 2I +agZ A A AC } 0in0a00 ahWWW""    :rc|j}tj|jt|jtjg|dddRtj|jfS)Nrrr)rr@rrrr)rrrrs r _rocm_aiter_per_token_quant_fakers[I AG:ah??? (in(a(( ahOOO r group_sizec Zddlm}||||ddd|t| \\}}}}} || |fSNr)fused_rms_fp8_group_quant)r dtype_quantres1 aiter.ops.triton.fused_fp8_quantrAITER_FP8_DTYPE) rrrrrrx_quantx_quant_scales_ress r1_rocm_aiter_rmsnorm_with_add_fp8_group_quant_implrslKJJJJJ+D+D    #  , , ,(Wnq!S   rc|j\}}|||zdz |zf}tj|t|jtj||jtj|tj|jfS)Nrr)rrr@rArrrr)rrrrrMN scale_shapes r1_rocm_aiter_rmsnorm_with_add_fp8_group_quant_faker su 7DAqq:~)j89K /!(CCC (/::: Ku}QXFFF rc Xddlm}||||ddd|td \\}}}}}||fSrr) rrrrrrrrrs r(_rocm_aiter_rmsnorm_fp8_group_quant_implr sd KJJJJJ+D+D    #  , , ,(Wnq!S ^ $$rc|j\}}|||zdz |zf}tj|t|jtj|tj|jfS)Nrrr)rrrrrrr s r(_rocm_aiter_rmsnorm_fp8_group_quant_fakers` 7DAqq:~)j89K /!(CCC Ku}QXFFF rc|jd|zdks Jdddlm}m}||j}||t S)Nrrz+Input shape must be divisible by group size)r6 get_hip_quant)r)rrr6r per_1x128 contiguousr)rrr6raiter_per1x128_quants r _rocm_aiter_group_fp8_quant_implrsr 72; #q ( ( (*W ( ( (........(=)<==   O L L LLrc|j\}}tj||ft|j}tj|||zdz |zftj|j}||fS)Nrrrr@rrrr)rrrrx_fp8out_bss r _rocm_aiter_group_fp8_quant_fakersr 7DAq KAoah G G GE [ ^a J . mx F &=rc6ddlm}||d|tS)Nr)act_mul_and_fp8_group_quantsilu)r:rr)aiter.ops.triton.activationrr)rrrs r,_rocm_aiter_act_mul_and_fp8_group_quant_implrs=HGGGGG & & #    rc|j\}}|dzdksJ|dz}tj||ft|j}tj|||zdz |zftj|j}||fS)Nrrrrr)rrrrN_halfrrs r,_rocm_aiter_act_mul_and_fp8_group_quant_faker!s 7DAq q5A:::: !VF KF ?18 L L LE [ j 1 $ 3 mx F &=rc"eZdZdZejZejZej Z ej Z ej ZejZejZejZejZejZejZejZejZejZ e!dZ"e!e#de$fdZ%e!e#de$fdZ&e!e#de$fdZ'e!e#de$fdZ(e!e#de$fdZ)e!e#de$fd Z*e!e#de$fd Z+e!e#de$fd Z,e!e#de$fd Z-e!e#de$fd Z.e!e#de$fdZ/e!e#de$fdZ0e!e#de$fdZ1e!e#de$fdZ2e!e#de$fdZ3e4e#ddZ5e4de6fdZ7e4de6fdZ8e4de6fdZ9e4de6fdZ:e4de6fdZ;e4de6fdZe4de6fdZ?e4de@jAde@jAd eBde@jAfd!ZCe4de@jAd"e@jAde@jAd eBdeDe@jAe@jAff d#ZEe4de@jFfd$e@jAd%e@jAd&e@jAd'e@jAd(e@jAdzd)e@jGde@jAfd*ZHe4e@jFfd$e@jAd%e@jAd&e@jAd'e@jAd+eIeJd)e@jGde@jAfd,ZKe4e@jFfd$e@jAd%e@jAd&e@jAd'e@jAd+eIeJd)e@jGde@jAfd-ZLe4 dd0e@jAd1e@jAd2e@jAd3e@jAd4e@jAd5e@jAdzd6eJd7eJd8e$d9e@jAdzd:e@jAdzd;e@jAdzdZMe4 dd0e@jAd1e@jAd2e@jAd?e@jAd4e@jAd@e@jAdzdAe@jAdzdBe@jAdzdCe@jAdzdDe$dEe@jAdzd5e@jAdzd6eJde@jAfdFZNe4d?e@jAdGe@jAdHe@jAdIe@jAdJe$deDe@jAdKff dLZOe4d?e@jAdGe@jAdHe@jAdIe@jAdJe$deDe@jAdKff dMZPe4 ddIe@jAdOe@jAd?e@jAd4e@jAdPeJdQeJdRe$dSeBddfdTZQe4 ddIe@jAd?e@jAd4e@jAdPeJdQeJdRe$dVeRdSeBddfdWZSe4 ddYe@jAdZe@jAd[e@jAd\eBd]e@jAd^eJd_e@jAdzd`e@jAdzdae@jAdzdbeBdce@jAdzdde@jAdzfdeZTe4 dde@jAdfe@jGdge@jAdzdeDe@jAe@jAffdhZUe4 dde@jAdfe@jGdge@jAdzdeDe@jAe@jAffdiZVe4e@jWdfde@jAde@jAdje@jAdke@jGdzdle@jAdzde@jAf dmZXe4dne@jAdoe@jAdpe@jAdqe@jAdreJdseJdte$fduZYe4 ddve@jAdwe@jAdxe@jAdye@jAdze$dzd{e$dzd|e@jAdzde@jAfd}ZZe4d~de@jWddd/dfdve@jAde@jAdxe@jAdeJd(e@jAdzde@jGdzdeJdzde@jAdzdze$dzde[dzde@jAfdZ\e4 dde@jAdeJdeDe@jAdKffdZ]e4deJdeJde$fdZ^e4deJdeJde$fdZ_e4 dde@jAdeDeJeJfde@jAfdZ`e4ddde@jAdeDeJeJfdeDe@jAdKffdZadS)rocm_aiter_opsa ROCm AITER operations wrapper for AMD GPU acceleration in vLLM. This class centralizes the import and registration of AITER ops, and provides a unified interface for checking if AITER is enabled. Operations are only available on supported gfx9 architectures when aiter is installed. The class uses environment variables to control which features are enabled, allowing fine-grained control over which AITER optimizations are used. Environment Variables: VLLM_ROCM_USE_AITER: Main toggle for all AITER operations. VLLM_ROCM_USE_AITER_LINEAR: Controls GEMM and quantization ops. VLLM_ROCM_USE_AITER_RMSNORM: Controls RMSNorm operations. VLLM_ROCM_USE_AITER_MOE: Controls MoE (Mixture of Experts) ops. VLLM_ROCM_USE_AITER_MLA: Controls MLA (Multi-head Latent Attention) ops. VLLM_ROCM_USE_AITER_MHA: Controls MHA ops including flash_attn_varlen. VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: Controls Triton unified attention. VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply. VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM. Note: The environment variables are assigned when the module is imported, so you can't change the environment variables after the module is imported. This is done out of performance consideration. Accessing environment variables is expensive as described in issue https://github.com/vllm-project/vllm/issues/17067 so we don't want to do it repeatedly, especially in the hot path (the forward pass). You can call the refresh_env_variables() function to reload the env variables after monkey patching the env variables in the unit test. Check Functions: All check functions (is_*_enabled) are decorated with @if_aiter_supported, which verifies: (1) platform is ROCm, (2) device arch is gfx9, and (3) aiter library is installed. The check function then also verifies the corresponding environment variable is enabled. i.e. ___ is_enabled() == current_platform.is_rocm() and | checked by current_platform.is_on_gfx9() and | @if_aiter_supported IS_AITER_FOUND and _______________| cls._AITER_ENABLED -----> Check by the logic in `is_enabled()` Example: from vllm._aiter_ops import rocm_aiter_ops # Check if aiter is enabled before using operations if rocm_aiter_ops.is_enabled(): result = rocm_aiter_ops.rms_norm(x, weight, epsilon) Operations: - RMS normalization: rms_norm, rms_norm2d_with_add - GEMM operations: gemm_a8w8, gemm_a8w8_blockscale - Fused MoE: fused_moe, asm_moe_tkw1 - Routing: topk_softmax, biased_grouped_topk, grouped_topk - MLA decode: mla_decode_fwd - Quantization: per_tensor_quant, per_token_quant, group_fp8_quant - Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale ctj|_tj|_tj|_tj|_tj |_ tj |_ tj |_tj|_tj|_tj|_tj|_tj|_tj|_tj|_dS)aT Since the environment variables are assigned when the module is imported, This is a helper function to reload all the env variables from the environment variables. for example, after monkey patching the env variables in the unit test, you can call this function to reload the env variables. N)envsVLLM_ROCM_USE_AITER_AITER_ENABLEDVLLM_ROCM_USE_AITER_LINEAR_LINEAR_ENABLEDVLLM_ROCM_USE_AITER_RMSNORM_RMSNORM_ENABLEDVLLM_ROCM_USE_AITER_MOE _FMOE_ENABLEDVLLM_ROCM_USE_AITER_MLA _MLA_ENABLEDVLLM_ROCM_USE_AITER_MHA _MHA_ENABLED!VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT_SHUFFLE_KV_CACHE_ENABLED%VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION_TRITON_UNIFIED_ATTN_ENABLEDVLLM_ROCM_USE_AITER_FP8BMM_FP8BMM_ENABLEDVLLM_ROCM_USE_AITER_FP4BMM_FP4BMM_ENABLED VLLM_ROCM_USE_AITER_FP4_ASM_GEMM_FP4_GEMM_DYNAMIC_QUANT_ASMVLLM_ROCM_USE_AITER_TRITON_ROPE_TRITON_ROTARY_EMBED)VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS_MOE_SHARED_EXPERTS_ENABLEDVLLM_ROCM_USE_AITER_TRITON_GEMM_TRITON_UNQUANT_GEMMclss rrefresh_env_variablesz$rocm_aiter_ops.refresh_env_variablesos"5"=#? 877(,(N%+/+U("="=*.*O'#'#G *.*X'#'#G   rr c|jSr)r'rBs r is_enabledzrocm_aiter_ops.is_enableds !!rc|jo|jSr)r'r)rBs ris_linear_enabledz rocm_aiter_ops.is_linear_enabled!9c&99rc*|Sr)rHrBs ris_linear_fp8_enabledz$rocm_aiter_ops.is_linear_fp8_enableds$$&&&rc|jo|jSr)r'r+rBs ris_rmsnorm_enabledz!rocm_aiter_ops.is_rmsnorm_enableds!:c&::rc|jo|jSr)r'r-rBs ris_fused_moe_enabledz#rocm_aiter_ops.is_fused_moe_enableds!7c&77rc8|o|jSr)rOr?rBs r$is_fusion_moe_shared_experts_enabledz3rocm_aiter_ops.is_fusion_moe_shared_experts_enableds''))Mc.MMrc|jo|jSr)r'r/rBs ris_mla_enabledzrocm_aiter_ops.is_mla_enabled!6c&66rc|jo|jSr)r'r1rBs ris_mha_enabledzrocm_aiter_ops.is_mha_enabledrTrc|jo|jSr)r'r3rBs ris_shuffle_kv_cache_enabledz*rocm_aiter_ops.is_shuffle_kv_cache_enableds!Cc&CCrc|jo|jSr)r'r5rBs ris_triton_unified_attn_enabledz-rocm_aiter_ops.is_triton_unified_attn_enableds!Fc&FFrc|jo|jSr)r'r7rBs ris_fp8bmm_enabledz rocm_aiter_ops.is_fp8bmm_enabledrIrc|jo|jSr)r'r9rBs ris_fp4bmm_enabledz rocm_aiter_ops.is_fp4bmm_enabledrIrc|jo|jSr)r'r;rBs r%is_asm_fp4_gemm_dynamic_quant_enabledz4rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enableds!Ec&EErc|jo|jSr)r'r=rBs ris_triton_rotary_embed_enabledz-rocm_aiter_ops.is_triton_rotary_embed_enabled!>c&>>rc|jo|jSr)r'rArBs ris_triton_gemm_enabledz%rocm_aiter_ops.is_triton_gemm_enabledrcrNc\ts#tdrtntjjf}t dtgttj t dtgttj t dtgdttj t dtdd gt tj t d t"dd gt$tj t d t&dd gt(tj t d t*dgt,|t dt.gt0tj t dt2t4t dt6t8t dt:t<t dt>t@tj t dtBtDtj t dtFtHtj t dtJtLt dtNtPt dtRtTt dtVtXt dtZgt\tj t dt^t`tj t dtbd gtdtj d!adSdS)"Nz2.7.0rocm_aiter_asm_moe_tkw1)op_nameop_func mutates_args fake_impl dispatch_keyrocm_aiter_fused_moerocm_aiter_topk_softmax)rDrRrSrocm_aiter_topk_sigmoidrDrRrocm_aiter_biased_grouped_topkr)rocm_aiter_grouped_topkrocm_aiter_mla_decode_fwdr)rhrirjrktagsrocm_aiter_gemm_a8w8&rocm_aiter_triton_gemm_a8w8_blockscale)rhrirkrocm_aiter_gemm_a8w8_blockscalerocm_aiter_rms_norm!rocm_aiter_rmsnorm2d_fwd_with_add)rhrirkrl&rocm_aiter_rmsnorm_fused_dynamic_quant*rocm_aiter_rmsnorm_fused_add_dynamic_quant"rocm_aiter_rmsnorm_fp8_group_quant+rocm_aiter_rmsnorm_with_add_fp8_group_quant&rocm_aiter_act_mul_and_fp8_group_quantrocm_aiter_group_fp8_quantrocm_aiter_per_tensor_quantrocm_aiter_per_token_quantrtopk_indices_bufferT)3_OPS_REGISTEREDrtupler@Tagneeds_fixed_stride_orderrrNrQrrlr<rCrXr\r_rarjrmrtrwrrrrrrrrrrrrrrrrr rrr rr!rrrrrrrr )rss rregister_ops_oncez rocm_aiter_ops.register_ops_onces^ #+733;i8:  &157-:      &.24-:      &15UUU7-:      &15,n=7-:      &8<,j9>-:      &15,j97-:      &37!U9      &.24-:      &@DF     &9=?     &-13     &;?A-:      &@DF-:      &DHJ-:      &<@B     &EIK     &@DF     &48:     &59;-:      &48:-:      &8634=-:     #OOO}^ #^ #rc8tjjjjSr)r@opsvllmrxdefaultrZrrget_rmsnorm_fused_add_opz'rocm_aiter_ops.get_rmsnorm_fused_add_opvsy~?GGrc8tjjjjSr)r@rrrwrrZrrget_rmsnorm_opzrocm_aiter_ops.get_rmsnorm_opzsy~199rc8tjjjjSr)r@rrrzrrZrr&get_rmsnorm_fused_add_dynamic_quant_opz5rocm_aiter_ops.get_rmsnorm_fused_add_dynamic_quant_op~sy~HPPrc8tjjjjSr)r@rrryrrZrr"get_rmsnorm_fused_dynamic_quant_opz1rocm_aiter_ops.get_rmsnorm_fused_dynamic_quant_opy~DLLrc8tjjjjSr)r@rrr{rrZrr get_rmsnorm_group_fused_quant_opz/rocm_aiter_ops.get_rmsnorm_group_fused_quant_opsy~@HHrc8tjjjjSr)r@rrr|rrZrr$get_rmsnorm_group_add_fused_quant_opz3rocm_aiter_ops.get_rmsnorm_group_add_fused_quant_opsy~IQQrc8tjjjjSr)r@rrrrrZrrget_per_token_quant_opz%rocm_aiter_ops.get_per_token_quant_opy~8@@rc8tjjjjSr)r@rrr~rrZrrget_group_quant_opz!rocm_aiter_ops.get_group_quant_oprrc8tjjjjSr)r@rrr}rrZrr$get_act_mul_fused_fp8_group_quant_opz3rocm_aiter_ops.get_act_mul_fused_fp8_group_quant_oprrrrrcNtjj|||Sr)r@rrrwrs rrzrocm_aiter_ops.rms_norms!y~11!V=MNNNrrcPtjj||||Sr)r@rrrx)rrrrs rrms_norm2d_with_addz"rocm_aiter_ops.rms_norm2d_with_adds*y~?? x!1   rrrrrrr3cTtjj||||||Sr)r@rrrt)rrrrrr3s r gemm_a8w8zrocm_aiter_ops.gemm_a8w8s&y~221aR|TTTr block_sizecRtjj|||||Sr)r@rrrurrrrrr3s rtriton_gemm_a8w8_blockscalez*rocm_aiter_ops.triton_gemm_a8w8_blockscales+y~DD q"b,   rcRtjj|||||Sr)r@rrrvrs rrz#rocm_aiter_ops.gemm_a8w8_blockscales+y~== q"b,   rrFr%r&r'r(r)r*r+r,r-r.r/r0r1r2cftjj|||||||||| | | | | |Sr)r@rrrmrBs rr7zrocm_aiter_ops.fused_moesM$y~22                  rrDrErFrGrHrIrJc btjj|||||||||| | | | Sr)r@rrrgrPs rrLzrocm_aiter_ops.asm_moe_tkw1sG y~55           "     rrRrSrTrU.cZtjj|||||||fSr)r@rrrnr[s rrWzrocm_aiter_ops.topk_softmaxs9  .. ,(>>>>>  **1--HCCC K IaL&,q/#*I     c63  1EEEr positionsquerykey cos_sin_cache head_size rotary_dim is_neox_stylec ddlm}|}|dd\} } |j} |j} |rdnd} ||d|}||d|}|dd|f}|dd|f}|j|jdd}|||| | || dd || }|| }dS) Nr)(rope_cached_thd_positions_2c_fwd_inplacerr)rr.TF)reuse_freqs_front_part nope_first)aiter.ops.triton.ropernumelchunkrr)rrrrrrrr num_tokenscossin query_shape key_shape rotate_stylequery_key_s rtriton_rotary_embedz"rocm_aiter_ops.triton_rotary_embeds  SRRRRR__&&  &&qb&11Sk I )0qqq  :r955hhz2y11sKZK'(3  #$"INEKO4 00      #'  ;''hhy!!rXWw_scaler transpose_bmprequantrc 2ddlm}||||||||S)Nr)batched_gemm_a16wfp4)rrrr)%aiter.ops.triton.batched_gemm_a16wfp4r)rrrrrrrrs rrz#rocm_aiter_ops.batched_gemm_a16wfp4sF ONNNNN## %    rWQrr8splitKYQconfigc 8ddlm} | ||||||||||  S)Nr)Gbatched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant)rrr8rrrr)Xaiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quantr) rrrrrr8rrrraiter_triton_fp8_bmms rtriton_fp8_bmmzrocm_aiter_ops.triton_fp8_bmmsZ      $#  !%    rinput_2dcl|dks Jdtjj||S)NrzGroup size must be 128)r@rrr~)rrs rgroup_fp8_quantzrocm_aiter_ops.group_fp8_quants8 S   ":   y~88:NNNrrkc||fdvS)N) ) )i@)i i)r)r)ir)ir)r)rrr)rrrZrrs ris_triton_gemm_w8a8_tunedz(rocm_aiter_ops.is_triton_gemm_w8a8_tuneds1v   rc||fdvS)N))rr)ir)@i)ir)ir)rr)i r)i(r)rr)rp)rr)iHrr)rr)ir)rr)rr)8r)rr)rirZrs r&is_triton_gemm_afp4wfp4_presh_ws_tunedz5rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuneds1v   rrtensorlayoutc(ddlm}|||S)Nrshuffle_weightr)aiter.ops.shuffler)selfrrrs rrzrocm_aiter_ops.shuffle_weight+s+ 544444~fV4444rrtensorscJddlmtfd|DS)a Applies shuffle_weight function from AITER to each input tensor and returns them. Rearranges (shuffles) the input tensor/s into a specified block layout for optimized computation. Args: *tensors: Variable number of torch.Tensor objects. layout: A pair of integers specifying the block sizes used to divide the tensors during shuffling. Default is (16, 16). Returns: A Tuple of shuffled tensors. rrc32K|]}|VdS)rNrZ).0rrrs r z1rocm_aiter_ops.shuffle_weights..Hs2QQv^^F6:::QQQQQQr)rrr)rrrs` @rshuffle_weightszrocm_aiter_ops.shuffle_weights3s>& 544444QQQQQQQQQQQr)r N NrrFNNNNNNNNNNFNNrrbrorb)NNNrNNr)FFN)r)r)b__name__ __module__ __qualname____doc__r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rA classmethodrDr#boolrFrHrKrMrOrQrSrVrXrZr\r^r`rbre staticmethodrrrrrrrrrrrr@Tensorfloatrrrfloat16r8rlistintrrr7rLrWr^ristrrrr{rrbfloat16rrrdictrrrrrr rZrrr#r#s*;;|-N5O70M/L/L $ F#'#M 5O5O"&"G?"&"P?HH[H."4"""[":$:::[:'d'''[';4;;;[;8T888[8NTNNN[N7t777[77t777[7DDDDD[DGtGGG[G:$:::[::$:::[:FdFFF[F?t???[??t???[?`#`#`#\`#DHjHHH\H:J:::\:QJQQQ\QM MMM\MIjIII\IR*RRR\RAJAAA\AA AAA\AM*MMM\MO <O!&OAFO OOO\O  < ,     u|U\) *    \  %)$)M UU <U <U LU L U lT! U k U UUU\U%*M   <  <   L   L  I  k       \  %*M   <  <   L   L  I  k       \  ,0!" %(,(,(,(,04+/! ! |! L!  L! \ ! , ! \D( ! ! ! ! ,%! ,%! ,%! ,%!  ,-! kD(! !! ! ! \! F*.)-04046:+/!"  | L  L l  ,  <$&  <$&  ,-  ,-  !& t 3 \D(      \ > *l *l *$l *| *  * u|S ! * * *\ * *l *l *$l *| *  * u|S ! * * *\ *(+  |  l ,        %     \ *&'*  | l ,         %     \ **.*.15'+(,  < <  <   <    <$& L4' !<$.  $ ,%   \ :&*QQ <Q[Q|d"Q u|U\) * QQQ\Q&*PP <P[P|d"P u|U\) * PPP\P ).(,  < l;%  ,%   \.!"<!"|!"\!"| !"  !"  !"!"!"!"\!"F %* %'+  < <   <  Tk  +  $     \ , $($)N!"&$)"  < L    lT!  {T!  d   L4  Tk t      \ <OO,OO u|S !OOO\O  S  S  T    \   # # $   \ 0>F55l5,1#s(O5 555\5:BRRR,R(-c3hR u|S !RRR\RRRrr#r rrr)NNNrbrNNr)Pr!collections.abcrr@ torch._opsr vllm.envsr%vllm.platformsrvllm.utils.torch_utilsrr+vllm.v1.attention.ops.rocm_aiter_mla_sparserr fp8_dtyperrrrrr#rr$fp8rrrr8r<rCrNrQrXr\r_rarrjrmrrtrwrx__annotations__rrrrrrrrrrrrrrrrrrrrrrrrr r rrrrr!rr#rrZrrr)s$$$$$$ !!!!!!++++++UUUUUUUU (  ' ) ) ***** !!dX(* !!!jO(,!$($($($(,0'+''<' '  ' ' l ' $ ''''lT!'lT!'lT!'lT!'lT)'+$'  \!''''`(,!$($($($(,0'+++<+ +  + + l + $ ++++lT!+lT!+lT!+lT!+lT)++$+  \!++++8&*%),0,026'+""<" "  ", " l " |d" "|d""lT)"lT)" ""L4/"$"" \""""V&*%),0,026'+++<+ +  +, + l + |d" +|d"+lT)+lT)+ +"L4/+$++ \++++$ , ,  , <         , ,  , <         <,<,<<< <<<< , , <      $'<\,l     ! @$'   <  \  ,  l        !        ("#&<,l     ! @"#&   <  ,  l          !        (,+++#d####D&*&*-1#'$($$ |$|$ |$| $  $ |d" $ t#$|d*$$$\D $lT!$ $$$$Z&*&*-1#'$(   | |  | |    |d"   t# |d*   \D  lT!      ,!% % :: |: |:  :  : ,  : + : \::::,!% %    |   |         ,   +   \     &!& B B | B | B   B  B + B  \ B B B B"!&    |   |         +   \     $!& B B | B | B   B  B + B  \ B B B B"!&    |   |         +   \      1 | 1"\ 1=B 1 \ 1 1 1 1 |"\=B \  |l L   5< %& * |l L   5< %& & |&l& L& &  &  5<u| 34 &&&&: & | &l & L & &  &  5<u| 34 & & & & | L   5< %& (  |  L     5< %&    "&77 |77 <$ 7 5< %& 7777"& | <$  5< %& MQ |"'+6;lT6I 5< %&0MQ |"'+6;lT6I 5< %& |l L     5<u| 34 6  | l  L      5<u| 34     % |% L%% %  5< %& %%%%,  |  L     5< %&    M |MM 5< %&MMMM | 5< %&"  |   5< %&     | 5< %&(k Rk Rk Rk Rk Rk Rk Rk R\ """""r