
    )`i/,              	          d Z ddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZ ddlmZ dd	lmZmZ dd
lmZ ddede
fdZddede
fdZddede
fdZddede
fdZddede
fdZ	 ddee         dedede
fdZde
fdZdS )a3  
Copyright (c) 2025 by FlashInfer team.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
    )List   )env   )ArtifactPathCheckSumHash)JitSpecgen_jit_speccurrent_compilation_contextsm90a_nvcc_flagssm89_nvcc_flags)is_cuda_version_at_least)	get_cubinget_meta_hash)generate_gemm_operationsFuse_fast_buildreturnc                 ^    g d}|t          j        dg          z  }t          |d|           S )N)-DCOMPILE_BLACKWELL_TMA_GEMMSz+-DCOMPILE_BLACKWELL_SM120_TMA_GROUPED_GEMMS-DENABLE_BF16-DENABLE_FP8-DENABLE_FP4-DUSING_OSS_CUTLASS_MOE_GEMM   supported_major_versions120r   get_nvcc_flags_listgen_cutlass_fused_moe_moduler   
nvcc_flagss     l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/jit/fused_moe.py"gen_cutlass_fused_moe_sm120_moduler$   !   sM      J -A"$   J (
E>JJJ    c                 ^    g d}|t          j        dg          z  }t          |d|           S )N)r   %-DCOMPILE_BLACKWELL_TMA_GROUPED_GEMMSr   r   r   r   z+-DCOMPILE_BLACKWELL_SM103_TMA_GROUPED_GEMMS
   r   103r   r!   s     r#   "gen_cutlass_fused_moe_sm103_moduler*   2   sM      J -A"$   J (
E>JJJr%   c                 `    g d}|t          j        ddg          z  }t          |d|           S )N)r   r'   r   r   r   r   r(      r   100r   r!   s     r#   "gen_cutlass_fused_moe_sm100_moduler.   D   sO      J -A"$b   J (
E>JJJr%   c           	          t           ddddt          d          rdndt          d          rdndd	gz   }t          |d
|           S )Nz-DCOMPILE_HOPPER_TMA_GEMMSz"-DCOMPILE_HOPPER_TMA_GROUPED_GEMMSr   r   12.8-DENABLE_FP8_BLOCK_SCALE r   r   90)r   r   r    r!   s     r#   !gen_cutlass_fused_moe_sm90_moduler4   U   s^    !$,&>v&F&FN""B26::B&% J (
D.IIIr%   c                 b    t           ddt          d          rdnddgz   }t          |d|           S )Nr   r   r0   r1   r2   r   89)r   r   r    r!   s     r#   !gen_cutlass_fused_moe_sm89_moduler7   b   sD     &>v&F&FN""B&	$ J (
D.IIIr%   r"   device_archc                     t           j        d| z  	                     dd           t          | d| d           n%# t          $ r}t          d|           |d}~ww xY wt          d| t           j        d	z  t           j        d
z  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  t           j        dz  gfd                    d          D             t           j        dz  t           j        dz  t           j        d z  t           j        d!z  t           j        d"z  t           j        d#z  t           j        d$z  t           j        d%z  | |rd&gng d'gt           j        d(z  t           j        d(z  d)z  t           j        d(z  d*z  d+z  d)z  t           j        d(z  d*z  d,z  d-z  d)z  t           j        d(z  d*z  d,z  d-z  g.          S )/z>
    Generate a JitSpec for the cutlass fused moe module.
    z0nv_internal/tensorrt_llm/cutlass_instantiations/T)parentsexist_ok;z-realz$Failed to generate Cutlass kernels: N
fused_moe_z`nv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_tma_warp_specialized_input.cuzWnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp8_uint4.cuzUnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp8_fp8.cuzUnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp8_fp4.cuzUnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp4_fp4.cuzWnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp32_fp32.cuzXnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp16_uint8.cuzXnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp16_uint4.cuzWnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp16_fp16.cuzXnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_bf16_uint8.cuzXnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_bf16_uint4.cuzVnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_bf16_fp8.cuzWnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_bf16_bf16.cuzVnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_bf16_fp4.cuzVnv_internal/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_gemm_kernels_fp16_fp4.cuz[nv_internal/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_gemm.cuzAfused_moe/cutlass_backend/flashinfer_cutlass_fused_moe_binding.cuz/fused_moe/cutlass_backend/deepgemm_jit_setup.cuz<fused_moe/cutlass_backend/cutlass_fused_moe_instantiation.cuc              3   "   K   | ]	}|z  V  
d S )N ).0kernel
output_dirs     r#   	<genexpr>z/gen_cutlass_fused_moe_module.<locals>.<genexpr>   s(      SSfj6!SSSSSSr%   z*.generated.cu#nv_internal/cpp/common/envUtils.cpp!nv_internal/cpp/common/logger.cpp&nv_internal/cpp/common/stringUtils.cpp(nv_internal/cpp/common/tllmException.cpp%nv_internal/cpp/common/memoryUtils.cuz7nv_internal/tensorrt_llm/kernels/preQuantScaleKernel.cuzFnv_internal/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cppz.nv_internal/tensorrt_llm/kernels/lora/lora.cppz-DFAST_BUILDz-lnvrtcnv_internalincludetensorrt_llmcutlass_extensionskernelscutlass_kernels)extra_cuda_cflagsextra_cflagsextra_ldflagsextra_include_paths)jit_envFLASHINFER_CSRC_DIRmkdirr   	ExceptionRuntimeErrorr
   rglob)r"   r8   r   erB   s       @r#   r    r    l   s    	#
J[
J
J	K 

N555 //[///	
 	
 	
 	

  N N NE!EEFFAMN "[""'pq'gh'ef'ef'ef'gh'hi'hi'gh'hi'hi'fg'gh'fg'fg'kl'QR'?@'LMK4	
P TSSS
0@0@AQ0R0RSSSQ4	
R '*OOS4	
T '*MMU4	
V '*RRW4	
X '*TTY4	
Z '*QQ[4	
\ 'GH]4	
` 'VWa4	
d '>?e4	
j %)7?n%%R k'-7'-7)C' ## 	
 '   	 
  '   	 
uN N N Ns   -A 
A%A  A%c                  "   t           j         d} d}t           j         d}t          |t          j                  }|sJ d|             t	          |          }t          |  d| d|          }|sJ | d            t          j        dg	          }t          d
t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  t          j	        dz  gdddddddt           j         dg|z   t          j
        | z  t          j	        dz  t          j	        d z  g!          S )"Nz/includeflashinferMetaInfoz/checksums.txtz!Failed to get checksums.txt from /z.hz.h not foundr(   r   fused_moe_trtllm_sm100rD   rE   rF   rG   rH   z#trtllm_fused_moe_kernel_launcher.cuztrtllm_fused_moe_runner.cuz$trtllm_fused_moe_routing_deepseek.cuz"trtllm_fused_moe_routing_llama4.cuz'trtllm_fused_moe_routing_renormalize.cuztrtllm_fused_moe_dev_kernel.cuztrtllm_batched_gemm_runner.cuz-DTLLM_GEN_EXPORT_INTERFACEz-DTLLM_GEN_EXPORT_FLASHINFERz-DTLLM_ENABLE_CUDAr   r   r   z-DTLLM_GEN_GEMM_CUBIN_PATH=\"z\"rI   znv_internal/include)rO   rR   )r   TRTLLM_GEN_BMMr   r   r   r   r   r
   rS   rT   FLASHINFER_CUBIN_DIR)include_pathheader_namechecksum_pathchecksum	meta_hashmetainfor"   s          r#   %gen_trtllm_gen_fused_moe_sm100_modulerf      s    #1;;;L&K $2BBBM(CDDHHHHHHHH8h''I ))+))) H
 11111118 -@"$  J  '*OO'*MM'*RR'*TT'*QQ'*OO'*FF'*PP'*NN'*SS'*JJ'*II	
 ** M\-HMMM
 	 (<7'-7'*??	
5       r%   N)F)__doc__typingr   r2   r   rS   	artifactsr   r   corer	   r
   r   r   r   cpp_extr   cubin_loaderr   r   gemm.cutlass.generate_kernelsr   boolr$   r*   r.   r4   r7   strr    rf   r?   r%   r#   <module>rp      s*                 2 2 2 2 2 2 2 2              . - - - - - 2 2 2 2 2 2 2 2 C C C C C CK Kt K K K K K"K Kt K K K K K$K Kt K K K K K"
J 
Jd 
Jw 
J 
J 
J 
JJ Jd Jw J J J J EJe eS	e(+e=Aee e e eP:w : : : : : :r%   