)`ipldZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z ddl Z ddlmZddlmZmZdd lmZdd lmZmZmZmZmZmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6m7Z7ddl8m9Z9ddl:m;Z;ddlm?Z?ddl@mAZAddlBmCZCddlDmEZEddlFmGZGmHZHddlImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRddlSmTZTmUZUddlSmVZWddlXmYZYddlZm[Z[de j\d e j\d!e]d"e]d#e^d$e^d%e eTfd&Z_d'e j\d e j\d(e j\d!e]d"e]d#e^d$e^d%e eTfd)Z`d*e e j\d+e e j\d,e e e]e]fd-e e e]e]fd.e e^d/e e^d0e^d1e^d2e^d3e^d%e eTfd4Zad5e e j\d6e e^d7e e]d8e e]d9e e]d.e e^d0e^d1e^d:e^d;e^d%e eTfd<Zbd*e e j\d+e e j\d,e e e]e]fd-e e e]e]fd.e e^d/e e^d=ecd>e^d2e^d3e^d?e^d@e^dAe^dBe^d%e eTfdCZddDe eTdEed%dfdFZe dXdEe edIedJedKecdLe^dMe^d%dfdNZfdOegd%e^fdPZhdQegd%e e]e]ffdRZidSZjdTZkd%e]fdUZldVZmendWkr emdSdS)Ya Copyright (c) 2025 by FlashInfer team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. AOT build script for FlashInfer. NOTE (Zihao): The following modules are intentionally excluded from the AOT build: - gen_pod_module - gen_deepgemm_sm100_module (it doesn't involve host-side compilation) N)product)Path)ListTupleIteratorOptional)Version)act_func_def_strgen_act_and_mul_module)gen_cascade_module) gen_fp4_quantization_sm90_module!gen_fp4_quantization_sm100_module!gen_fp4_quantization_sm103_module!gen_fp4_quantization_sm110_module!gen_fp4_quantization_sm120_module!gen_fp4_quantization_sm121_module)#gen_mxfp8_quantization_sm100_module)gen_gdn_prefill_sm90_module)"gen_cutlass_fused_moe_sm120_module"gen_cutlass_fused_moe_sm103_module"gen_cutlass_fused_moe_sm100_module!gen_cutlass_fused_moe_sm90_module%gen_trtllm_gen_fused_moe_sm100_module) gen_gemm_modulegen_gemm_sm90_modulegen_gemm_sm100_module!gen_gemm_sm100_module_cutlass_fp4!gen_gemm_sm100_module_cutlass_fp8gen_tgv_gemm_sm10x_modulegen_gemm_sm120_module!gen_gemm_sm120_module_cutlass_fp4gen_trtllm_gen_gemm_module"gen_trtllm_low_latency_gemm_module)gen_spdlog_module)gen_mla_module)!gen_selective_state_update_module&gen_selective_state_update_sm90_module)gen_norm_module)gen_page_module)gen_quantization_module)gen_rope_module)gen_sampling_module)gen_topk_module)gen_trtllm_utils_module)gen_xqa_modulegen_xqa_module_mla) gen_batch_attention_modulegen_batch_decode_modulegen_batch_mla_modulegen_batch_prefill_modulegen_cudnn_fmha_modulegen_fmha_cutlass_sm100a_modulegen_single_decode_modulegen_single_prefill_modulegen_trtllm_gen_fmha_module)JitSpecbuild_jit_specs)env)get_cuda_version)CompilationContextdtype_qodtype_kv head_dim_qk head_dim_vouse_sliding_windowuse_logits_soft_capreturnc #PK|j|jkr||krdS|jdkrdStd|||||d||d Vtd|||tj||d||d Vt |||||d||Vt |||tj||d|| VdS) Nr fa2rF) backenddtype_qrAdtype_orBrCpos_encoding_moderDrEuse_fp16_qk_reduction rIrJrArK dtype_idxrBrCrLrDrErM)rJrArKrBrCrLrDrE rJrArKrOrBrCrLrDrE)itemsizer9r5torchint32r8r3r@rArBrCrDrEs b/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/aot.pygen_fa2rV_sEH---(h2F2FA #-/#       #+-/#       #-/       "+-/        rJrKc #K||krdS|jdkr||krdS|jdkr|dks|dkrdStd|||tj||d||d VdS) Nr @fa3rFrN)rQr5rRrSrJrArKrBrCrDrEs rUgen_fa3r^s(1 g   FA #  !2!2 F "+-/#        rW f16_dtype_ f8_dtype_ fa2_head_dim_ fa3_head_dim_use_sliding_window_use_logits_soft_cap_has_sm90 has_sm100 add_gemma add_oai_ossc #Kd} d} t||||z||D]J\\} } }}}}t||| | ||Ed{Vt|||tj| | d|d VK|r>t|||z|||D]'\\} } }}}}t |||| | ||Ed{V(|rwt|||zdgD]$\}}\}}t||d d ||Ed{V%|r;t||z|dgD]%\}}\}}t |||d d ||Ed{V&| r5d d lm}|D],}d D]'}d D]"}|||||tjddd| V#(-|rRttj tj tj tjddddd VtVdg|rdgngz}|D]*}|D]%}t||||tj| | dV&+|rtVdSdS)Nir[rTrF) rJrArKrOrBrCrLrE use_profilerr])TTr )'gen_batch_prefill_attention_sink_module)rHr\)TF) rIrJrArKrOrBrCrLrDrPrHr\)rIrJrArKrO head_dim_ckv head_dim_kperj) rrVr2rRrSr^ jit.attentionrlr7bfloat16r:r4r&)r_r`rarbrcrdrerfrgrhrnrorBrCr@rArDrE dtype_qkvrKrldtyperIuse_swa mla_backend_s rU gen_attentionrvsULL Y       "k##1 3           )k## 3       "        &[+    !"''#5$7        $    " N       5 !4 !!#5$7            Y&   9#%8 #%&# # #'9(;JJJJJJ  E)  ,  GAA ' %!& %"'+$&$&*++2          +,N^Nk$ %    )*****79uggr:L  #  G& ! +))"        rW input_type_ fp8_kv_cache_token_per_page_ head_size_head_grp_size_ has_sm120 has_sm121c #NK|s|s|s| sdSt||||||D]M\} } } } }}| dzdks | dks| dkr| dvr$| r tj}n| }t| || | ||| VN|s| r0|D]/} t tjtj| ddd V.dSdS) z0Generate XQA modules for various configurations.Nrrkr r[rm) input_dtypekv_cache_dtype page_sizehead_dimhead_group_ratiorD output_dtypei@rmF)rrrrrrD)rrR float8_e4m3fnr0r1)rwrxryrzr{rcrerfr|r} input_type fp8_kv_cachetoken_per_page head_size head_grp_sizerDrs rUgen_xqarmsc Ii     " "   r>Q  )c//Y^^  !2 2 2   ("0NN'N")$*1#        I -  N$!/$2(!$#(         rWsm_capabilitiesadd_commadd_moeadd_actadd_miscadd_xqac4g}|t|dd}|dd}|dd}|dd}|dd}|dd}|dd}|tt |||||||||| z }| r,t D]$}|t |%| r||t|rc|t|t|t|r|t|t|t|t|t|t!t"jd |t!t"jd |t)|t+|t-|t/|r\|t!t"jd |t!t"jd |rB|t1|t3|r!|t5|r|t7|t9|t;|t=|r!|t?|rd d l m!}m"}d d l m#}d dl m$}d dl m%}d dl m&}|||||rW||||||||| r|tOtQtStUtWtYt[t]gz }|rc|t_|ta|tc| rotetgdkrPt"jt"jg}d dg}gd}gd} gd}!|tti|||| |!||||| z }|tktm}"g}#|D]:}$|$j7|"vr/|"8|$j7|#|$;|#S)Nsm90Fsm100sm100fsm103sm110sm120sm121) use_sm_100fTr )gen_trtllm_comm_modulegen_vllm_comm_module)gen_nvshmem_module)gen_comm_alltoall_module)gen_trtllm_mnnvl_comm_module)gen_moe_alltoall_module12.8r)r[rmrk)r rY)9appendr%getlistrvr r rrrrrrrrrr rRrqfloat16rr#r$rrrrrrr!r"rjit.commrrrrrrr r)r*r+r,r-r.r'r(r/rr>r rr6setnameadd)%r_r`rarbrcrdrrrgrhrrrr jit_specsrerf has_sm100f has_sm103 has_sm110r|r}act_namerrrrrrxqa_input_type_xqa_fp8_kv_cache_xqa_token_per_page_xqa_head_size_xqa_head_grp_size_namesretjit_specs% rUgen_all_modulesrsF "I &(()))""6511H##GU33I $$Xu55J##GU33I##GU33I##GU33I##GU33I              I?( ? ?H   3H== > > > >(B**+++  B   133 4 4 4   =?? @ @ @   >@@ A A A  F   >@@ A A A   ?AA B B B   244 5 5 5   >@@ A A A   >@@ A A A   )%.eLLL      )%-UKKK      @BB C C C   799 : : :   ?AA B B B   BDD E E E  Y   )%.dKKK      6u}RVWWW X X X  C   >@@ A A A   ?AA B B B  B   >@@ A A A  B   >@@ A A A   ?AA B B B   244 5 5 5   >@@ A A A  B   >@@ A A A 1JJJJJJJJ000000666666::::::555555++--...1133444  8   3355 6 6 6   99;; < < <   4466 7 7 7--//000<        # % %     ! !    - / /    <   CEE F F F   466 7 7 7   8:: ; ; ;  $&&88!=%.9!5M///')\\T !#"#     *,,--- EEEC!! = % % IIhm $ $ $ JJx JrWrout_dircX|rtj||dd|D]g}tj|jz |jdz }||jz |jdz }|jddtj||hdS)NTFparentsexist_okz.so)rr) existsshutilrmtreemkdirjit_envFLASHINFER_JIT_DIRrparentcopy2)rrrsrcdsts rUcopy_built_kernelsrIs~~ g MM$M///(8=8hm;P;P;PP %8=(=(=(== %777 S# rWFT build_dir project_rootconfigverbose skip_prebuiltcBt}||||}dtjvrt dt }|dz t _|dz t _|dz dz dz |dz dz dz d z dz gt _ |dz d z dz t _ |t _ |d z t _ |d z t _ t j d d t j d d |r2td|td|td|td|dtd|dtd|dtd|dtd|dtd|dtdtjdtd|D]\}} | rtd |d!d"D]} td#| d$|| |rtd%t#|d|d|d|d|d|d||d&|d'|d(|d)|d*|d+|d,} |rtd-t%| t'| ||.|t)| ||rtd/|dSdS)0a Compile and package modules based on the provided configuration. Args: out_dir: Output directory for packaged modules build_dir: Build directory for compilation project_root: Project root directory config: Configuration dictionary to override defaults (optional) verbose: Whether to print verbose build output skip_prebuilt: Whether to skip pre-built modules NFLASHINFER_CUDA_ARCH_LISTz8Please explicitly set env var FLASHINFER_CUDA_ARCH_LIST.csrcinclude3rdpartycutlasstoolsutilspdlog cached_ops generatedTrzAOT build summary:z out_dir:z build_dir:z fa2_head_dim: fa2_head_dimz fa3_head_dim: fa3_head_dimz f16_dtype: f16_dtypez f8_dtype:f8_dtypez use_sliding_window:rDz use_logits_soft_cap:rEz FLASHINFER_CUDA_ARCH_LIST:z SM capabilities detected:z z: Truerrgrhrrrrz :zGenerating JIT specs...rrgrhrrrrz Total ops:)rrzAOT kernels saved to:)get_default_configupdateosenviron RuntimeErrordetect_sm_capabilitiesrFLASHINFER_CSRC_DIRFLASHINFER_INCLUDE_DIRCUTLASS_INCLUDE_DIRSSPDLOG_INCLUDE_DIRFLASHINFER_WORKSPACE_DIRrFLASHINFER_GEN_SRC_DIRrprintitemsrlenr<r) rrrrrr final_configrsm_namehas_smkeyrs rUcompile_and_package_modulesrWs(&''L F### F""*44UVVV,..O#/"7G%1I%=G"z!I- 9z!I-7&@9L$G ". !:X!E !QG(1G$!*\!9G%.%z9detect_sm_capabilities..has_sm..s'BBt7d?BBBBBBrWF)anyr>r )rrgencode_flags_lists` rUrz&detect_sm_capabilities..has_smsHBBBB/ABBBBB 5!!WW%5%555rW compute_90z12.3 compute_100rz12.9 compute_103 compute_110z13.0 compute_120 compute_121)rrrrrrr)r?get_nvcc_flags_liststrbool)compilation_contextrrs @rUrrs,..,@@!%A66c6d666666|V,, v..&// v.. v.. v.. v..  rWc,t}t}t|d|d|d|d|d|d||d|d|d |d |d |d |d }t|S)z#Register the default set of modulesrrrrrDrErrgrhrrrr)rrrr)rrrs rUregister_default_modulesr#s  ! !F,..O{z~~#$$%z{}yyzyI y>>rWc<tjd}|dtd|dtd|dd d |d d d |dd ddgd|dd ddgd|dd d |dd d |dtd|dtd|dtd |d!td"|d#td$|d%td&|d'td(|}tt jd)}t}tj }d}|j rt|j }|j rt|j }|jrd*|jD|d+<|jrd,|jD|d-<|jrd.|jD|d/<|jrd0|jD|d1<|jrd2|jD|d3<|jrd4|jD|d5<d6D]}t)||d}||||<t+||||d7d89dS):Nz%Ahead-of-Time (AOT) build all modules) descriptionz --out-dirzOutput directory)typehelpz --build-dirzBuild directoryz--fa2-head-dim*z2FA2 head dim pair of qk and vo, separated by comma)nargsr'z--fa3-head-dimz2FA3 head dim pair of qk and vo, separated by commaz --f16-dtyperrqz16-bit data type)r)choicesr'z --f8-dtyper float8_e5m2z8-bit data typez--use-sliding-windowzUse sliding window attentionz--use-logits-soft-capzUse logits soft capz --add-commz2Add communication kernels (trtllm_comm, vllm_comm)z --add-gemmazSAdd kernels for Gemma Model (head_dim=256, use_sliding_window, use_logits_soft_cap)z --add-oai-ossz?Add kernels for OAI OSS Model (head_dim=64, use_sliding_window)z --add-moezAdd MoE kernelsz --add-actzAdd activation kernelsz --add-misczAdd miscellaneous kernelsz --add-xqaz'Add XQA (Cross-Query Attention) kernelsr c,g|]}t|Sr rrdims rU zmain..V !S!S!S#."5"5!S!S!SrWrc,g|]}t|Sr r-r.s rUr0zmain..Xr1rWrc8g|]}tt|Sr getattrrRrrss rUr0zmain..Zs"QQQwue44QQQrWrc8g|]}tt|Sr r4r6s rUr0zmain..\s"OOOgeU33OOOrWrc,g|]}t|Sr rrrs rUr0zmain..^s'W'W'W! 1 'W'W'WrWrDc,g|]}t|Sr r9r:s rUr0zmain..`s+) ) ) JqMM) ) ) rWrErTF)rrrrrr)argparseArgumentParser add_argumentrr parse_args__file__resolverrrrrrrrrrrDrEr5r)parserargsrrrrr arg_values rUmainrEs  $;F  $5GHHH  D7HIII  A   A  J'     -0    c0N /sAVWWW   A    b    N   *;LMMM  *;STTT  :>))++3A6L  ! !F0I"G |%t|$$ ~)((  T!S!SAR!S!S!S~ T!S!SAR!S!S!S~ ~RQQ$.QQQ{ }POOOOOz X'W'Wt?V'W'W'W#$  ) ) #'#;) ) ) $% $ $D#t,,  #F3K ! rW__main__)NFT)o__doc__r<rr itertoolsrpathlibrtypingrrrrrRpackaging.versionr jit.activationr r jit.cascader jit.fp4_quantizationrrrrrrjit.fp8_quantizationrjit.gdnr jit.fused_moerrrrrjit.gemmrrrrrr r!r"r#r$ jit.spdlogr%jit.mlar& jit.mambar'r(jit.normr)jit.pager*jit.quantizationr+jit.roper, jit.samplingr-jit.topkr.jit.tllm_utilsr/jit.xqar0r1rpr2r3r4r5r6r7r8r9r:jitr;r<r=r jit.cpp_extr>r!r?rsrr rVr^rvrdictrrrrrrrrr#rE__name__r rWrUrbs, 222222222222 %%%%%%DDDDDDDD++++++FEEEEE000000                        *)))))######&%%%%%%%%%%%555555%%%%%%------%%%%%%33333377777777                      *)))))))))))))333333=k=k== =  =  =g====@ [k[     gDiU[!iEK ic3h(ic3h( i d i t* iiiiigiiiiX=ek"=:=#Y=S = I = d =====g====@YU[!YEK Yc3h(Yc3h( Y d Y t* YYYYYYYYY ']YYYYx G}        $ g0g0 d^g0g0g0  g0  g0  g0 g0g0g0g0T8#8$8888SU38_ &2#0fffR zDFFFFFrW