)`ia fdZddlmZddlmZmZddlZddlmZddl m Z m Z ddl m Z ddlZddlmZmZmZmZmZmZdd lmZmZdd lmZdd lmZejd Zd ejdejdejdejddf dZ edejdeej!ejfdejfdZ"dS)a3 Copyright (c) 2025 by FlashInfer team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. )SimpleNamespace)DictListN)flashinfer_api)convert_to_block_layout!get_w2_permute_indices_with_cache)"gen_trtllm_low_latency_gemm_module) AutoTuner TuningConfigDynamicTensorSpecConstraintSpec TunableRunnerOptimizationProfile)&get_last_power_of_2_num_tokens_bucketslast_positive_power_of_2)setup_cubin_loader)_get_cache_bufct}|tt|Gfddt fd}t |S)Nc eZdZdeejdedeeffd Z d deejdede dejffd Z d S) Fget_trtllm_low_latency_gemm_module..TrtllmLowLatencyGemmRunnerinputsprofilereturnc  d}d}||}||}|d}|d}|d} |\}}} } d} d} t||| | | }|S)Nr)get_opt_shapeslisttrtllm_low_latency_gemm_tactics)selfrra_tensor_indexb_tensor_indexabmnk global_scaleout type_e4m3 type_bf16 valid_tacticsops v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/trtllm_low_latency_gemm.pyget_valid_tacticszXget_trtllm_low_latency_gemm_module..TrtllmLowLatencyGemmRunner.get_valid_tactics5s NN&&((8A&&((8A!A!A!A   II 221aIyQQM! Ftacticdo_preparationc|\}}}}|dkr|S|jd} |jd} |jd} | | | |} td| |j} | ||||||S)Nrrtrllm_low_latency_gemm)shapeget_workspace_size_in_bytesrdevicetrtllm_low_latency_gemm)r rr2r3kwargsr#r$r(r)r%r&r'workspace_sizeworkspace_bufferr-s r.forwardzNget_trtllm_low_latency_gemm_module..TrtllmLowLatencyGemmRunner.forwardPs  zz  A A A;;Aq!VLLN-(.!(     & &     Jr0N)r1F) __name__ __module__ __qualname__rtorchTensorrintr/boolr=)r-sr.TrtllmLowLatencyGemmRunnerr4s !& !) !#Y  ! ! ! ! ! !<#(   &  !   \         r0rEcS)N)rEsr. gemm_runnerz7get_trtllm_low_latency_gemm_module..gemm_runnerps))+++r0)rH)r build_and_loadrstrget_library_pathrr)modrHrEr-s @@r."get_trtllm_low_latency_gemm_modulerM.s , . .C    Bs3//1122333:::::::]:::x,,,,,    r0ABr(r)rc | tj}d d}tt fdtt ft |d fdf}||||g}g}|t | d|||\} } | || |S) aGEMM optimized for low M dimension. B needs to be shuffled and its layout needs to be adjusted. Only supported on Blackwell GPUs. Parameters ---------- A: torch.Tensor Input tensor, shape (m, k), fp8 e4m3. B: torch.Tensor Mat2 tensor, shape (k // block_size, n, block_size), fp8 e4m3. block_size is 128 for e4m3. global_scale: torch.Tensor Scale tensor for the output, float. out: torch.Tensor Out tensor, shape (m, n), bf16. Examples -------- >>> import torch >>> from flashinfer import mm_fp8, prepare_low_latency_gemm_weights >>> m = 16 >>> n = 2560 >>> k = 32768 >>> a = torch.randn([m, k], device="cuda", dtype=torch.bfloat16) >>> a_fp8, a_inv_s = to_float8(a, dtype=torch.float8_e4m3fn) >>> b = torch.randn([n, k], device="cuda", dtype=torch.bfloat16) >>> b_fp8, b_inv_s = to_float8(b, dtype=torch.float8_e4m3fn) >>> prepared_b = prepare_low_latency_gemm_weights(b_fp8, _cache_permute_indices) >>> prepared_b.shape torch.Size([256, 16, 128]) >>> global_scale = a_inv_s * b_inv_s >>> out = torch.zeros([m, n], device="cuda", dtype=torch.bfloat16) >>> mm_fp8(a_fp8, prepared_b, global_scale, out) >>> out.shape torch.Size([16, 2560]) r)rRc |dS)NrRrG)shapesr!s r.z)trtllm_low_latency_gemm..sVN5KB5Or0)dynamic_tensor_specsconstraint_specsr9)rr2) r getr r rrr appendrMrH choose_one) rNrOr(r)tunerout_tensor_index tuning_configrrunnersrunnerr2r!s @r.r9r9ysX MOOEN !6(      "&O&O&O&O   ML# &F#%G NN577CCEEFFF%%! NFF F&(((( Jr0wpermutation_indices_cachecd}t|||}|||j}d}t ||}|S)aHelper method to prepare the input weight tensor for low-latency TRTLLM GEMM. It includes shuffling and converting to block layout. Parameters ---------- w: torch.Tensor The weight tensor to shuffle, shape (n, k), fp8 e4m3. permutation_indices_cache: dict Some location to cache permutation indices. Calculating them is expensive. Returns ------- block_layout_shuffled_weights: torch.Tensor The shuffled and block-layout weight tensor, shape (k // 128, n, 128), fp8 e4m3. )r8)rtor8 contiguousr)r`raepilogue_tile_mpermute_indicesshuffled_weightsblock_kblock_layout_weightss r. prepare_low_latency_gemm_weightsrksg(O7!1oO++18+<<=HHJJG23CWMM r0)#__doc__typesrtypingrr functoolsflashinfer.api_loggingrflashinfer.fused_moe.corerrflashinfer.jit.gemm.corer rAflashinfer.autotunerr r r r rrflashinfer.fused_moe.utilsrrflashinfer.jitrflashinfer.utilsrcacherMrBr9SizerkrGr0r.rys "!!!!!111111HGGGGG .-----++++++ GGGTI |I |I,I  I  IIIIX  | 04UZ5M0N  \      r0