)`i42 dZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z ddl Z ddlmZddlmZmZddlmZmZdd lmZdd lmZmZmZmZmZmZmZmZm Z m!Z!dd l"m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@ejAdZBejAdeCdefdZDejAdZEejAdZFedddddede jGde jGde jGdeCdeHdeIfdZJejAdZKe d`de jGde jGde jGdeCd!eCd"eId#e eLd$e eLd%e eLdeHd&e eLd'e eLd(e eLd)e eLdedd*e jGf d+ZMe dade jGde jGde jGdeCd!eCd"eId#e eLd$e eLd%e eLdeHd&e eLd'e eLd(e eLd)e eLded,d*e e jGe jGff d-ZMe d`de jGde jGde jGdeCd!eCd"eId#e eLd$e eLd%e eLdeHd&e eLd'e eLd(e eLd)e eLdeId*e e jGe e jGe jGfff d.ZMGd/d0ZNGd1d2eNZOGd3d4ZPGd5d6ZQejAd7ZRe dbd;e jGde jGd?e jGd@eHdAe eLe jGfdBe eLe jGfdeHdCe e e jGe*fdDe e e jSeCfdEe eLdFe eHdGe ee jGdeCdHe eIdIeCdJe eHdKe eLdLe e jGdMe eHdNe e jGd*e e jGe*ff.dOZTe dcd;e jGde jGd?e jGd@eHdAe eLe jGfdBe eLe jGfdeHdCe e jGdGe e jGdeCdHeIdJe eHdKe eLdLe e jGd*e jGf"dPZU dddQe jGdRe jGdSe jGdTeHdUeHdVeHdWeHd!eCdeHd&e eLdXe e eCe jSfdYe e eCe jSfdZe e eCe jSfd'e eLd(e eLd)e eLd[eId\e eHd]eId^e e jGd*df*d_ZVdS)ea3 Copyright (c) 2023 by FlashInfer team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. N)SimpleNamespace)AnyListLiteralOptionalTupleUnionoverload)flashinfer_api)%trtllm_batch_decode_with_kv_cache_mla"xqa_batch_decode_with_kv_cache_mla)xqaxqa_mla) cudnn_batch_decode_with_kv_cache) gen_batch_decode_mla_modulegen_batch_decode_module!gen_customize_batch_decode_module"gen_customize_batch_prefill_modulegen_single_decode_moduleget_batch_decode_uriget_batch_prefill_uriget_single_decode_urisetup_cubin_loadergen_trtllm_gen_fmha_module) get_seq_lens)get_batch_prefill_jit_moduleget_batch_prefill_moduleget_single_prefill_module)log2e FP4TensorMaskModePosEncodingMode TensorLayout_check_cached_qkv_data_type_check_kv_layout_check_pos_encoding_modecheck_shape_dtype_device_get_cache_alibi_slopes_buf_get_cache_buf_get_range_buf_unpack_paged_kv_cachecanonicalize_torch_dtypedetermine_attention_backenddevice_support_pdlget_device_sm_count is_float8register_custom_opregister_fake_opceil_divround_upget_compute_capabilityGPUArchitectureErrorcNt|}t|}|jt d|dddt jdt jdt jdt jd t jd tt jd tt jd td tdtdtdtdtddffd }td|ddt jdt jdt jdt jd t jd tt jd tt jd td tdtdtdtdtddfd}t|S)N flashinfer::_run)tmpo mutates_argsqkvr;r< maybe_lse alibi_slopeskv_layout_code window_leftlogits_soft_capsm_scale rope_scale rope_thetareturnc B |||||||||| | d| z d| z dSN?)r?r@rAr;r<rBrCrDrErFrGrHrIrun_funcs e/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/decode.pyrun_single_decodez3get_single_decode_module..run_single_decodeWsT         *  *      c dSNrN) r?r@rAr;r<rBrCrDrErFrGrHrIs rP_fake_run_single_decodez9get_single_decode_module.._fake_run_single_decodews rR)run) rrbuild_and_loadrVr2torchTensorrintfloatr3r)argsurimodulerQrUrOs @rPget_single_decode_moduler_Os  &C %t , ; ; = =FzH0s000|LLL < <  < \  <  EL)  u|,             ML >.S...// < <  < \  <  EL)  u|,           0/ $ 0 1 1 11rR module_name jit_modulec|j}|jtd|dddtjdtjdt t dtjd ttjd ttjd tjd tjd tjdtjdttjdt dt dtddffd }td|ddtjdtjdt t dtjd ttjd ttjd tjd tjd tjdtjdttjdt dt dtddfd}t||S)Nr9r:float_workspace_bufferint_workspace_buffer paged_k_cache paged_v_cacher<rBr=rdre plan_info_vecr?rfrgpaged_kv_indptrpaged_kv_indicespaged_kv_last_page_lenr<rBrDrE enable_pdlrJc4|||||||||| | | | | g|RdSrTrN)rdrerhr?rfrgrirjrkr<rBrDrErlr\rOs rPrun_batch_decodez5get_batch_decode_jit_module..run_batch_decodes\8  "      "           rRcdSrTrN)rdrerhr?rfrgrirjrkr<rBrDrErlr\s rP_fake_run_batch_decodez;get_batch_decode_jit_module.._fake_run_batch_decodes $ rRplanrV) rrrVr2rXrYrrZrboolr3r)r`ra plan_funcrnrprOs @rPget_batch_decode_jit_modulerus%I~H({(((    ! % ! #l! Cy!  < !   - !   - ! !  ,! !& !  <! EL)! ! ! ! !! ! ! ! !   ! F6[66677 %  #l Cy  <    -    -    , !&   < EL)    !   87 &      rRc)t|}t|}|j}|jt d|dddt jdt jdttdt jd tt jd tt jd t jd t jd t jdt jdtt jdtdtdtdtt jdtdtdtdtddf(fd }td|ddt jdt jdttdt jd tt jd tt jd t jd t jd t jdt jdtt jdtdtdtdtt jdtdtdtdtddf(d}t||S)Nr9r:rcr=rdrerhr?rfrgrirjrkr<rBrDrErlrCrFrGrHrIrJcN|||||||||| | | | | |||d|z d|z dSrLrN)rdrerhr?rfrgrirjrkr<rBrDrErlrCrFrGrHrIrOs rPrnz1get_batch_decode_module..run_batch_decodesg@  "      "        *  * '     rRcdSrTrN)rdrerhr?rfrgrirjrkr<rBrDrErlrCrFrGrHrIs rPrpz7get_batch_decode_module.._fake_run_batch_decodes , rRrq)rrrWrrrVr2rXrYrrZrrsr[r3r)r\r]modrtrnrprOs @rPget_batch_decode_modulerzs  %C !4 ( 7 7 9 9CIwH s       ) % ) #l) Cy)  < )   - )   - ) )  ,) !& )  <) EL)) ) ) ) u|,) !) "#) $%) &') ( )) ) ) ) )   ) V.S...// %  #l Cy  <    -    -    , !&   < EL)    u|, ! "# $% &' ( )   0/ 6      rRct}|}t||SrT)rrWrget_library_path)ryops rPget_trtllm_gen_fmha_moduler~;s= $ & &C    Bs++--... IrRNHDF) kv_layoutrE return_lser?r@rArrErc |j}tdd|} tj|} |r5tj|dtj|} nd} |j|||| | | t|j |g|R| S)Nsingle_decode_with_kv_cache_tmprdtypedevice) rr*rX empty_likeemptysizefloat32rVr$value) rar?r@rArrErr\rr;r<lses rP+single_decode_with_kv_cache_with_jit_modulerCsXF : >>rRNONEpos_encoding_modeuse_tensor_coresq_scalek_scalev_scalerFrGrHrIrJcdSrTrNr?r@rArrrrrrrErFrGrHrIrs rPsingle_decode_with_kv_cacherhs "3rRTcdSrTrNrs rPrr|s "),rRcLt|t|tdd|j}|jd}| d} | dt j|z } || |z} || |z} | d} | d} |jd}d}|r'tj|ftj |j }|rtj | d}td |j |j |j ||t|j| dk| dkd | d|||||| dndt"jjt&|j| dt)||j| | ddd| | |d}|r|d}ntj |}t-|j |j |j ||t|j| dk| dk||||||t)||jt&|j| | | | | |F|jd kr6|t2|z|j }n||z}|r||fS|S) a,Decode attention with KV Cache for single request, return attention output. Parameters ---------- q : torch.Tensor The query tensor, shape: ``[num_qo_heads, head_dim]``. k : torch.Tensor The key tensor, shape: ``[kv_len, num_kv_heads, head_dim]`` if :attr:`kv_layout` is ``NHD``, or ``[num_kv_heads, kv_len, head_dim]`` if :attr:`kv_layout` is ``HND``. v : torch.Tensor The value tensor, shape: ``[kv_len, num_kv_heads, head_dim]`` if :attr:`kv_layout` is ``NHD``, or ``[num_kv_heads, kv_len, head_dim]`` if :attr:`kv_layout` is ``HND``. kv_layout : str The layout of the input k/v tensors, could be either ``NHD`` or ``HND``. pos_encoding_mode : str The position encoding applied inside attention kernels, could be ``NONE``/``ROPE_LLAMA`` (LLAMA style rotary embedding) /``ALIBI``. Defaults to ``NONE``. use_tensor_cores: bool Whether to use tensor cores for the computation. Will be faster for large group size in grouped query attention. Defaults to ``False``. q_scale : Optional[float] The calibration scale of query for fp8 input, if not provided, will be set to ``1.0``. k_scale : Optional[float] The calibration scale of key for fp8 input, if not provided, will be set to ``1.0``. v_scale : Optional[float] The calibration scale of value for fp8 input, if not provided, will be set to ``1.0``. window_left : int The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. logits_soft_cap : Optional[float] The attention logits soft capping value (used in Gemini, Grok and Gemma-2, etc.), if not provided, will be set to ``0``. If greater than 0, the logits will be capped according to formula: :math:`\texttt{logits_soft_cap} \times \mathrm{tanh}(x / \texttt{logits_soft_cap})`, where :math:`x` is the input logits. sm_scale : Optional[float] The scale of softmax, if not provided, will be set to ``1 / sqrt(head_dim)``. rope_scale : Optional[float] The scale used in RoPE interpolation, if not provided, will be set to ``1.0``. rope_theta : Optional[float] The theta used in RoPE, if not provided, will be set to ``1e4``. return_lse : bool Whether to return the log sum exp value of the attention logits. Returns ------- Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If :attr:`return_lse` is ``False``, the attention output, shape: ``[qo_len, num_qo_heads, head_dim_vo]``. If :attr:`return_lse` is ``True``, a tuple of two tensors: * The attention output, shape: ``[num_qo_heads, head_dim_vo]``. * The log sum exp value, shape: ``[num_qo_heads]``. Examples -------- >>> import torch >>> import flashinfer >>> kv_len = 4096 >>> num_qo_heads = 32 >>> num_kv_heads = 32 >>> head_dim = 128 >>> q = torch.randn(num_qo_heads, head_dim).half().to("cuda:0") >>> k = torch.randn(kv_len, num_kv_heads, head_dim).half().to("cuda:0") >>> v = torch.randn(kv_len, num_kv_heads, head_dim).half().to("cuda:0") >>> o = flashinfer.single_decode_with_kv_cache(q, k, v) >>> o.shape torch.Size([32, 128]) Note ---- The ``num_qo_heads`` must be a multiple of ``num_kv_heads``. If ``num_qo_heads`` is not equal to ``num_kv_heads``, the function will use `grouped query attention `_. rrrNrM@rrfa2Fr )r'r&r*rshapemathsqrtrXrrr unsqueezerrr#rrVr" NON_CAUSALr$r)squeezer_itemsizetor[)r?r@rArrrrrrrErFrGrHrIrr;head_dim num_qo_headsrouts rPrrs@.///Y : d0ejd1eejeejejffd2e ed3e ed4e ed7e ejd8e ejd9ed:e ed!e ed?e ejd@e edeejeejejfffdAZ dGd0ejd1ejd ed2e ed3e ed4e ed!ed"e ed'e ed(e ed)e edeejejffdBZejedCZdHdDZ dS)I"BatchDecodeWithPagedKVCacheWrapperaf Wrapper class for decode attention with paged kv-cache (first proposed in `vLLM `_) for batch of requests. Check :ref:`our tutorial` for page table layout. Examples -------- >>> import torch >>> import flashinfer >>> num_layers = 32 >>> num_qo_heads = 64 >>> num_kv_heads = 8 >>> head_dim = 128 >>> max_num_pages = 128 >>> page_size = 16 >>> # allocate 128MB workspace buffer >>> workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.uint8, device="cuda:0") >>> decode_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper( ... workspace_buffer, "NHD" ... ) >>> batch_size = 7 >>> kv_page_indices = torch.arange(max_num_pages).int().to("cuda:0") >>> kv_page_indptr = torch.tensor( ... [0, 17, 29, 44, 48, 66, 100, 128], dtype=torch.int32, device="cuda:0" ... ) >>> # 1 <= kv_last_page_len <= page_size >>> kv_last_page_len = torch.tensor( ... [1, 7, 14, 4, 3, 1, 16], dtype=torch.int32, device="cuda:0" ... ) >>> kv_cache_at_layer = [ ... torch.randn( ... max_num_pages, 2, page_size, num_kv_heads, head_dim, dtype=torch.float16, device="cuda:0" ... ) for _ in range(num_layers) ... ] >>> # create auxiliary data structures for batch decode attention >>> decode_wrapper.plan( ... kv_page_indptr, ... kv_page_indices, ... kv_last_page_len, ... num_qo_heads, ... num_kv_heads, ... head_dim, ... page_size, ... pos_encoding_mode="NONE", ... data_type=torch.float16 ... ) >>> outputs = [] >>> for i in range(num_layers): ... q = torch.randn(batch_size, num_qo_heads, head_dim).half().to("cuda:0") ... kv_cache = kv_cache_at_layer[i] ... # compute batch decode attention, reuse auxiliary data structures for all layers ... o = decode_wrapper.run(q, kv_cache) ... outputs.append(o) ... >>> outputs[0].shape torch.Size([7, 64, 128]) Note ---- To accelerate computation, FlashInfer's batch decode attention creates some auxiliary data structures, these data structures can be reused across multiple batch decode attention calls (e.g. different Transformer layers). This wrapper class manages the lifecycle of these data structures. rFNautordruse_cuda_graphrpaged_kv_indptr_bufferpaged_kv_indices_bufferpaged_kv_last_page_len_bufferbackendjit_argsrJc t|| p|r9t| dt|g| R|_n$%9"05  & ,,2 1 1 1 ---rRrrfloat16Tindptrindices last_page_lenr num_kv_headsr page_sizerrErF q_data_type kv_data_type o_data_type data_typerGrHrI non_blocking block_tablesseq_lensfixed_split_sizedisable_split_kvc\ |j|jz|_t |}| d} t |dzd}|jr||jkr(td ||jt |t |j krtd|j |||j |||j dt | ||j|jko|n||j||_ ||j||_ ||j||_ ||j||_|d}|d}|| |} | |} t#| } | | } t#| } | | } t#| } ||jstd|d }| |_| |_| |_||_||_||_||_d|_|t7||}n|}|jd krO| dksJt=||_|j dt | |||jfd |D}t=|}tCj"||ftBj#|j |_|d }tI|D]-}||} |j ||| z|j|d| f<|| z }.tK| | | |j&||tN|j(| d k| d kd |_)|j)*|_+n|jr8t=||_|j, |j,|_)ny|jdkr.t[|jtN|j(dd| | |_t]|j| | | |j&||tN|j(| d k| d kd |_)|j|j/|j0||||||||j||d| g}!|jdkr?|!1||!1||!1d |j)j*|!|_+n|j, |j,|_)n9te| | | |j&||tN|j(| d k| d k |_)|j)*|j|j/|j0|||||j| | ||tCj3d | tCj3d | |_+||_4| |_5| |_6||_7||_8||_9dS)a|Plan batch decode for given problem specification. Parameters ---------- indptr : torch.Tensor The indptr of the paged kv cache, shape: ``[batch_size + 1]``, dtype: ``torch.int32`` indices : torch.Tensor The page indices of the paged kv cache, shape: ``[kv_indptr[-1]]``, dtype: ``torch.int32`` last_page_len : torch.Tensor The number of entries in the last page of each request in the paged kv cache, shape: ``[batch_size]``, dtype: ``torch.int32`` num_qo_heads : int The number of query/output heads num_kv_heads : int The number of key/value heads head_dim : int The dimension of the heads page_size : int The page size of the paged kv cache pos_encoding_mode : str The position encoding applied inside attention kernels, could be ``NONE``/``ROPE_LLAMA`` (LLAMA style rotary embedding) /``ALIBI``. Defaults to ``NONE``. window_left : int The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. logits_soft_cap : Optional[float] The attention logits soft capping value (used in Gemini, Grok and Gemma-2, etc.), if not provided, will be set to ``0``. If greater than 0, the logits will be capped according to formula: :math:`\texttt{logits_soft_cap} \times \mathrm{tanh}(x / \texttt{logits_soft_cap})`, where :math:`x` is the input logits. q_data_type : Optional[Union[str, torch.dtype]] The data type of the query tensor, defaults torch.float16. kv_data_type : Optional[Union[str, torch.dtype]] The data type of the key/value tensor. If None, will be set to ``q_data_type``. Defaults to ``None``. o_data_type : Optional[Union[str, torch.dtype]] The data type of the output tensor. If None, will be set to :attr:`q_data_type`. For FP8 inputs, this should typically be set to torch.float16 or torch.bfloat16. data_type: Optional[Union[str, torch.dtype]] The data type of both the query and key/value tensors. Defaults to torch.float16. data_type is deprecated, please use q_data_type and kv_data_type instead. non_blocking : bool Whether to copy the input tensors to the device asynchronously, defaults to ``True``. seq_lens: Optional[torch.Tensor] A uint32 1D tensor indicating the kv sequence length of each prompt. shape: ``[batch_size]``. block_tables: Optional[torch.Tensor] A uint32 2D tensor indicating the block table of each prompt. shape: ``[batch_size, max_num_blocks_per_seq]``. fixed_split_size : Optional[int], The fixed split size for FA2 split-kv decode, in pages. Only supported by tensor core decode for now. Recommend setting to the average sequence length of your workload. When enabled, will lead to deterministic softmax score reduction in the merge_states kernel, and therefore batch-size invariant outputs. See https://thinkingmachines.ai/blog/defeating-nondeterminism-in-llm-inference/ Note that compatibility with CUDA graph is NOT guaranteed, as even when bs is fixed, kv seq len can change and lead to a varied number of launched CTAs. disable_split_kv : bool, Whether to disable the split-kv for determinism in CUDA Graph, defaults to ``False``. Note ---- The :meth:`plan` method should be called before any :meth:`run` or :meth:`run_return_lse` calls, auxiliary data structures will be created during this call and cached for multiple run calls. The ``num_qo_heads`` must be a multiple of ``num_kv_heads``. If ``num_qo_heads`` is not equal to ``num_kv_heads``, the function will use `grouped query attention `_. The :meth:`plan` method cannot be used in Cuda Graph or in ``torch.compile``. Nrr rThe batch size should be fixed in cudagraph mode, the runtime batch size {} mismatches the batch size set during initialization {}HThe size of indices should be less than or equal to the allocated bufferrzAfixed_split_size is only supported by tensor core decode for now.rrc&g|] }|zdz zS)r rN).0seq_lenrs rP z;BatchDecodeWithPagedKVCacheWrapper.plan..s8"""y(1,:"""rRrrFrrr):rnumel element_size_workspace_sizerr+rrrformatrrcopy_rrrrr-r_cached_q_data_type_cached_kv_data_type_cached_o_data_type _batch_size _num_qo_heads _num_kv_heads _block_tables _max_kv_lenrrrmaxitemrrXzerosrZrangeget_trtllm_gen_decode_modulerr#r_cached_modulerr _plan_inforr.rrrappendrzr_pos_encoding_mode _window_left_logits_soft_cap _sm_scale _rope_scale _rope_theta)"rrrrrrrrrrErFrrrrrGrHrIrrrrr batch_sizeqo_indptr_host indptr_hostlast_page_len_hostkv_lens_arr_hostblocks_per_seqmax_num_blocks_per_seqblock_idinum_blocks_neededr\s" ` rPrrz'BatchDecodeWithPagedKVCacheWrapper.plan7s<@  ( . . 0 0*7799 :  ''  "!O' Q>>  % T333 NNTf"D$:OO 7||c$"<==== ^  % + +F + N N N  , 2 2L 3     &~W~ 6 < <w~'D&V, =    )/  ,)2))D %*1 ,*4**D &0=/?/? ,0@00D ,#1"3"3 ,#4##D ii&& *--e44  "' #( .{;;  &L/ ==  %K.{;;  '0E 'S   #! #. $0!#. %))5A*.  +K9KYWW  '||~~  =L ( ("c))))"#34499;;D   !83'7#8#8!8 9 ? ? | @   !)""""#3"""*-^)<)<&%*[!78);&&&" "!9z**22A(6q(9%2$x2C'CC&q*<+<*<'<=  11HH">  128q !# # #D #16688DOO  "T "#34499;;D +&*&6##=F**$? '(9:@#$ %%DM'?M L#$56<2%#a' ' '#,*5 *!D$}%% ,--- ,--- A6d16DOO+&*&6##&= L#$56<2%#a' ' '##166,*5* A[111 A\222DO$#4' /!%%rRr?paged_kv_cacherrrc ||_||_||_| |_| |_| |_||||||S)zEWarning: this function is deprecated, please use :meth:`run` instead.)rrrrrrrrrrV rr?r rrrrrErFrGrHrIs rPforwardz*BatchDecodeWithPagedKVCacheWrapper.forwardlsY#4' /!%%xx ~w   rR)rrrrrrrlrErrrrlcdSrTrN rr?r rrrrrrrlrEr\s rPrVz&BatchDecodeWithPagedKVCacheWrapper.runs srRcdSrTrNr&s rPrVz&BatchDecodeWithPagedKVCacheWrapper.runs -0CrRr ) rrrrrrrlrEsinks q_len_per_reqr(r)c  | t|j} t||j\}}|jdkr|jd}n |jd}t |||j|j|jdkr7|jdkr,| dd}| dd}|j }| |j n| } |jdkr | |j ksJ|j }|j }|j}|j}t!||d}|$|jd }d t#j|z }|||z}|||z}|d }|d }|r|Ot'j|d |dft&j|j }nJt/||d |dft&j|jd|Rt1|ddp|j}t'j|jdd |jd dz||j }n5t1|ddp|j}t/||j||jd|jdkrT||d | z| |d|d}|jr+|j|j|j||||j|j |j!|j"||tFj$j%tL|jj%| | g}|j'#|(tS| nd}d}d}tU|r+tW| dkr| d }| d}| d}|ddtY|jd|jddd|||||||d |j-||j.|j/|j0|j1||j2| gz }|j3j4|n|jdkr |jg}n|j}| Jd|j|j|j||||j |j!|j"||tL|jj%| | g}|j'#|(tS| n(|tY|jd|j||||gz }|j3j5|tm|tno|d k}|Q|sOtU|r;|8t&j|z8|j}n||z}|r||fn|S)a Compute batch decode attention between query and paged kv cache. Parameters ---------- q : torch.Tensor The query tensor, shape: ``[batch_size, num_qo_heads, head_dim]`` paged_kv_cache : Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] The paged KV-Cache stored as a tuple of tensors or a single tensor: * a tuple ``(k_cache, v_cache)`` of 4-D tensors, each with shape: ``[max_num_pages, page_size, num_kv_heads, head_dim]`` if :attr:`kv_layout` is ``NHD``, and ``[max_num_pages, num_kv_heads, page_size, head_dim]`` if :attr:`kv_layout` is ``HND``. * a single 5-D tensor with shape: ``[max_num_pages, 2, page_size, num_kv_heads, head_dim]`` if :attr:`kv_layout` is ``NHD``, and ``[max_num_pages, 2, num_kv_heads, page_size, head_dim]`` if :attr:`kv_layout` is ``HND``. Where ``paged_kv_cache[:, 0]`` is the key-cache and ``paged_kv_cache[:, 1]`` is the value-cache. *args Additional arguments for the custom kernel. q_scale : Optional[float] The calibration scale of query for fp8 input, if not provided, will be set to ``1.0``. k_scale : Optional[float] The calibration scale of key for fp8 input, if not provided, will be set to ``1.0``. v_scale : Optional[float] The calibration scale of value for fp8 input, if not provided, will be set to ``1.0``. out : Optional[torch.Tensor] The output tensor, if not provided, will be allocated internally. lse : Optional[torch.Tensor] The log-sum-exp of attention logits, if not provided, will be allocated internally. return_lse : bool Whether to return the logsumexp of attention scores, defaults to ``False``. enable_pdl : bool Whether to enable Programmatic Dependent Launch (PDL). See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#programmatic-dependent-launch-and-synchronization Only supported for >= sm90, and currently only for FA2 and CUDA core decode. q_len_per_req : int The number of query tokens per request, if not provided, will be set to ``1``. Returns ------- Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If :attr:`return_lse` is ``False``, the attention output, shape: ``[batch_size, num_qo_heads, head_dim]``. If :attr:`return_lse` is ``True``, a tuple of two tensors: * attention output, shape: ``[batch_size, num_qo_heads, head_dim]`` * logsumexp of attention scores, shape: ``[batch_size, num_qo_heads]``. Nrr rrrrMrrrrrrzplan info is not initialized)9r/rr,rrr%rrr transposerrrrrrr'rrrXrrrr(getattrrviewrrrrrrrrr"rrr$rextendlistr1rr)rrrrrrr  paged_runrV isinstancer[r)rr?r rrrrrrrlrEr(r)r\k_cachev_cacherrrFrGrHrIr out_dtyperun_args fp8_scale_q fp8_scale_k fp8_scale_v plan_info is_float_ones rPrVz&BatchDecodeWithPagedKVCacheWrapper.runs@  +AH55J1.$/RR ?e # # a(II a(I# w0$2K   =L ( (T_-E-E''B//G''B//G 3+6+>d''K =L ( ($"33333/>% %  !2333  "!O  wr{HTYx000H    H    H  J  J  {kVVAYYq *%-)!&&))QVVAYY/% ;&;TBBMagI+ w}RSS1118CC &;TBBMagI $S!'9ah N N N =L ( (qvvayyM1=!&&))QVVTUYYWWA  ^ /,*#)*0#)T_-3!H&+T ++++# " " Q<<*CIINN"&q'K"&q'K"&q'K/ AHEE#("&&&($/4 *D  )8 4 4 4} ,,1H')  O ((*H(((,*)*0T_-3H"+T ++++/ AHEE#  $D  #X . .!'511Dgn  | ~~ vvem,,w6::39EEw'0SzzS0rRc ||_||_||_| |_| |_| |_||||||dS)zPWarning: this function is deprecated, please use :meth:`run_return_lse` instead.T)rrrrr"r#s rPforward_return_lsez5BatchDecodeWithPagedKVCacheWrapper.forward_return_lses`#4' /!%%xx     rRrcdS)z7Warning: this function is deprecated and has no effect.NrNrs rP end_forwardz.BatchDecodeWithPagedKVCacheWrapper.end_forwards rR)rFFNNNrN)rrNrNNNNNNTNNNF) rNNNrNNNNrJN)!__name__ __module__ __qualname____doc__r rXrYstrrsrrrrpropertyrrrrZr[r rrr begin_forwardrr$r rrVr@ functools partialmethodrun_return_lserCrNrRrPrrQsX??B$!&9=:>@D(,A A % A A  A  A !) 6 A "*%,!7A (0 '=A A 49%A  A A A ^A F&$&&&X&$t$$$X$ &+l JO,     0"(+/9B:>9=7;$(&*&*!/3+/*.!&/p&p& p&p&| p&  p&  p&p&p&p&p&"%p&eC$456p&uS%+%567p&eC$456p&E#u{"234p& 5/!p&"UO#p&$UO%p&&'p&(u|,)p&*5<(+p&,#3--p&./p&0 1p&p&p&^p&d M "(#'#'#'+/$(&*&*  < elE%, 2L,MMN   %  %  %  "% 5/ UO UO      2 $(#'#'&*&*%*%)%)    < elE%, 2L,MMN  %  % % el # el # EN TN c]     X  $(#'#'&*&*$(%)%) 0 0 0 < 0elE%, 2L,MMN 0 % 0 % 0% 0el # 0el # 0DM 0TN 0c] 0 u|U\) * 0 0 0X 0 $(#'#'&*&* %)%)(,'(h1h1h1 <h1elE%, 2L,MMNh1 % h1 % h1%h1el #h1el #h1h1TNh1c]h1 %h1 }h1 u|U5<#=>> ?h1h1h1^h1\"(#'#'#'+/$(&*&*  <     %  %  %  "% 5/ UO UO  u|U\) *    <-Y,STBBBN      rRrcneZdZdZ d dejdejdejdejded ed d ffd ZxZ S)+CUDAGraphBatchDecodeWithPagedKVCacheWrappera|CUDAGraph-compatible Wrapper class for decode attention with paged kv-cache (first proposed in `vLLM `_) for batch of requests. Note that this wrapper may not be as efficient as :class:`BatchDecodeWithPagedKVCacheWrapper` because we won't dispatch to different kernels for different batch sizes/sequence lengths/etc to accommodate the CUDAGraph requirement. Check :ref:`our tutorial` for page table layout. Note ---- The :meth:`plan` method could not be captured by CUDAGraph. See Also -------- :class:`BatchDecodeWithPagedKVCacheWrapper` rFworkspace_buffer indptr_bufferindices_bufferlast_page_len_bufferrrrJNc Xt||d||||dS)aConstructor of :class:`BatchDecodeWithPagedKVCacheWrapper`. Parameters ---------- workspace_buffer : torch.Tensor The user reserved workspace buffer on GPU used to store auxiliary data structures, recommended size is 128MB, the device of the workspace buffer should be the same as the device of the input tensors. indptr_buffer : torch.Tensor The user reserved buffer on GPU to store the indptr of the paged kv cache, should be large enough to store the indptr of maximum batch size (``[max_batch_size + 1]``) during the lifecycle of this wrapper. indices_buffer : torch.Tensor The user reserved buffer on GPU to store the page indices of the paged kv cache, should be large enough to store the maximum number of page indices (``max_num_pages``) during the lifecycle of this wrapper. last_page_len_buffer : torch.Tensor The user reserved buffer on GPU to store the number of entries in the last page, should be large enough to store the maximum batch size (``[max_batch_size]``) during the lifecycle of this wrapper. use_tensor_cores : bool Whether to use tensor cores for the computation. Will be faster for large group size in grouped query attention. Defaults to ``False``. kv_layout : str The layout of the input k/v tensors, could be either ``NHD`` or ``HND``. T)rrrrrN)superr)rrQrRrSrTrr __class__s rPrz4CUDAGraphBatchDecodeWithPagedKVCacheWrapper.__init__sFP   -#0$2*>      rR)rF) rErFrGrHrXrYrIrsr __classcell__)rWs@rPrPrPs0!&0 0 ,0 |0  0 $l 0  0 0  0 0 0 0 0 0 0 0 0 0 rRrPceZdZdZe d.dejdededeejdeejd eejd dfd Z e d efd Z e d efd Z dejdejd dfdZ e d/dejdejdejdedededededeedeeejfdeeeejfdeedeed dfdZe d0d ejd!ejd"ejd#ejd$eed%eed&eed'eejd(eejd)ed*ed eejeejejfffd+Zejed,-ZdS)1%BatchDecodeMlaWithPagedKVCacheWrapperaWarning: this class is deprecated and will be removed in a future release. Please use :class:`flashinfer.mla.BatchMLAPagedAttentionWrapper` instead, which provides a more efficient and general MLA implementation that supports decode and incremental prefill. FNrdrrrrrrJcx||_|j|_tjdtj|j|_tjdtjdd|_|rtj|stdtj|stdtj|stdt||_ t||j d zkrtd nd |_ ||_ ||_ ||_ ||_||_d S) a'Constructor of :class:`BatchDecodeWithPagedKVCacheWrapper`. Parameters ---------- float_workspace_buffer : torch.Tensor The user reserved float workspace buffer used to store intermediate attention results in the split-k algorithm. The recommended size is 128MB, the device of the workspace buffer should be the same as the device of the input tensors. use_cuda_graph : bool Whether to enable CUDAGraph for batch decode attention, if enabled, the auxiliary data structures will be stored as the provided buffers. The ``batch_size`` cannot change during the lifecycle of this wrapper when CUDAGraph is enabled. use_tensor_cores : bool Whether to use tensor cores for the computation. Will be faster for large group size in grouped query attention. Defaults to ``False``. paged_kv_indptr_buffer : Optional[torch.Tensor] The user reserved buffer on GPU to store the indptr of the paged kv cache, the size of the buffer should be ``[batch_size + 1]``. Only needed when ``use_cuda_graph`` is ``True``. paged_kv_indices_buffer : Optional[torch.Tensor] The user reserved buffer on GPU to store the page indices of the paged kv cache, should be large enough to store the maximum number of page indices (``max_num_pages``) during the lifecycle of this wrapper. Only needed when ``use_cuda_graph`` is ``True``. paged_kv_last_page_len_buffer : Optional[torch.Tensor] The user reserved buffer on GPU to store the number of entries in the last page, the size of the buffer should be ``[batch_size]``. Only needed when ``use_cuda_graph`` is ``True``. rrTrrrrrr rrN)rrrXrrrrrrrrrrrrr)rrdrrrrrs rPrz.BatchDecodeMlaWithPagedKVCacheWrapper.__init__slX(>$,3 %*[ ek$+& & & "16 + 1 1 1 -  '?#9::  W?#:;;  X?#@AA  ^&))F%G%GD ")**d.Dq.HHH QI &'D "!1$:!%<"+H(-rRc|jSrTrrs rPrz;BatchDecodeMlaWithPagedKVCacheWrapper.is_cuda_graph_enabledTrrRc|jSrTrrs rPrz6BatchDecodeMlaWithPagedKVCacheWrapper.use_tensor_coresXrrRrec||_||_tj|jj|jjdd|_dSrrrs rPrz(>D %)0DK)@)@D &/= sm90, and currently only for FA2 and CUDA core decode. Returns ------- Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If :attr:`return_lse` is ``False``, the attention output, shape: ``[batch_size, num_qo_heads, head_dim]``. If :attr:`return_lse` is ``True``, a tuple of two tensors: * attention output, shape: ``[batch_size, num_qo_heads, head_dim]`` * logsumexp of attention scores, shape: ``[batch_size, num_qo_heads]``. Pz2MLA decode kernel is not supported on this GPU (SMz ). Supported architecture: SM80.NrrMr)rrrr rr)r6rr7rrrrrrXrr(rrrrrr rVrrrrrrtuple)rrarbrcrdrrrrrrrlmajorminor device_archrErFrGrHrIrs rPrVz)BatchDecodeMlaWithPagedKVCacheWrapper.runsCn.fm<< ubj5( "  &0[000 ' />% %  "!O    H    H  J  J ;"6&999CC $V\6<     {k[[^^V[[^^4-! )[[^^V[[^^4LM    (  & O      %  &  ,        %   ('1sCjjSE   FFFg FFF'3uSzzzSV3rRTrA)FFNNN)rNrNNN)NNNNNFF)rErFrGrHr rXrYrsrrrJrrrrZr[r rIrrrrrVrLrMrNrNrRrPrZrZsS  %!&9=:>@DP.P. % P.P. P. !) 6 P. "*%,!7 P.(0 '=P. P.P.P.^P.d$t$$$X$&$&&&X& &+l JO,     0+/-69=&*&*q&q& q&q&| q&  q& !$ q&q&q&q&"%q&ek)*q&eC$456q&UOq&UOq& q&q&q&^q&f$(#'#'&*&*  |4|4 |4l|4 |4  |4 % |4%|4%|4el #|4el #|4|4|4 u|U5<#=>> ?|4|4|4^|4|-Y,STBBBNNNrRrZceZdZddZ ddejdejdejdejd ejd ejd ed eeejfd eeejfdedede de ejde ejdejfdZ dZ dS)TrtllmGenDecodeModulerJNcd|_t|_|j|_ddlm}||jdS)Nr)r) _sm_countr_modrW_opflashinfer.jit.cubin_loaderrr|)rrs rPrzTrtllmGenDecodeModule.__init__ksb(,.00 9++--BBBBBB49557788888rRrqueryr6r7rQrr max_seq_len bmm1_scale bmm2_scaleworkspace_sizerErlrr(c| tj|} |jt|j|_t |tjr!|jtjksJ|tz}t | tjr| jtjksJt| dksJ| d}| d}| dd}|j | d|||||||||| ddd|| d|j| | |d| S)Nrr r)rXrror0rr5rYrrr rrflattenrqtrtllm_paged_attention_decode)rrsr6r7rQrrrtrurvrwrErlrr(r max_q_lens rP _paged_runz TrtllmGenDecodeModule._paged_runssJ" ;"5))C > !0>>DN j%, / / ,#u}4444#e+J j%, / / 5#u}44445::<<  A%%%%ZZ]] JJqMM  a## ..                 N    /   2 rRcdSrTrN)rr\kwargss rP_planzTrtllmGenDecodeModule._plans rRrD)rNNN) rErFrGrrXrYrZr r[rsrr}rrNrRrPrmrmjs'9999(&*(,::|:: :  , : l :,::%-.:%-.::::el #: %: !::::x     rRrmcR tdg|R}ttd|dd d2dtjdtjdt t d tjd tjd tjd tjd tjdtjdtjdtjdttjdt dt dt dtdttjdttjdttjdttjdttjdttjdtdtdttjdttjd ttjd!td"td#t d$t d%ttjd&tt d'tt d(ttjd)ttjd*tt d+tt d,ttjd-dfPfd. }td|d/ d2dtjdtjdt t d tjd tjd tjd tjd tjdtjdtjdtjdttjdt dt dt dtdttjdttjdttjdttjdttjdttjdtdtd!td"td#t d%ttjd&tt d'tt d(ttjd)ttjd*tt d+tt d,ttjd-dfHd0}tj |1S)3Nrr9 _ragged_run)rdrer<rBr=rdrerhr?rfrg qo_indptrrirjrkr<rB mask_modelayoutrErlmaybe_custom_maskmaybe_mask_indptrmaybe_alibi_slopesmaybe_prefix_len_ptrmaybe_token_pos_in_items_ptrmaybe_max_item_len_ptrrFrGscale_qscale_kscale_vrHrItoken_pos_in_items_lenrwr rrrkv_lens_bufferr max_kv_lenr(rJc'| J|J| J|!J|"J|#J|$J|%J|J|dks Jd'|||||"|#|%|d|||| |&} dS)Nrz%workspace_size must be greater than 0rM)rr()r} contiguous)(rdrerhr?rfrgrrirjrkr<rBrrrErlrrrrrrrFrGrrrrHrIrrwr rrrrrrr(r^s( rPr4z/get_trtllm_gen_decode_module..paged_runsd   )))''''''''')))$$$%%%%%%!!!#J!!!    LLNN   "           rRr}c#dSrTrN)#rdrerhr?rfrgrrirjrkr<rBrrrErlrrrrrrrFrGrHrIrr rrrrrrr(s# rP_fake_paged_runz5get_trtllm_gen_decode_module.._fake_paged_runs L rR)rrr4)NNNNNNNN) rrmr2rXrYrrZrrsr[r3rr)r\r]r4rr^s @rPr r sC  4t 4 4 4C " $ $F's''' R26&*&*/315#'$((,OB B % B #lB CyB  < B | B | B <B B  ,B !& B  <B EL)B B B B !B "$EL1#B $$EL1%B &%U\2'B ('u|4)B *'/u|&<+B ,!) 6-B ./B 01B 2%,'3B 4%,'5B 6%,'7B 89B :;B <!$=B >?B @!.AB BsmCB DsmEB Fu|,GB H!.IB JC=KB LSMMB N %OB P QB B B B B B H4S44455:26&*&*/315#'$((,G% % % % #l% Cy%  < % | % | % <% %  ,% !& %  <% EL)% % % % !% "$EL1#% $$EL1%% &%U\2'% ('u|4)% *'/u|&<+% ,!) 6-% ./% 01% 23% 45% 6!$7% 8!.9% :sm;% <sm=% >u|,?% @!.A% BC=C% DSME% F %G% H I% % % 65% V  \   rRrMHNDrrskv_cacherQrrrtrurvrr8 o_sf_scale o_sf_vec_sizer(rlrr)o_scalemaskr|cum_seq_lens_qc |t|jn|}t|tr|\}}nJ|jddkr||}}n4|jddks Jd|d\}}|dkr"t |jddkrd nd }|d kr| d ks| $t| trtd | | td ||td| | | j n|j } | tj || } td5id|d||fd|d|d|d|d|d|d|d| d| d|d|d|d|d|S|d krr|d kr,| d!d"}| d!d"}tj}t!|j}| d ks| Jt| tr4|j tjks Jd#| J| d$vs Jd%| pd&} |jdd't%|jd'dfz}t| trX| jjdt)|jd|jdz| zd(f}| j}| j}| j} | pd } n| t)|jdd)t)|jd|jdz| zd(f}tj|tj|j*}d}tj|tj|j*} ntd+| | d ksJt| tjsJt5| |tj|jdt5||tj|jd,|dks|| jdz|jdkr.td-|d.| jdd/|jdnt| tj s| | J| Jd}d}| | | j n|j } | | ntj || } | |j tjtjfvrtd0| t5| |j| |jdntd1| t|tjr!|j tjksJ|t<z}t|tjr|j tjksJ||}|d|z}n|J|ddz }|| |||||||||||| pd2| pd'|||d||| |!z| || d kr| nt| |||jStEd3|d4)6a Parameters ---------- query : torch.Tensor query tensor with shape [num_tokens, num_heads, head_dim], num_tokens = total query tokens in the batch. kv_cache : Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If kv_cache is a single tensor, it should be a tensor with shape [num_pages, 1 or 2, num_kv_heads, page_size, head_dim] if :attr:`kv_layout` is ``HND``, or [num_pages, 1 or 2, page_size, num_kv_heads, head_dim] if :attr:`kv_layout` is ``NHD``. If kv_cache is a tuple of two tensors, it should be a tuple of two tensors with shape [num_pages, num_kv_heads, page_size, head_dim] if :attr:`kv_layout` is ``HND``, or [num_pages, page_size, num_kv_heads, head_dim] if :attr:`kv_layout` is ``NHD``. The first tensor is the key cache, and the second tensor is the value cache. workspace_buffer : torch.Tensor. Must be initialized to 0 for its first use. workspace block_tables : torch.Tensor page_table of kv cache, [batch_size, num_pages] seq_lens : torch.Tensor A uint32 1D tensor indicating the kv sequence length of each prompt. shape: ``[batch_size]`` max_seq_len : int max sequence length for kv_cache bmm1_scale : Union[float, torch.Tensor] fused scale for bmm1 input. when using trtllm-gen backend, it can be a torch.Tensor with dtype torch.float32. bmm2_scale : Union[float, torch.Tensor] fused scale for bmm2 input. when using trtllm-gen backend, it can be a torch.Tensor with dtype torch.float32. window_left : int = -1 The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. out : Optional[Union[torch.Tensor, FP4Tensor]] = None output tensor, if not provided, will be allocated with ``out_dtype``, if ``out_dtype`` is not provided, will use the type of ``query``. out_dtype : Optional[Union[torch.dtype, str]] = None output dtype, if not provided, will use the type of ``out``. For nvfp4, use string ``nvfp4``. o_sf_scale : Optional[float] = None scale for nvfp4 output tensor scale factor. o_sf_vec_size : Optional[int] = None vector size for nvfp4 output tensor scale factor. sinks : Optional[List[torch.Tensor]] = None additional value per head in the denominator of the softmax. kv_layout : str = "HND" The layout of the input k/v tensors, could be either ``NHD`` or ``HND``. Defaults to ``HND``. enable_pdl : Optional[bool] = None Whether to enable Programmatic Dependent Launch (PDL). See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#programmatic-dependent-launch-and-synchronization When set to ``None``, the backend will be chosen based on the device architecture and kernel availability. backend : str = "auto" The implementation backend, could be ``auto``/``xqa`` or ``trtllm-gen``. Defaults to ``auto``. When set to ``auto``, the backend will be chosen based on the device architecture and kernel availability. For sm_100 and sm_103 (blackwell architecture), ``auto`` will choose ``trtllm-gen`` backend. For sm_90 (hopper architecture) and sm_120 (blackwell architecture), ``auto`` will choose ``xqa`` backend. o_scale : Optional[float] = 1.0 output scale factor for xqa fp8 output. mask : Optional[torch.Tensor] = None causal attention mask for xqa speculative decoding. max_q_len: Optional[int] = None The maximum query sequence length across all requests when using variable-length queries. Only supported by trtllm-gen backend. Must be provided together with ``cum_seq_lens_q``. When None, all requests use uniform query length specified by ``q_len_per_req``. cum_seq_lens_q : Optional[torch.Tensor] = None Cumulative query sequence lengths for variable-length query support, shape: ``[batch_size + 1]``, dtype: ``torch.int32``. Only supported by trtllm-gen backend. Must be provided together with ``max_q_len``. When None, all requests use uniform query length specified by ``q_len_per_req``. Returns ------- out : Union[torch.Tensor, FP4Tensor] output torch.Tensor or FP4Tensor. Nr r+EWhen kv_cache is a single tensor, the second dimension must be 1 or 2dimrrrfrrnvfp4z)xqa backend does not support nvfp4 outputz8xqa backend does not support o_sf_scale or o_sf_vec_sizez+xqa backend does not support cum_seq_lens_qrrsrrQrrrtrurvrErr(rrlr)rrrr,r-z*query must be fp8 when out_dtype is nvfp4.)Nz$only o_sf_vec_size = 16 is supportedrrryrz Invalid out: out_scale_factorzQo_sf_start_index is out of the valid range of out_scale_factor. o_sf_start_index=z, out.shape[0]=z, out_scale_factor.shape[0]=zUnsupported out_dtype: zInvalid out_dtype: gzBackend z not supportedrN)#r/rr5rhrunbindr6r!rrrXrxqa_batch_decode_with_kv_cacher/r~r{r0 float8_e4m3fnr4scaler5scale_start_indexdatarrrYr(rbfloat16rr rrrKeyError)rsrrQrrrtrurvrErr8rrr(rrlrr)rrr|rr6r7rOsm_count fp4_out_shapefp4_out_scale_shapero_sf_start_indexrs rP!trtllm_batch_decode_with_kv_cacher7s`6@5G#EL111ZJ(E"" 6# >!  ! !'WGG>!$)))W*)) (155 GW&25<@@CrIILLu %   I$5*S):T:T$5HII I  !]%>WXX X  N$>JKK K  %(_ %+I ;"5 :::C.   % w'' .- &  X  $  "z "z $   %  i "z (- G !  $ L   ''B//G''B//G-//M&u|44   I$5*S):T:T$5;%"5555<655))) J...0V...)/RM!K,R!0L0L/NNM#y)) 8IOA&U[^ek!n< MqQQ'#$'9 # h%0 U[^S11U[^ek!n< MqQQ'#$);'u/B5<$$$ $% k-u{5<XXX !6!6!6777''''c5<00 0 00 %]EKu    % ## "    !1$$#cil25E5KA5NNN M(8MMILSTMM1A1G1JMMO 5; / / @93D%%% (((#   ),CIIek ##e.>uI.V.V.VCemU^ LLL !F9!F!FGGG $S%+y%,PU V V V V>9>>?? ? j%, / / ,#u}4444#e+J j%, / / 5#u}4444  $%IA-7JJ(((',,Q//!3J              $  R       " " $ $'7'D'D'F'F F  /   8G## C3 02BEKPP 9'999:::rRc | t|jn| } t|tr|\}}nJ|jddkr||}}n4|jddks Jd|d\}}t |j}| dkr(|jd}|jd}|jd}n'|jd}|jd}|jd}|tj }|dd}|dd}||z}||z |d zz}| dkr>|jd | z}||| |jd|jd}| d}| d}| | |d nd}| tj |} | d}t||||||||||||||d kr|dznd | || d |z | | | S)ar Parameters ---------- query : torch.Tensor query tensor with shape [num_tokens, num_heads, head_dim], num_tokens = batch_size * q_len_per_request kv_cache : Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If kv_cache is a single tensor, it should be a tensor with shape [num_pages, 1 or 2, page_size, num_kv_heads, head_dim] if :attr:`kv_layout` is ``NHD``, or [num_pages, 1 or 2, num_kv_heads, page_size, head_dim] if :attr:`kv_layout` is ``HND``. If kv_cache is a tuple of two tensors, it should be a tuple of two tensors with shape [num_pages, page_size, num_kv_heads, head_dim] if :attr:`kv_layout` is ``NHD``, or [num_pages, num_kv_heads, page_size, head_dim] if :attr:`kv_layout` is ``HND``. workspace_buffer : torch.Tensor. Must be initialized to 0 for its first use. workspace block_tables : torch.Tensor page_table of kv cache, [batch_size, num_pages] seq_lens : torch.Tensor A uint32 1D tensor indicating the kv sequence length of each prompt. shape: ``[batch_size]`` max_seq_len : int max sequence length for kv_cache bmm1_scale : Union[float, torch.Tensor] fused scale for bmm1 input. bmm2_scale : Union[float, torch.Tensor] fused scale for bmm2 input. window_left : int = -1 The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. out : Optional[torch.Tensor] = None output tensor, if not provided, will be allocated with ``query.dtype``. sinks : Optional[torch.Tensor] = None additional value per head in the denominator of the softmax. kv_layout : str The layout of the kv cache. Can be either ``NHD`` or ``HND``. Defaults to ``NHD``. enable_pdl : bool Whether to enable Programmatic Dependent Launch (PDL). See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#programmatic-dependent-launch-and-synchronization Only supported for >= sm90, and currently only for FA2, CUDA core, and trtllm-gen decode. o_scale : Optional[float] = 1.0 output scale factor for fp8 output. mask : Optional[torch.Tensor] = None causal attention mask for xqa speculative decoding. Returns ------- out : torch.Tensor output torch.Tensor. Nr r+rrrr.rg?rrrM) r(rkv_scalesliding_win_sizerrrl rcp_out_scale q_seq_lenr)r/rr5rhrrr0r1rXrrreshaperr) rsrrQrrrtrurvrErr(rrlr)rrr6r7rrrr workspace_u8 semaphorescratchkv_scale_value q_scale_valuer query_new seq_lens_new sinks_newout_4ds rPrrb sjZ6@5G#EL111ZJ(E"" 6# >!  ! !'WGG>!$)))W*)) (155 GW"5<00HEM!$ }Q' =#}Q' M!$ =##((55L../I?,,-G')N/8S=AMq[^}4  :}ek!nekRSnUU""I%%a((L383D lB///$I {u%% ]]1  F,71,<,<q!Gm ). JrRrrrrrrrrrrrrrglobal_override_indptr_cpuct|}| d} | | | } | | } n| d} | | } |jrt|dzd}|d}|jrh||jkr(t d||jt|t|jkrt dn=||_||_||_ |jr!| |j | |_ tjd t| t rt#t| n| |j }tjd t| t rt#t| n| |j }||n|}tj |j 5|jr|dkr#tj|ftjd }n|}t-|||} |j|j|j|||||||||j||d | g}|jd kr?|||||d |jj||_n#t>$r}tAd||d}~wwxYw |j|j|j|j||||||j| | |||||_n%#t>$r}tAd||d}~wwxYwdddn #1swxYwY||_!| |_"| |_#||_$||_%||_&dS)a A faster version of BatchDecodeWithPagedKVCacheWrapper::plan used for FlashInferMultiStepDraftBackend. Modifications: - Remove unnecessary device-to-device copy for the cuda graph buffers. - Remove unnecessary host-to-device copy for the metadata buffers. Nrrr rrrrrrrFrzError in standard plan: )'rrr+rrrrrrrrrrrXrr5rIr0rcudaonesrrrrrrrr rrr Exception RuntimeErrorrrrrrr)rrrrrrrrrrErFrrrrGrHrIrrrrrr empty_q_dataempty_kv_cacherrrr\es rPfast_decode_planr s:]##J  #K  $L   "  "' Q>>  #!  ! / / /JJP& 6KK  w<<#d899 9 9Z  : %+!%,"+8(   "0"3"3 ,#4##D  ; +5k3+G+G XGE; ' ' '[{ L[ ,,, GE< ( ( ({N & 1 #" ZZ\\   4; ' '@J@J  ? JA~~&+ZMU&&&""&3%6%6%8%8"+K9KYWW  J0.9"$  .!$=E))KK 0111KK 0111KKNNN":$"5":# J J J"#Aa#A#ABBI J J"&"5":":0.9  .# "##" J J J"#Aa#A#ABBI J@J@J@J@J@J@J@J@J@J@J@J@J@J@J@JD0D#D+DDN!D!Ds]9AL.BJL. J7J22J77L.;AK=<L.= LLLL..L25L2) rrFNNNrNNNNF) rrFNNNrNNNNT)rMrMrNNNNNrNrr rMNNN) rMrMrNNrNr rMN) rrNNNNNNNTNFN)WrHrLrtypesrtypingrrrrrr r rX api_loggingr mlar rrrcudnnrjitrrrrrrrrrrpagerprefillrrrutilsr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7cacher_rIrurzr~rYrZrsrrr[rrrPrZrmr rrrrrNrRrPrs   !!!!!!GGGGGGGGGGGGGGGGGG '''''')(((((((WWWWWW                         8 :2:2:2z ISIcIIIIX ]]]@      |  |  |         > ??? #"###'+ $"&"&!& | | |     e_e_e_e_uo  \! & #"###'+ $"&"& $,, |, |, |, ,  ,  ,e_,e_,e_,,e_,uo,,, ,  5< %&!,,, ,& #"###'+ $"&"&}} |} |} |} }  }  }e_}e_}e_}}e_}uo}}}}  5<u|U\9: :;!}}}}@a  a  a  a  a  a  a  a  HC C C C C 2TC C C LlClClClClClClClC^ F F F F F F F F R @@@F.1-04837"&#'*.!%#$"#'#-1-f;f; <f;EL% el(B"CCDf;lf;, f; l f;  f;eU\)*f;eU\)*f;f; % i/0 1f;ek3./0f;f;C=f; D& 'f;f; !f;"#f;$C=%f;&e_'f;( 5< )f;*}+f;,U\*-f;. 5< "#/f;f;f;f;T .1-0"&$(#$"#'!RR <REL% el(B"CCDRlR, R l R  ReU\)*ReU\)*RR %, R EL !RRRC=Re_R 5< !R" \#RRRR|$'+596:37 $"&"&&*"9=+h"h" Lh"\h"< h"  h"  h"h"h"h"h"e_h"%U[ 012h"5ek!123h"c5;./0h"uoh" !h""#h"$%h"&sm'h"()h"*!) 6+h", -h"h"h"h"h"h"rR