)`ib60dZddlZddlZddlZddlmZddlmZmZm Z m Z m Z m Z m Z mZddlZddlmZddlmZmZmZmZmZmZmZmZmZddlmZdd lmZdd l m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m ddej?dej?dej?dej?de@de@de@deAdeAdejBdeAfdZCdZDeD ddeEdeEdej?dej?dej?dej?de@de@de eEde eEd e eEd!e eEd"eEd#eEde@deAdeAdeAd$eAf&d%ZFej>d&ZGej>d'ZHej>d(ZIej>d)eEd*efd+ZJed,e&jKjLd-d d.d*ed/ejMd0ejMd1ejMd2eEd3e@d4e@d5eAd6e ejMe ejMejMfffd7ZNe dd/ejMd0ejMd1ejMd:e ejMd;e ejMde ejMd?e ejMd@eAd2eEdeEdeAdAe eOd4e@dBe eOdCe eOdDe eOdeEd5e d d6ejMf*dEZPe dd/ejMd0ejMd1ejMd:e ejMd;e ejMde ejMd?e ejMd@eAd2eEdeEdeAdAe eOd4e@dBe eOdCe eOdDe eOdeEd5e dFd6e ejMejMff*dGZPe dd/ejMd0ejMd1ejMd:e ejMd;e ejMde ejMd?e ejMd@eAd2eEdeEdeAdAe eOd4e@dBe eOdCe eOdDe eOdeEd5eAd6e ejMe ejMejMfff*dHZPejQePdFIZRdJejMdKejMdLejMdMe@d6ejMf dNZSGdOdPZTdJejMdQejMd6ejMfdRZUGdSdTZVdUejMdVejMdWe@d@eAfdXZWe dd/ejMd0ejMd1ejMdUejMdVejMdYe e ejMdZe e@d[e ejMd\e ejMd@eAdAe eOd]e eOd^e eOd_e eOd`e eOd5e d d6ejMf"daZXe dd/ejMd0ejMd1ejMdUejMdVejMdYe e ejMdZe e@d[e ejMd\e ejMd@eAdAe eOd]e eOd^e eOd_e eOd`e eOd5e dFd6e ejMejMff"dbZX dd/ejMd0ejMd1ejMdUejMdVejMdYe e ejMdZe e@d[e ejMd\e ejMd@eAdAe eOd]e eOd^e eOd_e eOd`e eOd5eAd6e ejMe ejMejMfff"dcZXej>ddZYe ddeejMdfejMdgejMdhejMdiejMdje@dke@dle eOejMfdme eOejMfdneOdoe@d4e@dpejMdqejMdreAdseAd5eAdte ejMd[e ejMd\e ejMd6e ejMe ejMejMfff*duZZe ddeejMdwe ejMe ejMejMffdhejMdxejMdiejMdje@dke@dle eOejMfdme eOejMfdoe@dpejMdqejMd4e@d[e e ejMe%fdye e ej?eEfdne eOdze e@d2eEdre eAd{e e ejMd6e ejMe%ff*d|Z[e ddeejMdfejMdgejMd[ejMd}e@d~e@de@deOde eOde eOd5eAd\e ejMd6e ejMe ejMejMfffdZ\dS)a3 Copyright (c) 2023 by FlashInfer team. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. N)SimpleNamespace)AnyDictListLiteralOptionalTupleUnionoverload)flashinfer_api) gen_batch_prefill_module"gen_customize_batch_prefill_modulegen_fmha_cutlass_sm100a_modulegen_single_prefill_moduleget_batch_prefill_uriget_single_prefill_urisetup_cubin_loadergen_trtllm_gen_fmha_moduleget_trtllm_fmha_v2_module)!cudnn_batch_prefill_with_kv_cache) get_seq_lens)packbitssegment_packbits)log2e FP4TensorMaskModePosEncodingMode TensorLayout_check_cached_qkv_data_type_check_kv_layout_check_pos_encoding_modecheck_shape_dtype_device_get_cache_alibi_slopes_buf_get_cache_buf_unpack_paged_kv_cachecanonicalize_torch_dtypedetermine_attention_backenddevice_support_pdlget_device_sm_count is_float8is_sm100a_supportedis_sm110a_supportedis_sm120a_supportedis_sm121a_supportedregister_custom_opregister_fake_opceil_divround_upch|dSt|tjr|dfSdt|fS)aZSplit scale parameter into tensor and scalar components. Args: scale: Can be a torch.Tensor (per-head), a scalar float, or None. Returns: tuple: (tensor_ptr, scalar_val) where: - If scale is tensor: (scale, 1.0) - If scale is scalar: (None, scale) - If scale is None: (None, 1.0) N)N?r5) isinstancetorchTensorfloat)scales f/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/flashinfer/prefill.py_split_scale_paramr<Es= }y E5< ( ("czU5\\!!Fdtype_qdtype_kvdtype_o dtype_idx head_dim_qk head_dim_vopos_encoding_modeuse_sliding_windowuse_logits_soft_capdeviceuse_fp16_qk_reductionc t| s-t| st| st| r)t ||||||||| St d)Nz&SM100A is not supported on this device)r,r-r.r/rbuild_and_load ValueError) r>r?r@rArBrCrDrErFrGrHs r;get_fmha_modulerLYs F##C v & &C v & &C v & & C .             .   ABBBr=cvtjfdtjfd}|S)z Decorator that converts unhashable arguments (like lists) to hashable ones (tuples) before applying functools.cache. c|i|SN)argskwargsfuncs r;cached_wrapperz+make_hashable_cache..cached_wrapperstT$V$$$r=cNg}|D]O}t|tr#|t|:||Pi}|D]2\}}t|trt|||<-|||<3|i|SrO)r6listappendtupleitems)rQrR hashable_argsarghashable_kwargskeyvaluerTs r;wrapperz$make_hashable_cache..wrappers  * *C#t$$ *$$U3ZZ0000$$S)))) ,,.. - -JC%&& -',U||$$',$$~}@@@@r=) functoolscachewraps)rSr_rTs` @r;make_hashable_cacherc|sg _%%%%_%_TAAAAA$ Nr=backenduriidtypeadditional_tensor_namesadditional_tensor_dtypesadditional_scalar_namesadditional_scalar_dtypes variant_name variant_decl fp8_enabledcht|||||||||| | | | | |||||SrO)rrJ)rdrer>r?r@rfrBrCrgrhrirjrkrlrDrErFrHrms r;"get_customize_batch_prefill_moduleros\, .   '  (n)r=c't}|t| ddtjdtjdtjdtjdtjdtjdt d t d tttjfd tttjfd t d tjdtjdtdt dt dttjdttjdtjf&fd }d}d}t|||S)Nqueryk_cachev_cacheworkspace_buffer block_tablesseq_lens max_q_len max_kv_len bmm1_scale bmm2_scale batch_sizecum_seq_lens_qcum_seq_lens_kv enable_pdlworkspace_size window_leftoutsinksreturnct|j}|tj|}t |tjr!|jtjksJ|tz}t | tjr| jtjksJ |d|||||||||| ddd| || | || |||S)Nrqr) r*rGr7 empty_liker6r8dtypefloat32rtrtllm_paged_attention_context)rrrsrtrurvrwrxryrzr{r|r}r~rrrrrsm_countops r; _paged_runz1get_trtllm_gen_prefill_module.._paged_runs('u|44 ;"5))C j%, / / ,#u}4444#e+J j%, / / 5#u}4444 ))                      /   2 r=c td)Nz>Variable length is not implemented for trtllm-gen backend yet.)NotImplementedErrorrQrRs r; _ragged_runz2get_trtllm_gen_prefill_module.._ragged_runs" L   r=cdSrOrPrs r;_planz,get_trtllm_gen_prefill_module.._plan s r=) paged_run ragged_runplan)rqNN) rrJrget_library_pathr7r8intr r9boolrr)modrrrrs @r;get_trtllm_gen_prefill_modulers $ & &C    Bs++--...$&*(,%55|555 , 5 l 5 , 555%-.5%-.55 5555 !5"el ##5$ %%5& '555555n           r=c'dtg|R}tg|R}|jt d|dddt jdt jdt jdt jd t jd tt jd td td tdtt jdtt jdtdtdtt jdtt jdtt jdtdtddf&fd }td|ddt jdt jdt jdt jd t jd tt jd td td tdtt jdtt jdtdtdtdtddf d}t|S)N flashinfer::_run)tmpo maybe_lse mutates_argsqkvrrr mask_modelayoutrmaybe_packed_custom_maskmaybe_alibi_slopeslogits_soft_capsm_scalescale_qscale_kscale_v rope_scale rope_thetarcBdkrxt|\}}t|s||||||||||| | | n^t| \}}t|\}}||||||||||||| |||n|||||||||| | | | d|z d|z |S)Nfa3r5)r<r+)rrrrrrrrrrrrrrrrrrscale_v_tensorscale_v_scalarscale_q_tensorscale_q_scalarscale_k_tensorscale_k_scalarrdrun_funcs r;run_single_prefillz5get_single_prefill_module..run_single_prefills4. e  -?-H-H *NNQ<<% "#""2DG1L1L.1CG1L1L.""""""!& H("j j    "r=cdSrOrP)rrrrrrrrrrrrrrrs r;_fake_run_single_prefillz;get_single_prefill_module.._fake_run_single_prefillrs $ r=)run) rrrJrr0r7r8rrr9r1r)rdrQremodulerrrs` @r;get_single_prefill_modulers  04 0 0 0C &w 6 6 6 6 E E G GFzH s   /HN <N <N <N\ N < N EL) NNNN#+5<"8N%U\2NNN%,'N%,'N %,'!N"#N$%N& 'NNNNNNN`.S...// < <  < \  <  EL)     #+5<"8 %U\2     !   0/ ( 1 2 2 22r=cX dkr&d}t}|j}|j |j nAt g|R}t g|R}|j}|j |j td|dd ddtj dtj d ttd tj d(tj d)tj d tj d*tj d+tj d,tj dtj dttj dtdtdtdtdttj dttj dttj dttj dttj dttj dtdtdtdtdtd-td.ttd/ttd0ttj d1ttj d2ttd3ttd4ttd5ttd6ttj d7ttj d#dfNd:}t!|||;S)?N trtllm-gentrtllm_gen_contextrrfloat_workspace_bufferint_workspace_bufferrrrrr plan_info_vecrrr qo_indptr kv_indptrrrrrrrmaybe_custom_maskmaybe_mask_indptrrmaybe_prefix_len_ptrmaybe_token_pos_in_items_ptrmaybe_max_item_len_ptrrrrrtoken_pos_in_items_lenrrrrc|du}#dkr*$|||||||||| | | | | ||||||||d|z d|z |n|rVt|\}}t|\}} t|\}!}"$|||||||||| | | | | |||!||| |"n2t|\}!}"$|||||||||| | | | | ||||!|||"||S)Nfa2r5)r<)%rrrrrrrrrrrrrrrrrrrrrrrrrrrris_fp8rrrrrrrdragged_run_funcs% r;rz,get_batch_prefill_module..ragged_runsP$ e   O&$!!"$,&j j &3    67 -?-H-H *NN-?-H-H *NN-?-H-H *NN O&$+    4.@-H-H *NN O&$$,&&-   2r=cdSrOrP)rrrrrrrrrrrrrrrrrrrrrrrrrs r;_fake_ragged_runz2get_batch_prefill_module.._fake_ragged_runs 8 r=rrr paged_k_cache paged_v_cacherrrrpaged_kv_indptrpaged_kv_indicespaged_kv_last_page_lenr num_qo_heads num_kv_headsrvkv_lens_buffer page_sizerxryr|r}r~rc*0dkrl| J|J| J|!J|"J|#J|%J|&J|'J|(J|J|dks Jd1|||||!|"|$|%|d|&|'|(|||| |)} n̉0dkr=t|rJ1|||||||||| | | | | ||||||||||d|z d|z |nt|\}*}+t|s#1|||||||||| | | | | ||||||*|||+|nEt|\},}-t|\}.}/1|||||||||| | | | | |||,|.|*||-|/|+| S)Nrrz%workspace_size must be greater than 0r5)rrr) contiguousr+r<)2rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrvrrrxryr|r}r~rrrrrrrrdpaged_run_funcs2 r;rz+get_batch_prefill_module..paged_run<sn l " "$$$+++++++++!---((())))))!---"...)))!A%%%'N%%% $%AA(   || # ## N&$ &!!"$,&j j &7    <.@-H-H *NNQ<<6 *(!!!#$*(0*"#"*162DG1L1L.1CG1L1L.*(!!!#$*""""""/2r=c&dSrOrP)&rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrvrrrxryr|r}r~s& r;_fake_paged_runz1get_batch_prefill_module.._fake_paged_runs R r=rrrNNN) NNNNNNNNNNN) NNNNNNNNNN)rrrrrrrJr0r7r8rrrrr9r1r) rdrQrer plan_funcrrrrrrs ` @@r;get_batch_prefill_modulers ,".00K  +)#G3d333)'9D999HHJJK  +)'s''' F+/*.*.9vv % v#lvCyv < v < v < v<v<v <vEL)vvvvv$EL1v $EL1!v"%U\2#v$'u|4%v&'/u|&<'v(!) 6)v*+v,-v./v01v2!$3v4%,'5v6%,'7v8%,'9v: ;vvvvvvvp5S55566 %  #l Cy  <  <  <  < <  < EL)     $EL1 $EL1! "%U\2# $'u|4% &'/u|&<' (!) 6) *+ ,- ./ 01 2!$3 4 5   76 >&s&&&    V'+&*/315#'#'$($(1526(,Uee % e#leCye < e | e | e<ee ,e!& e <eEL)eeee !e"$EL1#e$$EL1%e&%U\2'e('u|4)e*'/u|&<+e,!) 6-e./e01e2%,'3e4%,'5e6%,'7e89e:;e<!$=e>?e@smAeBsmCeDu|,EeF!.GeHC=IeJC=KeLSMMeNSMOeP!.QeR"%,/SeT %UeV Weeeeee  eN4S44455<'+&*/315#'#'$($(1526M( ( % ( #l( Cy(  < ( | ( | ( <( (  ,( !& (  <( EL)( ( ( ( !( "$EL1#( $$EL1%( &%U\2'( ('u|4)( *'/u|&<+( ,!) 6-( ./( 01( 23( 45( 6!$7( 89( :sm;( <sm=( >u|,?( @!.A( BC=C( DC=E( FSMG( HSMI( J!.K( L"%,/M( N O( ( ( 65( \     r= module_name jit_modulec!|j}|j|jtd|dddtjdtjdt tdtjd tjd tjd tjd tjd tjdttjdtdtdtddffd }td|ddtjdtjdt tdtjd tjd tjd tjd tjd tjdttjdtdtdtddfd}td|dddtjdtjdt tdtjdtjdtjd tjdtjdtjdtjd tjdttjdtdtdtddf fd }td|ddtjdtjdt tdtjdtjdtjd tjdtjdtjdtjd tjdttjdtdtdtddf d}t|||S)Nrrrrrrrrrrrrrrrrrrc 2|||||||||| | | | g | RdSrOrP)rrrrrrrrrrrrrrQrs r;rz0get_batch_prefill_jit_module..ragged_run+sY2  "              r=c dSrOrP)rrrrrrrrrrrrrrQs r;rz6get_batch_prefill_jit_module.._fake_ragged_runUs " r=rrrrrrrc6|||||||||| | | | | |g|RdSrOrP)rrrrrrrrrrrrrrrrQrs r;rz/get_batch_prefill_jit_module..paged_runis_:  "       "     !      r=cdSrOrP)rrrrrrrrrrrrrrrrQs r;rz5get_batch_prefill_jit_module.._fake_paged_runs & r=r) rrrr0r7r8rrrr1r) rrrrrrrrrs @@r;get_batch_prefill_jit_moduler$sI +O)N/{///  %  #l Cy  <  <  <  < <  < EL)           B=[===>> %  #l Cy  <  <  <  < <  < EL)        ?> &.{...    # % # #l# Cy#  < # | # | # <# #  ,# !& #  <# EL)# # # # " ## # # # #   # J<[<<<== %  #l Cy  <  |  |  <   , !&   < EL)    " #   >= 0     r=NHDrq) kv_layoutrr return_lserrrrrrrrc |j} tdd| } tj|jdd|jddz|j| } d} |rItj|d|dftj| } |j|||| | | |t|j |g |R|r| | fn| S)N single_prefill_with_kv_cache_tmp)rGrqrrGrr ) rGr%r7emptyshapersizerrrr^) rrrrrrrrrQrGrrlses r;,single_prefill_with_kv_cache_with_jit_modulersXF *,>> import torch >>> import flashinfer >>> qo_len = 128 >>> kv_len = 4096 >>> num_qo_heads = 32 >>> num_kv_heads = 4 >>> head_dim = 128 >>> q = torch.randn(qo_len, num_qo_heads, head_dim).half().to("cuda:0") >>> k = torch.randn(kv_len, num_kv_heads, head_dim).half().to("cuda:0") >>> v = torch.randn(kv_len, num_kv_heads, head_dim).half().to("cuda:0") >>> o = flashinfer.single_prefill_with_kv_cache(q, k, v, causal=True, use_fp16_qk_reduction=True) >>> o.shape torch.Size([128, 32, 128]) >>> mask = torch.tril( >>> torch.full((qo_len, kv_len), True, device="cuda:0"), >>> diagonal=(kv_len - qo_len), >>> ) >>> mask tensor([[ True, True, True, ..., False, False, False], [ True, True, True, ..., False, False, False], [ True, True, True, ..., False, False, False], ..., [ True, True, True, ..., True, False, False], [ True, True, True, ..., True, True, False], [ True, True, True, ..., True, True, True]], device='cuda:0') >>> o_custom = flashinfer.single_prefill_with_kv_cache(q, k, v, custom_mask=mask) >>> torch.allclose(o, o_custom, rtol=1e-3, atol=1e-3) True Note ---- The ``num_qo_heads`` must be a multiple of ``num_kv_heads``. If ``num_qo_heads`` is not equal to ``num_kv_heads``, the function will use `grouped query attention `_. rrNr5rq@littlebitorderrr rr)r"r!r%rGmathsqrtrrrviewrCUSTOMr^CAUSAL NON_CAUSALr7rrr+rronesr(rrrrr$)rrrrrrrrrrrrDrHrrrrrrdrrrrrrs r;rrsq`.///Y ;=Mqx X XC166"::...  #5#=%  " " $ $ ) )" - -   %O)  2 -II +1I CXk166!99affQii0 ahWWW|| Sb    w!',,,,QW,,,,,,wr{agbk8888QWR[888888 ?j5=RRRG ?j5=RRRG ?j5=RRRG&- H - . 4 ! d * G G   ' +agcrclQWRSS\1 R R RC &       )*0q!  F JJ      Y%#AGAJ99%*$ ,C::,r=rrrrrc.t|t|krtdtj|}d|d<tj|dd|ddz |dd|ddz dz |z|zzd|dd<|S)Nz?The length of qo_indptr and paged_kv_indptr should be the same.rr rqlenrKr7rcumsum)rrrr mask_indptrs r;_compute_page_mask_indptrrs  9~~_---- M   "9--KKNl 1223B3 ' QRR ?3B3#7 7! ;y H$ % KO r=cH eZdZdZe dRdejdeded e ejd e ejd e ejd e ejd e ejde ejdede e e de e ee fddfdZ edefdZdejdejddfdZe dSdejdejdejdejd ed!ed"ed#ed$e ed%e ejd&e ejd'ed(ed)ed*e ed+ed,e ed-e ed.e ed/eeejfd0e eeejfd1e eeejfd2ed3e ejd4e ejd5ed6e ejd7e ejd8e ejd9e ejd:e ed;e edZeZ dTd?ejd@eejeejejffd'ed(ed)edAe edBe ed+ed,e ed*e ed-e ed.e edejfdCZeddddddddDd?ejd@eejeejejffdAe edBe edEe ejdFe ejdGeddHe ed+e edejfdIZeddddddddDd?ejd@eejeejejffdAe edBe edEe ejdFe ejdGeddHe ed+e edeejejffdJZeddddddddddK d?ejd@eejeejejffdLe eeejfdAe eeejfdBe eeejfdEe ejdFe ejdGedHe ed+e edMe ejdeejeejejfffdNZejedOZ dTd?ejd@eejeejejffd'ed(ed)edAe edBe ed+ed,e ed*e ed-e ed.e edeejejffdPZdUdQZ dS)V#BatchPrefillWithPagedKVCacheWrapperaWrapper class for prefill/append attention with paged kv-cache for batch of requests. Check :ref:`our tutorial ` for page table layout. Example ------- >>> import torch >>> import flashinfer >>> num_layers = 32 >>> num_qo_heads = 64 >>> num_kv_heads = 16 >>> head_dim = 128 >>> max_num_pages = 128 >>> page_size = 16 >>> # allocate 128MB workspace buffer >>> workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.uint8, device="cuda:0") >>> prefill_wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( ... workspace_buffer, "NHD" ... ) >>> batch_size = 7 >>> nnz_qo = 100 >>> qo_indptr = torch.tensor( ... [0, 33, 44, 55, 66, 77, 88, nnz_qo], dtype=torch.int32, device="cuda:0" ... ) >>> paged_kv_indices = torch.arange(max_num_pages).int().to("cuda:0") >>> paged_kv_indptr = torch.tensor( ... [0, 17, 29, 44, 48, 66, 100, 128], dtype=torch.int32, device="cuda:0" ... ) >>> # 1 <= paged_kv_last_page_len <= page_size >>> paged_kv_last_page_len = torch.tensor( ... [1, 7, 14, 4, 3, 1, 16], dtype=torch.int32, device="cuda:0" ... ) >>> q_at_layer = torch.randn(num_layers, nnz_qo, num_qo_heads, head_dim).half().to("cuda:0") >>> kv_cache_at_layer = torch.randn( ... num_layers, max_num_pages, 2, page_size, num_kv_heads, head_dim, dtype=torch.float16, device="cuda:0" ... ) >>> # create auxiliary data structures for batch prefill attention >>> prefill_wrapper.plan( ... qo_indptr, ... paged_kv_indptr, ... paged_kv_indices, ... paged_kv_last_page_len, ... num_qo_heads, ... num_kv_heads, ... head_dim, ... page_size, ... causal=True, ... ) >>> outputs = [] >>> for i in range(num_layers): ... q = q_at_layer[i] ... kv_cache = kv_cache_at_layer[i] ... # compute batch prefill attention, reuse auxiliary data structures ... o = prefill_wrapper.run(q, kv_cache) ... outputs.append(o) ... >>> outputs[0].shape torch.Size([100, 64, 128]) >>> >>> # below is another example of creating custom mask for batch prefill attention >>> mask_arr = [] >>> qo_len = (qo_indptr[1:] - qo_indptr[:-1]).cpu().tolist() >>> kv_len = (page_size * (paged_kv_indptr[1:] - paged_kv_indptr[:-1] - 1) + paged_kv_last_page_len).cpu().tolist() >>> for i in range(batch_size): ... mask_i = torch.tril( ... torch.full((qo_len[i], kv_len[i]), True, device="cuda:0"), ... diagonal=(kv_len[i] - qo_len[i]), ... ) ... mask_arr.append(mask_i.flatten()) ... >>> mask = torch.cat(mask_arr, dim=0) >>> prefill_wrapper.plan( ... qo_indptr, ... paged_kv_indptr, ... paged_kv_indices, ... paged_kv_last_page_len, ... num_qo_heads, ... num_kv_heads, ... head_dim, ... page_size, ... custom_mask=mask, ... ) >>> for i in range(num_layers): ... q = q_at_layer[i] ... kv_cache = kv_cache_at_layer[i] ... # compute batch prefill attention, reuse auxiliary data structures ... o_custom = prefill_wrapper.run(q, kv_cache) ... assert torch.allclose(o_custom, outputs[i], rtol=1e-3, atol=1e-3) ... Note ---- To accelerate computation, FlashInfer's batch prefill/append attention operators create some auxiliary data structures, these data structures can be reused across multiple prefill/append attention calls (e.g. different Transformer layers). This wrapper class manages the lifecycle of these data structures. rFNrrruse_cuda_graph qo_indptr_bufpaged_kv_indptr_bufpaged_kv_indices_bufpaged_kv_last_page_len_bufcustom_mask_bufmask_indptr_bufrdjit_args jit_kwargsrc 0t|| .| i} t| dt| g| Ri| |_nd|_||_| dkr|dks Jd||_|j|jz|_|j |_ tj dtj |j |_ tj dtj|j |_tj |jj|jjd d |_||_|rtj|st+d tj|st+d tj|st+dtj|st+dt-|dz |_t-||jdzkrt+dt-||jkrt+dnd|_||_||_||_||_||_| |_d|_| |_d|_ d|_!d|_"d|_#d|_$dS)am Constructor of :class:`BatchPrefillWithPagedKVCacheWrapper`. Parameters ---------- float_workspace_buffer : torch.Tensor The user reserved workspace buffer used to store intermediate attention results in split-k algorithm. The recommended size is 128MB, the device of the workspace buffer should be the same as the device of the input tensors. kv_layout : str The layout of the input k/v tensors, could be either ``NHD`` or ``HND``. use_cuda_graph : bool Whether to enable CUDA graph capture for the prefill kernels, if enabled, the auxiliary data structures will be stored in provided buffers. The ``batch_size`` cannot change during the lifecycle of this wrapper when CUDAGraph is enabled. qo_indptr_buf : Optional[torch.Tensor] The user reserved buffer to store the ``qo_indptr`` array, the size of the buffer should be ``[batch_size + 1]``. This argument is only effective when ``use_cuda_graph`` is ``True``. paged_kv_indptr_buf : Optional[torch.Tensor] The user reserved buffer to store the ``paged_kv_indptr`` array, the size of this buffer should be ``[batch_size + 1]``. This argument is only effective when ``use_cuda_graph`` is ``True``. paged_kv_indices_buf : Optional[torch.Tensor] The user reserved buffer to store the ``paged_kv_indices`` array, should be large enough to store the maximum possible size of the ``paged_kv_indices`` array during the lifetime of the wrapper. This argument is only effective when ``use_cuda_graph`` is ``True``. paged_kv_last_page_len_buf : Optional[torch.Tensor] The user reserved buffer to store the ``paged_kv_last_page_len`` array, the size of the buffer should be ``[batch_size]``. This argument is only effective when ``use_cuda_graph`` is ``True``. custom_mask_buf : Optional[torch.Tensor] The user reserved buffer to store the custom mask tensor, should be large enough to store the maximum possible size of the packed custom mask tensor during the lifetime of the wrapper. This argument is only effective when ``use_cuda_graph`` is set to ``True`` and the custom mask will be used in attention computation. mask_indptr_buf : Optional[torch.Tensor] The user reserved buffer to store the ``mask_indptr`` array, the size of the buffer should be ``[batch_size + 1]``. This argument is only effective when ``use_cuda_graph`` is ``True`` and the custom mask will be used in attention computation. backend : str The implementation backend, could be ``auto``/``fa2``/``fa3``/``cudnn`` or ``trtllm-gen``. Defaults to ``auto``. If set to ``auto``, the wrapper will automatically choose the backend based on the device architecture and kernel availability. jit_args : Optional[List[Any]] If provided, the wrapper will use the provided arguments to create the JIT module, otherwise, the wrapper will use default attention implementation. jit_kwargs : Optional[Dict[str, Any]] The keyword arguments to create the JIT module, defaults to None. Nrcudnnrz&CUDNN backend only supports NHD layout)iricpuTrrG pin_memoryz9qo_indptr_buf should be a torch.Tensor in CUDA graph modez?paged_kv_indptr_buf should be a torch.Tensor in CUDA graph modez@paged_kv_indices_buf should be a torch.Tensor in CUDA graph modezFpaged_kv_last_page_len_buf should be a torch.Tensor in CUDA graph moder z;The length of paged_kv_indptr_buf should be batch_size + 1.z>The length of paged_kv_last_page_len_buf should be batch_size.)%r!rro _jit_module _kv_layout_float_workspace_buffernumel element_size_workspace_sizerGr7rint32_kv_lens_bufferuint8_int_workspace_bufferrr _pin_memory_int_workspace_buffer_use_cuda_graph is_tensorrKr_fixed_batch_size_qo_indptr_buf_paged_kv_indptr_buf_paged_kv_indices_buf_paged_kv_last_page_len_buf_custom_mask_buf_mask_indptr_buf_max_total_num_rows_backend _plan_info_cached_module _seq_lens_kv _seq_lens_q _block_tables) selfrrrrrrr r!r"rdr#r$s r;__init__z,BatchPrefillWithPagedKVCacheWrapper.__init__s^ ###  ! ; 27TXTTTTT  D   $D # g  %%%'O%%%'=$  ( . . 0 0*7799 : -3 ${ EK    &+[ ek$+& & & "16  & ,,2 1 1 1 - .  '?=11  O?#677  U?#788  V?#=>>  \&)%7%7!%;D "&''4+AA+EEE Q-..$2HHH TI &'D "+$7!%9"+E( / /26  " !r=c|jSrOr6rFs r;is_cuda_graph_enabledz9BatchPrefillWithPagedKVCacheWrapper.is_cuda_graph_enabled ##r=rc||_||_tj|jj|jjdd|_dSaReset the workspace buffer. Parameters ---------- float_workspace_buffer : torch.Tensor The new float workspace buffer, the device of the new float workspace buffer should be the same as the device of the input tensors. int_workspace_buffer : torch.Tensor The new int workspace buffer, the device of the new int workspace buffer should be the same as the device of the input tensors. r(Tr)Nr-r4r7rrrr5rFrrs r;reset_workspace_bufferz:BatchPrefillWithPagedKVCacheWrapper.reset_workspace_buffer"K(>$%9"05  & ,,2 1 1 1 ---r=rrqfloat16TrrrrrrrrBrrCrrrrDrHrrrrr q_data_type kv_data_type o_data_type non_blockingprefix_len_ptrtoken_pos_in_items_ptrrmax_item_len_ptrrw seq_lens_qrvmax_token_per_sequencemax_sequence_kvfixed_split_sizedisable_split_kvc#tt|}||}t|}||}t|}|d}| |} |!d}!t|dz }#|#|_||_||_| | t |||}$| <| :t | d|$d\} }$||_ ||_ ||_ ||_ |||_ nc|d}%t|%dd|%ddz |_ t#|%d}&| | |_n|d}'|d}(|t'|'|(})n&|})|jdt|)|)|t|)|_|jr|j|&|_n3|&|jkr(t5d |&|j|#|jkr(t5d |#|jt|t|jkrt5d |j|||j|||j |||jdt|||j!|j!ko|| tEj#|j$st5d tEj#|j%st5d |j$dt| | | j!|j!ko||j%|$|n||j!||_||j!||_||j!||_||j!||_ | C| |j!||_$|$|j!||_%nd|_$d|_%||_&||_'||_(|j) |j)|_*n|j+dkr5tY|j!tZ| j.||j$du|||_+|j+dkr@||||j/|| tZ| j.|dk|dk|f }*ta|j+g|*R|_*||_1|j+dkr| sted|dksJ|j1fd|)D}+t|+},tEj3|#|,ftDj|j!|_1|'d}-ti|#D]9}.|+|.}/|j1 Jd||-|-|/z|j1|.d|/f<|-|/z }-:|j*|j5|j6|j7|%|'|)|jp|&|#|||j|| | |g}0|j+dkrA|08|!pd|08|"|08d|j*j9|0|_:| |_;| |_<||_=||_>||_?||_@||_A||_B||_C||n||_DdS)aPlan batch prefill/append attention on Paged KV-Cache for given problem specification. Parameters ---------- qo_indptr : torch.Tensor The indptr of the query/output tensor, shape: ``[batch_size + 1]``. paged_kv_indptr : torch.Tensor The indptr of the paged kv-cache, shape: ``[batch_size + 1]``. paged_kv_indices : torch.Tensor The page indices of the paged kv-cache, shape: ``[paged_kv_indptr[-1]]``. paged_kv_last_page_len : torch.Tensor The number of entries in the last page of each request in the paged kv-cache, shape: ``[batch_size]``. num_qo_heads : int The number of query/output heads. num_kv_heads : int The number of key/value heads. head_dim_qk : int The dimension of the query/key heads. page_size : int The size of each page in the paged kv-cache. head_dim_vo : Optional[int] The dimension of the value/output heads, if not provided, will be set to ``head_dim_qk``. custom_mask : Optional[torch.Tensor] The flattened boolean mask tensor, shape: ``(sum(q_len[i] * k_len[i] for i in range(batch_size))``. The elements in the mask tensor should be either ``True`` or ``False``, where ``False`` means the corresponding element in the attention matrix will be masked out. Please refer to the :ref:`mask layout ` for more details about flattened layout of mask tensor. When :attr:`custom_mask` is provided, and :attr:`packed_custom_mask` is not, the function will pack the custom mask tensor into a 1D packed mask tensor, which introduces additional overhead. packed_custom_mask : Optional[torch.Tensor] The 1D packed uint8 mask tensor, if provided, the :attr:`custom_mask` will be ignored. The packed mask tensor is generated by :func:`flashinfer.quantization.packbits`. causal : bool Whether to apply causal mask to the attention matrix. This is only effective when :attr:`custom_mask` is not provided in :meth:`plan`. pos_encoding_mode : str The position encoding applied inside attention kernels, could be ``NONE``/``ROPE_LLAMA`` (LLAMA style rotary embedding) /``ALIBI``. Default is ``NONE``. use_fp16_qk_reduction : bool Whether to use f16 for qk reduction (faster at the cost of slight precision loss). window_left : int The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. logits_soft_cap : Optional[float] The attention logits soft capping value (used in Gemini, Grok and Gemma-2, etc.), if not provided, will be set to ``0``. If greater than 0, the logits will be capped according to formula: :math:`\texttt{logits_soft_cap} \times \mathrm{tanh}(x / \texttt{logits_soft_cap})`, where :math:`x` is the input logits. sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim)``. rope_scale : Optional[float] The scale used in RoPE interpolation, if not provided, will be set to ``1.0``. rope_theta : Optional[float] The theta used in RoPE, if not provided, will be set to ``1e4``. q_data_type : Union[str, torch.dtype] The data type of the query tensor, defaults torch.float16. kv_data_type : Optional[Union[str, torch.dtype]] The data type of the key/value tensor. If None, will be set to :attr:`q_data_type`. o_data_type : Optional[Union[str, torch.dtype]] The data type of the output tensor. If None, will be set to :attr:`q_data_type`. For FP8 inputs, this should typically be set to torch.float16 or torch.bfloat16. non_blocking : bool Whether to copy the input tensors to the device asynchronously, defaults to ``True``. prefix_len_ptr :Optional[torch.Tensor] prefix length. A uint32 1D tensor indicating the prefix length of each prompt. The tensor size is equal to the batch size. token_pos_in_items_ptr : Optional[torch.Tensor] A uint16 1D tensor (it will be converted to uint16 in flashinfer) indicating the token position of each item and started from 0 (delimiter) for each item. E.g., if we have 3 items of length 3, 2, 4 respectively for this member. This vector will be looking like `[0, 1, 2, 3, 0, 1, 2, 0, 1, 2, 3, 4, 0]` with 4 delimiters indexed as 0. For batch size > 1, we will concat them as 1D with zero paddings to make sure each has the same length, the padding length is defined by `token_pos_in_items_len` - length of the raw `token_pos_in_items_ptr` for each prompt. token_pos_in_items_len : int zero padding length for `token_pos_in_items_ptr` to better handle the bsz > 1 case. Still using the above 3,2,4 example. If we set `token_pos_in_items_len` to be 20, it will be `[0, 1, 2, 3, 0, 1, 2, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0]` with 7 padded zeros. (note there're 8 zeros in the end where the first one is the delimiter token 0 in the end of the prompt) max_item_len_ptr : Optional[torch.Tensor] a uint16 vector contains the max token length of all items for each prompt seq_lens: Optional[torch.Tensor] A uint32 1D tensor indicating the kv sequence length of each prompt. shape: ``[batch_size]``. seq_lens_q: Optional[torch.Tensor] A uint32 1D tensor indicating the q sequence length of each prompt. shape: ``[batch_size]``. If not provided, will be set to the same value as ``seq_lens``. block_tables: Optional[torch.Tensor] A uint32 2D tensor indicating the block table of each prompt. shape: ``[batch_size, max_num_blocks_per_seq]``. max_token_per_sequence: Optional[int], Required for cudnn backend. This is the scalar max token length of each sequence. max_sequence_kv: Optional[int], Required for cudnn backend. This is the scalar max sequence length of each sequence in kv cache. fixed_split_size : Optional[int], The fixed split size for FA2 split-kv prefill/decode in pages. Recommend setting to the average sequence length of your workload. When enabled, will lead to deterministic softmax score reduction in the merge_states kernel, and therefore batch-size invariant outputs. See https://thinkingmachines.ai/blog/defeating-nondeterminism-in-llm-inference/ Note that compatibility with CUDA graph is NOT guaranteed, as even when bs is fixed, kv seq len can change and lead to a varied number of launched CTAs. disable_split_kv : bool, Whether to disable the split-kv for determinism in CUDA Graph, defaults to ``False``. Note ---- The :meth:`plan` method should be called before any :meth:`run` or :meth:`run_return_lse` calls, auxiliary data structures will be created during this call and cached for multiple kernel runs. The ``num_qo_heads`` must be a multiple of ``num_kv_heads``. If ``num_qo_heads`` is not equal to ``num_kv_heads``, the function will use `grouped query attention `_. The :meth:`plan` method cannot be used in Cuda Graph or in ``torch.compile``. Nrrqr r r r(rWzThe total number of rows in qo_indptr {} in cuda graph mode cannot exceed the number of rows set during initialization {}.zThe batch size should be fixed during the lifecycle of the wrapper in cuda graph mode, the runtime batch size {} mismatches the batch size {} set during initialization.zAThe length of paged_kv_indices exceeds the allocated buffer size.zcustom_mask_buf must be initialized with a torch.Tensor in cuda graph mode if we use custom mask in attention computation.zzmask_indptr_buf must be initialized with a torch.Tensor in cuda graph mode if we use custom mask in attention computation.rr&rrzNon-causal attention is not supported for trtllm-gen backend with paged KV cache. Please use causal=True or choose a different backend (e.g., fa2, fa3, cudnn).c&g|] }|zdz zS)r rP).0seq_lenrs r; z.s8"""y(1,:"""r=rzblock_tables is not initializedr)Er'r _batch_size _num_qo_heads _num_kv_headsrrrr_prefix_len_ptr_token_pos_in_items_ptr_token_pos_in_items_len_max_item_len_ptr _max_q_lentomaxitemr _max_kv_lenrr(flattenr2copy_rKr?rKformatr8r;r9r:r<rGr7r7r=r>_cached_q_data_type_cached_kv_data_type_cached_o_data_typer+rBr@r(rr^rrrErzerosranger-r4r5rWrrA_causal_pos_encoding_mode_use_fp16_qk_reduction _window_left_logits_soft_cap _sm_scale _rope_scale _rope_thetarCrD)1rFrrrrrrrBrrCrrrrDrHrrrrrrTrUrVrWrXrYrrZrwr[rvr\r]r^r_r|rqo_indptr_hosttotal_num_rowspaged_kv_indptr_hostpaged_kv_last_page_len_hostkv_lens_arr_hostget_module_argsblocks_per_seqmax_num_blocks_per_seqblock_idinum_blocks_neededrQs1 ` r;rz(BatchPrefillWithPagedKVCacheWrapper.plan:s~/{;;  &L/ ==  %K.{;;  "!O  %K  #! ^^a' %))  "&8&D3& K  %+*A.>&&((--b11!/// +  .'=$'=$!1 " -4DOO&\\%00N!."4~crc7J"JKKPPRRDO !344N  &.D  #2#5#5e#<#< *@*C*CE*J*J '#/(*Ey$$  $,<<>>#9#9#;#;  !83'7#8#8!8 9 ? ? | @    ##34499;;D   %F -'/+9(($"::: NNTf&(@OOT333 228&"D$:33#$$s4+E'F'FFF W   % %il % K K K  % + +O, + W W W  , 2 2&\ 3     &'>-=)>)>'> ? E E .5DV, F    "-t'<==$Ut'<==$U%&?,>(?(?&?@FF&"4";t{"J"%$G %++Kl+SSS"+,,t{,"V"VD (7(:(: ,);))D %*:)<)< ,*=**D &0F/H/H ,0I00D ,"-(:(=(=Kl)>))%)4Kl)7))%%)-%(,%#. $0!#.   '"&"2D  }&& ;K#$56<))5 !! }'' #)#$56<1$#a') #'?M'$3'''#* =L ( ( )d#c))))!)""""#3"""*-^)<)<&%*[!78);&&&" 02z**22A(6q(9%-999:99AQ 8.?#??AD&q*<+<*<'<= 11HH   *,*5$ (:N*!D$}%% ,2333 ,--- A6d16DO "3&;#' /!%%$)3)?::Xr=rpaged_kv_cachek_scalev_scalec ||_||_||_||_| |_| |_| |_| |_|||||S)EWarning: This function is deprecated, please use :meth:`run` instead.rr r|r}r~rrrrrr rFrrrrDrHrrrrrrrs r;forwardz+BatchPrefillWithPagedKVCacheWrapper.forwards\  "3&;#' /!%%xx>7GxLLLr=)rrrrrrrrrrrcdSrOrP rFrrrrrrrrrrQs r;rz'BatchPrefillWithPagedKVCacheWrapper.runs sr=cdSrOrPrs r;rz'BatchPrefillWithPagedKVCacheWrapper.runs -0Cr=) q_scalerrrrrrrrrrc  | t|j} t||j\} }t || |j|j|j}|&|j|krtd|jd|d|jdkr| j d}n | j d}| |j n| } |j dkr | |j ksJ|j }|j}|j}|j}|d }|*d t#j|d z }|j d kr|||z}|||z}|d }|d }|r|Ot)j|d|dft(j|j}nJt/||d|dft(j|jd|Rt1|ddp|j}t)j|j dd |j d dz||j}nMt1|ddp|j}t/||j dd |j d dz||jd|j dkr7|jdkr,| dd} |dd}|jt8jj}n*|jrt8j j}nt8j!j}|j"t8j#j}|j d kr|j$D|j$%dkr'|j$&|j'ddd|_$|j(D|j(%dkr'|j(&|j'ddd|_(tS|| ||j|j*f|j$|j(|j+|j,|j-|j|||||j.|j.|||dn|j dkr|j/ Jd|j*|j0|j/|| ||j.|j1|j2|j3|||th|jj| | g}|j5#|6to| nd}d}d}tq|r+ts| dkr| d}| d}| d}||j|j:tw|j d|j|j"|j<|j=||||||||j>|j?|j@|jA|j-|jB||j+|j,|j'|j.|j1| gz }|jC Jd|jCjD|t|to|d k}|Q|sOtq|r;|Gt(j|zG|j}n||z}|r||fn|S)a Compute batch prefill/append attention between query and paged kv-cache. Parameters ---------- q : torch.Tensor The query tensor, shape: ``[qo_indptr[-1], num_qo_heads, head_dim]`` paged_kv_cache : Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] The paged KV-Cache stored as a tuple of tensors or a single tensor: * a tuple ``(k_cache, v_cache)`` of 4-D tensors, each with shape: ``[max_num_pages, page_size, num_kv_heads, head_dim]`` if :attr:`kv_layout` is ``NHD``, and ``[max_num_pages, num_kv_heads, page_size, head_dim]`` if :attr:`kv_layout` is ``HND``. * a single 5-D tensor with shape: ``[max_num_pages, 2, page_size, num_kv_heads, head_dim]`` if :attr:`kv_layout` is ``NHD``, and ``[max_num_pages, 2, num_kv_heads, page_size, head_dim]`` if :attr:`kv_layout` is ``HND``. Where ``paged_kv_cache[:, 0]`` is the key-cache and ``paged_kv_cache[:, 1]`` is the value-cache. *args Additional arguments for custom kernels. q_scale : Optional[Union[float, torch.Tensor]] The calibration scale of query for fp8 input, if not provided, will be set to ``1.0``. k_scale : Optional[Union[float, torch.Tensor]] The calibration scale of key for fp8 input, if not provided, will be set to ``1.0``. v_scale : Optional[Union[float, torch.Tensor]] The calibration scale of value for fp8 input, if not provided, will be set to ``1.0``. out : Optional[torch.Tensor] The output tensor, if not provided, will be allocated internally. lse : Optional[torch.Tensor] The log-sum-exp of attention logits, if not provided, will be allocated internally. return_lse : bool Whether to return the logsumexp of attention output enable_pdl : bool Whether to enable Programmatic Dependent Launch (PDL). See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#programmatic-dependent-launch-and-synchronization Only supported for >= sm90, and currently only for FA2 and CUDA core decode. Returns ------- Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If :attr:`return_lse` is ``False``, the attention output, shape: ``[qo_indptr[-1], num_qo_heads, head_dim]``. If :attr:`return_lse` is ``True``, a tuple of two tensors: * The attention output, shape: ``[qo_indptr[-1], num_qo_heads, head_dim]``. * The logsumexp of attention output, shape: ``[qo_indptr[-1], num_qo_heads]``. NzThe dtype of out z does not match the o_data_type z specified in plan function.rr rrr5rqr&rrrrryr)actual_seq_lens_qactual_seq_lens_kvr\r]rvrrrrrbatch_offsets_qbatch_offsets_orrrVzplan info is not initialized cached module is not initialized)Hr)rGr&r,r rwrxryrrKrrr@rrrrr r rr7rrr#getattrrz transposer=rrr^r|rrrkMULTIITEMSCORINGrDdimreshaperhrCrr-rorsrEr9rAr4r:r;r<rr+extendrVr+rr>r$rlrnrmr0rirjr2rBrr6r9rp)rFrrrrrrrrrrrrQrsrtrrrrrr out_dtyperrun_args fp8_scale_q fp8_scale_k fp8_scale_v is_float_ones r;rz'BatchPrefillWithPagedKVCacheWrapper.runs|  +AH55J1.$/RR# w0$2K   * ?syG33tCIttwttt  ?e # # a(II a(I+6+>d''K =L ( ($"33333/>% %  "!O  TYqvvbzz222H =G # #"G#"G#  J  J  {kVVAYYq *%-)!&&))QVVAYY/% ;&;TBBMagI+ w}RSS1118CC &;TBBMagI $QWSbS\GM"##$66 18U    =L ( (T_-E-E''B//G''B//G  , -II| 6$O1 $/5   + 17I =G # #+0@0D0D0F0F!0K0K#'#3#;#;DNPQSTVW$X$X! -,   #'"2#'#4'+ $ 0!/|% $ 3 $ 3%)     .} ,,224R222,*#)*0T_-3!H$+T ++++# " " Q<<*CIINN"&q'K"&q'K"&q'K))/ AHEE(0*#0(&&&(O$$'-5:&224V222 )D  )8 4 4%gu55H'S.L"<"S>>#66%-007:>>syIICC7NC'0SzzS0r=rc ||_||_||_||_| |_| |_| |_| |_|||||S)PWarning: This function is deprecated, please use :meth:`run_return_lse` instead.r r|r}r~rrrrrrun_return_lsers r;forward_return_lsez6BatchPrefillWithPagedKVCacheWrapper.forward_return_lses_  "3&;#' /!%%""1ngw"WWWr=cdSz7Warning: this function is deprecated and has no effect.NrPrJs r; end_forwardz/BatchPrefillWithPagedKVCacheWrapper.end_forward  r=) rFNNNNNNrNN)NNNFrFNrqNNNrSNNTNNrNNNNNNNF) FrFNNrqNNNNrN)!__name__ __module__ __qualname____doc__r r7r8strrrrrrrGpropertyrKrQrr9r rr begin_forwardr rr rrr` partialmethodrrrrPr=r;rrs ccJ$046:7;=A2626(,/3\"\" % \"\" \"   - \" &el3 \"'u|4\"%-U\$:\""%,/\""%,/\"\"49%\"T#s(^,\" \"\"\"^\"|$t$$$X$ &+l JO,     0&*.259!'&+$(+/&*&*/8:>9=!159=&'37+/-1/304)-*.!&GININ<ININ , IN !& IN  ININININc]INel+IN%U\2INININ $IN 5/!IN"#IN$"%%IN&UO'IN(UO)IN*3 +,+IN,uS%+%567-IN.eC$456/IN01IN2!.3IN4!) 65IN6!$7IN8#5<09IN:5<(;IN<U\*=IN>u|,?IN@!) AINB"#CIND#3-EINFGINH IINININ^INV M !'&+#'#'+/$(&*&*MM <MelE%, 2L,MMNM M  M $ M%M%MM"%M5/MUOMUOM MMMM4 $(#'&*&*%*%)%)    < elE%, 2L,MMN  %  % el # el # EN TN c]     X  $(#'&*&*$(%)%) 0 0 0 < 0elE%, 2L,MMN 0 % 0 % 0el # 0el # 0DM 0TN 0c] 0 u|U\) * 0 0 0X 0 9=8<8<&*&* %)%)(,o1o1o1 <o1elE%, 2L,MMNo1 %u| 345 o1 %u| 345 o1%u| 345o1el #o1el #o1o1TNo1c]o1 %o1 u|U5<#=>> ?o1o1o1^o1b-Y,STBBBN !'&+#'#'+/$(&*&*XX <XelE%, 2L,MMNX X  X $ X%X%XX"%X5/XUOXUOX u|U\) *XXXX4      r=rrct|t|krtdtj|}d|d<tj|dd|ddz |dd|ddz zd|dd<|S)Nz9The length of qo_indptr and kv_indptr should be the same.rr rqr)rrrs r;_compute_mask_indptrr s 9~~Y''TUUU"9--KKNl 1223B3 'IabbMIcrcN,JK KO r=c8>eZdZdZe dHdejdeded e ejd e ejd e ejd e ejd ede e e de e ee fddfdZ edefdZdejddfdZe dIdejdejdededede ede ejd e ejd!ed"ed#ed$ed%e ed&e ed'e ed(e ed)eeejfd*e eeejfd+e eeejfd,ed-e ejd.e ejd/ed0e ejd1e ed2eddf6d3ZeZ dJd4ejd5ejd6ejd!ed"ed#ed$ed%e ed&e ed'e ed(e edejfd7Zeddddd8d4ejd5ejd6ejd9e ejd:e ejd;eddZeddddddddd?d4ejd5ejd6ejd@e edAe edBe edCe ed9e ejd:e ejd;ed` for ragged kv-cache layout. Example ------- >>> import torch >>> import flashinfer >>> num_layers = 32 >>> num_qo_heads = 64 >>> num_kv_heads = 16 >>> head_dim = 128 >>> # allocate 128MB workspace buffer >>> workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda:0") >>> prefill_wrapper = flashinfer.BatchPrefillWithRaggedKVCacheWrapper( ... workspace_buffer, "NHD" ... ) >>> batch_size = 7 >>> nnz_kv = 100 >>> nnz_qo = 100 >>> qo_indptr = torch.tensor( ... [0, 33, 44, 55, 66, 77, 88, nnz_qo], dtype=torch.int32, device="cuda:0" ... ) >>> kv_indptr = qo_indptr.clone() >>> q_at_layer = torch.randn(num_layers, nnz_qo, num_qo_heads, head_dim).half().to("cuda:0") >>> k_at_layer = torch.randn(num_layers, nnz_kv, num_kv_heads, head_dim).half().to("cuda:0") >>> v_at_layer = torch.randn(num_layers, nnz_kv, num_kv_heads, head_dim).half().to("cuda:0") >>> # create auxiliary data structures for batch prefill attention >>> prefill_wrapper.plan( ... qo_indptr, ... kv_indptr, ... num_qo_heads, ... num_kv_heads, ... head_dim, ... causal=True, ... ) >>> outputs = [] >>> for i in range(num_layers): ... q = q_at_layer[i] ... k = k_at_layer[i] ... v = v_at_layer[i] ... # compute batch prefill attention, reuse auxiliary data structures ... o = prefill_wrapper.run(q, k, v) ... outputs.append(o) ... >>> outputs[0].shape torch.Size([100, 64, 128]) >>> >>> # below is another example of creating custom mask for batch prefill attention >>> mask_arr = [] >>> qo_len = (qo_indptr[1:] - qo_indptr[:-1]).cpu().tolist() >>> kv_len = (kv_indptr[1:] - kv_indptr[:-1]).cpu().tolist() >>> for i in range(batch_size): ... mask_i = torch.tril( ... torch.full((qo_len[i], kv_len[i]), True, device="cuda:0"), ... diagonal=(kv_len[i] - qo_len[i]), ... ) ... mask_arr.append(mask_i.flatten()) ... >>> mask = torch.cat(mask_arr, dim=0) >>> prefill_wrapper.plan( ... qo_indptr, ... kv_indptr, ... num_qo_heads, ... num_kv_heads, ... head_dim, ... custom_mask=mask ... ) >>> outputs_custom_mask = [] >>> for i in range(num_layers): ... q = q_at_layer[i] ... k = k_at_layer[i] ... v = v_at_layer[i] ... # compute batch prefill attention, reuse auxiliary data structures ... o_custom = prefill_wrapper.run(q, k, v) ... assert torch.allclose(o_custom, outputs[i], rtol=1e-3, atol=1e-3) ... >>> outputs_custom_mask[0].shape torch.Size([100, 64, 128]) Note ---- To accelerate computation, FlashInfer's batch prefill/append attention operators create some auxiliary data structures, these data structures can be reused across multiple prefill/append attention calls (e.g. different Transformer layers). This wrapper class manages the lifecycle of these data structures. rFNrrrrr kv_indptr_bufr!r"rdr#r$rc Lt|| .| i} t| dt|g| Ri| |_nd|_||_||_|j|_tjdtj |j|_ tj|j j tj dd|_ ||_ |rtj|stdtj|std t!|d z |_t!||jd zkr5td t!||j||_||_||_||_d|_||_d|_dS) ac Constructor of :class:`BatchPrefillWithRaggedKVCacheWrapper`. Parameters ---------- float_workspace_buffer : torch.Tensor The user reserved float workspace buffer used to store intermediate attention results in the split-k algorithm. The recommended size is 128MB, the device of the workspace buffer should be the same as the device of the input tensors. kv_layout : str The layout of the input k/v tensors, could be either ``NHD`` or ``HND``. use_cuda_graph : bool Whether to enable CUDA graph capture for the prefill kernels, if enabled, the auxiliary data structures will be stored as the provided buffers. qo_indptr_buf : Optional[torch.Tensor] The user reserved GPU buffer to store the ``qo_indptr`` array, the size of the buffer should be ``[batch_size + 1]``. This argument is only effective when ``use_cuda_graph`` is ``True``. kv_indptr_buf : Optional[torch.Tensor] The user reserved GPU buffer to store the ``kv_indptr`` array, the size of the buffer should be ``[batch_size + 1]``. This argument is only effective when ``use_cuda_graph`` is ``True``. custom_mask_buf : Optional[torch.Tensor] The user reserved GPU buffer to store the custom mask tensor, should be large enough to store the maximum possible size of the packed custom mask tensor during the lifetime of the wrapper. This argument is only effective when ``use_cuda_graph`` is ``True`` and custom mask will be used in attention computation. mask_indptr_buf : Optional[torch.Tensor] The user reserved GPU buffer to store the ``mask_indptr`` array, the size of the buffer should be ``[batch_size]``. This argument is only effective when ``use_cuda_graph`` is ``True`` and custom mask will be used in attention computation. backend : str The implementation backend, could be ``auto``/``fa2``/``fa3`` or ``cutlass``. Defaults to ``auto``. If set to ``auto``, the wrapper will automatically choose the backend based on the device architecture and kernel availability. jit_args : Optional[List[Any]] If provided, the wrapper will use the provided arguments to create the JIT module, otherwise, the wrapper will use default attention implementation. jit_kwargs : Optional[Dict[str, Any]] The keyword arguments to create the JIT module, defaults to None. Nrr'rTr()rr*rGz9qo_indptr_buf should be a torch.Tensor in cuda graph modez9kv_indptr_buf should be a torch.Tensor in cuda graph moder zJThe length of kv_indptr_buf ({}) should be the same as qo_indptr_buf ({}).)r!rror+r,r-rGr7rr3r4rr5r6r7rKrr8rvr9_kv_indptr_bufr=r>r?r@rB) rFrrrrrr!r"rdr#r$s r;rGz-BatchPrefillWithRaggedKVCacheWrapper.__init__| sB ###  ! ; 27TXTTTTT  D   $D #'=$,3 %*[ ek$+& & & "16  & ,+ 1 1 1 - .  ?=11  O?=11  O&)%7%7!%;D "=!!T%;a%??? `ggM**D,B,+ / /26  "r=c|jSrOrIrJs r;rKz:BatchPrefillWithRaggedKVCacheWrapper.is_cuda_graph_enabled rLr=c||_||_tj|jj|jjdd|_dSrNrOrPs r;rQz;BatchPrefillWithRaggedKVCacheWrapper.reset_workspace_buffer rRr=rrqrSTrrrrrrBrCrrrrDrHrrrrrrTrUrVrWrXrYrrZr^r_c< t|}||}t|}||}t|}||}|d}| d} t|dz }t||dzkrtd||t||}|<|:t |d|d\}}|d}|d}t|d}|j r5|j ||_ n3||j kr(td ||j ||j kr(td ||j |j || |j|| |t!j|jstd t!j|jstd ||jdt|<|j|| n||j| |_ ||j| |_|B||j| |_||j| |_||_||_||_|dd|ddz } ||_||_||_||_|j |j|_n|jdkr5t?|jt@| j!| |jdu|||_||||j"||t@| j!| dk| dk| f }!|jdkr-|!dd|jfz|!ddz}"tG|"|_ntI|jg|!R|_|jdkr\tK|j|||| |_&t!j'|dd|ddz (|_)n|j Jd|j*|j+|j,||| |j p||||d|j ||| | g}#|jdkrA|#-|pd|#-||#-d|jj.|#|_&| |_/| |_0| |_1| |_2| |_3||_4||_5||_6dS)aPlan batch prefill/append attention on Ragged KV-Cache for given problem specification. Parameters ---------- qo_indptr : torch.Tensor The indptr of the query/output tensor, shape: ``[batch_size + 1]``. kv_indptr : torch.Tensor The indptr of the key/value tensor, shape: ``[batch_size + 1]``. num_qo_heads : int The number of query/output heads. num_kv_heads : int The number of key/value heads. head_dim_qk : int The dimension of the heads on query/key tensor. head_dim_vo : Optional[int] The dimension of the heads on value/output tensor. If not provided, will be set to ``head_dim_qk``. custom_mask : Optional[torch.Tensor] The flattened boolean mask tensor, shape: ``(sum(q_len[i] * k_len[i] for i in range(batch_size))``. The elements in the mask tensor should be either ``True`` or ``False``, where ``False`` means the corresponding element in the attention matrix will be masked out. Please refer to the :ref:`mask layout ` for more details about flattened layout of mask tensor. When :attr:`custom_mask` is provided, and :attr:`packed_custom_mask` is not, the function will pack the custom mask tensor into a 1D packed mask tensor, which introduces additional overhead. packed_custom_mask : Optional[torch.Tensor] The 1D packed uint8 mask tensor, if provided, the :attr:`custom_mask` will be ignored. The packed mask tensor is generated by :func:`flashinfer.quantization.packbits`. If provided, the custom mask will be added to the attention matrix before softmax and after scaling. The mask tensor should be in the same device as the input tensors. causal : bool Whether to apply causal mask to the attention matrix. This argument is ignored if ``mask`` is provided in :meth:`plan`. pos_encoding_mode : str The position encoding applied inside attention kernels, could be ``NONE``/``ROPE_LLAMA`` (LLAMA style rotary embedding) /``ALIBI``. Default is ``NONE``. use_fp16_qk_reduction : bool Whether to use f16 for qk reduction (faster at the cost of slight precision loss). window_left : int The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. logits_soft_cap : Optional[float] The attention logits soft capping value (used in Gemini, Grok and Gemma-2, etc.), if not provided, will be set to ``0``. If greater than 0, the logits will be capped according to formula: :math:`\texttt{logits_soft_cap} \times \mathrm{tanh}(x / \texttt{logits_soft_cap})`, where :math:`x` is the input logits. sm_scale : Optional[float] The scale used in softmax, if not provided, will be set to ``1.0 / sqrt(head_dim_qk)``. rope_scale : Optional[float] The scale used in RoPE interpolation, if not provided, will be set to ``1.0``. rope_theta : Optional[float] The theta used in RoPE, if not provided, will be set to ``1e4``. q_data_type : Union[str, torch.dtype] The data type of the query tensor, defaults to torch.float16. kv_data_type : Optional[Union[str, torch.dtype]] The data type of the key/value tensor. If None, will be set to :attr:`q_data_type`. o_data_type : Optional[Union[str, torch.dtype]] The data type of the output tensor. If None, will be set to :attr:`q_data_type`. For FP8 inputs, this should typically be set to torch.float16 or torch.bfloat16. non_blocking : bool Whether to copy the input tensors to the device asynchronously, defaults to ``True``. prefix_len_ptr :Optional[torch.Tensor] prefix length. A uint32 1D tensor indicating the prefix length of each prompt. The tensor size is equal to the batch size. token_pos_in_items_ptr : Optional[torch.Tensor] A uint16 1D tensor (it will be converted to uint16 in flashinfer) indicating the token position of each item and started from 0 (delimiter) for each item. E.g., if we have 3 items of length 3, 2, 4 respectively for this member. This vector will be looking like `[0, 1, 2, 3, 0, 1, 2, 0, 1, 2, 3, 4, 0]` with 4 delimiters indexed as 0. For batch size > 1, we will concat them as 1D with zero paddings to make sure each has the same length, the padding length is defined by `token_pos_in_items_len` - length of the raw `token_pos_in_items_ptr` for each prompt. token_pos_in_items_len : int zero padding length for `token_pos_in_items_ptr` to better handle the bsz > 1 case. Still using the above 3,2,4 example. If we set `token_pos_in_items_len` to be 20, it will be `[0, 1, 2, 3, 0, 1, 2, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0]` with 7 padded zeros. (note there're 8 zeros in the end where the first one is the delimiter token 0 in the end of the prompt) max_item_len_ptr : Optional[torch.Tensor] a uint16 vector contains the max token length of all items for each prompt fixed_split_size : Optional[int], The fixed split size for split-kv FA2 prefill/decode, in pages. Recommend setting to the average sequence length of your workload. When enabled, will lead to deterministic softmax score reduction in the merge_states kernel, and therefore batch-size invariant outputs. See https://thinkingmachines.ai/blog/defeating-nondeterminism-in-llm-inference/ Note that compatibility with CUDA graph is NOT guaranteed, as even when bs is fixed, kv seq len can change and lead to a varied number of launched CTAs. disable_split_kv : bool, Whether to disable the split-kv for determinism in CUDA Graph, defaults to ``False``. Note ---- The :meth:`plan` method should be called before any :meth:`run` or :meth:`run_return_lse` calls, auxiliary data structures will be created during this plan call and cached for multiple kernel runs. The ``num_qo_heads`` must be a multiple of ``num_kv_heads``. If ``num_qo_heads`` is not equal to ``num_kv_heads``, the function will use `grouped query attention `_. The :meth:`plan` method cannot be used in Cuda Graph or in ``torch.compile``. Nrqrr z;The kv_indptr length should be equal to mask_indptr length.r r r(rbzThe batch size should be fixed in cudagraph mode, the runtime batch size {} mismatches the batch size set during initialization {}.rarcz~mask_indptr_buf must be initialized with a torch.Tensor in cuda graph mode if we use custom mask in the attention computation.rrcutlass rr)7r'rrKrrrrrprrKr?rvr8r9rurr7r7r=r>rGrwrxryrkrlrmrnr+rBr@r(rr^rrLrfmha_varlen_planrArqrr _max_qo_lenr-r4r5rWrr|r}r~rrrrr)$rFrrrrrBrCrrrrDrHrrrrrrTrUrVrWrXrYrrZr^r_r|rrkv_indptr_hostr kv_len_arrrnew_get_module_argsrQs$ r;rz)BatchPrefillWithRaggedKVCacheWrapper.plan sN/{;;  &L/ ==  %K.{;;  %K  #!   "!O^^a' y>>Z!^ + +M   "&8&D.y)DDK  %+*A.>&&((--b11!/// +  #e,,"e,,^B/00  %( '/+9(($"::: NNTf&(@OOT333 OOUv"D$:PP   % %il % K K K   % %il % K K K!-t'<==$Ut'<==$YDV%&?,>(?(?&?@%++Kl+SSS"+,,t{,"V"VD "+,,t{,"V"VD !-(:(=(=Kl)>))%)4Kl)7))%$/ $0!#. #ABB'."*== -'=$'=$!1   '"&"2D  }&& ;K#$56<))5 !!  128q !#% O} ))$BQB'9+;*==PQPRPR@SS$'67J&K##&>M'$3'''# =I % %.#Y <DO %y1223B3)GHHMMOOD  &224V222,*5(:N*!D$}%% ,2333 ,--- A6d16DO "3&;#' /!%%r=rrrc ||_||_||_||_||_| |_| |_| |_||||S)rr rFrrrrrDrHrrrrrs r;rz,BatchPrefillWithRaggedKVCacheWrapper.forward: sX "3&;#' /!%%xx1a   r=)rrrrrrrrcdSrOrP rFrrrrrrrrQs r;rz(BatchPrefillWithRaggedKVCacheWrapper.runS s sr=cdSrOrPrs r;rz(BatchPrefillWithRaggedKVCacheWrapper.run` s -0Cr=)rrro_scalerrrrrrrrc| t|j} t|||j|j|j} |j}|j}|j}|j }|d}|*dtj | dz }|d}|d}| r| Otj| d| dftj|j} nJt!| | d| dftj|jd |]|jjdkr tjn|j}tj|jdd|jddz||j}n:t!||jdd|jddz|j|jd |jd krBt/||||j|j|j|j||||||j|| \}} | r|| fn|St;|r||jd krqt=jd| tj!}| tj!}| tj!}|j"tFj$j%}n*|jrtFj&j%}ntFj'j%}|j(|j)|j||||j|j|| |tT|j+j%| | g}|j,#|-t]| n}||j"|j/ta|jd|j|j1|j2|j3|||||j4g z }t;|r"|-t]| |j5 Jd|j5j6|| r|| fn|S)aOCompute batch prefill/append attention between query and kv-cache stored as ragged tensor. Parameters ---------- q : torch.Tensor The query tensor, shape: ``[qo_indptr[-1], num_qo_heads, head_dim_qk]`` k : torch.Tensor The key tensor, shape: ``[kv_indptr[-1], num_kv_heads, head_dim_qk]`` v : torch.Tensor The value tensor, shape: ``[kv_indptr[-1], num_kv_heads, head_dim_vo]`` *args Additional arguments for the custom kernel. q_scale: Optional[float] The calibration scale of fp8 query, if not provided, will be set to ``1.0``. k_scale: Optional[float] The calibration scale of fp8 key, if not provided, will be set to ``1.0``. v_scale: Optional[float] The calibration scale of fp8 value, if not provided, will be set to ``1.0``. o_scale: Optional[float] The calibration scale of output, if not provided, will be set to ``1.0``. out : Optional[torch.Tensor] The output tensor, if not provided, will be allocated internally. lse : Optional[torch.Tensor] The log-sum-exp of attention logits, if not provided, will be allocated internally. return_lse : bool Whether to return the logsumexp of attention output enable_pdl : bool Whether to enable Programmatic Dependent Launch (PDL). See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#programmatic-dependent-launch-and-synchronization Only supported for >= sm90, and currently only for FA2 and CUDA core decode. Returns ------- Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If :attr:`return_lse` is ``False``, the attention output, shape: ``[qo_indptr[-1], num_qo_heads, head_dim_vo]``. If :attr:`return_lse` is ``True``, a tuple of two tensors: * The attention output, shape: ``[qo_indptr[-1], num_qo_heads, head_dim_vo]``. * The logsumexp of attention output, shape: ``[qo_indptr[-1], num_qo_heads]``. Nrr5rqrrr rrrr) plan_inforrrrrr max_qo_lenrrrzOur current prefill kernel implementation needs f16 input, the f8 inputs are casted to f16, which could result in performance degradation.r)7r)rGr rwrxrrrrrr r rr7rrr#ritemsizebfloat16rryr@ fmha_varlenr9rrAr|rr+loggingwarningrprSr=rrr^rrr-r4rr,r+rrVr>r$rkrlrnrmrBr)rFrrrrrrrrrrrrQrrrrrrrrs r;rz(BatchPrefillWithRaggedKVCacheWrapper.runm sn  +AH55J# q$*D,E   ' />% %  "!O  TYqvvbzz222H  J  J  {kVVAYYq *%-)!&&))QVVAYY/% ;*+'*:a*?*?QWI+ qwrss|+xCC % qwrss|+(     =I % %"##/|!+HC"",4C:: 4 Q<< $DMU22 OU   U]##AU]##AU]##A  , -II| 6$O1 $/5   (  & O         ) /      ' OODJJ ' ' ' ' %%+AGAJ DD$,&,  H|| ,T +++"..0R...&&11'0SzzS0r=rc ||_||_||_||_||_| |_| |_| |_||||S)rrrs r;rz7BatchPrefillWithRaggedKVCacheWrapper.forward_return_lse! sZ "3&;#' /!%%""1a+++r=cdSrrPrJs r;rz0BatchPrefillWithRaggedKVCacheWrapper.end_forward: rr=) rFNNNNrNN)NNNFrFrqNNNNrSNNTNNrNNF)FrFrqNNNNr)!rrrrr r7r8rrrrrrrGrrKrQrr9r rrrrr rrr r`rrrrrPr=r;rr! sXXt$04042626(,/3q#q# % q#q# q#   - q#   - q#"%,/q#"%,/q#q#49%q#T#s(^,q# q#q#q#^q#f$t$$$X$ &+l     0&*.259!'&++/$(&*&*/8:>9=!159=&'37*.!&7i&i&<i&<i& i&  i&  i&c]i&el+i&%U\2i&i&i& $i&i&"%i&5/i& UO!i&"UO#i&$3 +,%i&&uS%+%567'i&(eC$456)i&*+i&,!.-i&.!) 6/i&0!$1i&2#5<03i&4#3-5i&67i&8 9i&i&i&^i&V M!'&++/$(&*&*!! <! <! < !  !  ! $!!"%!5/!UO!UO! !!!!2'+&*%*%)    <  <  <  el # el # EN TN     X '+&*$(%) 0 0 0 < 0 < 0 < 0 el # 0el # 0DM 0TN 0 u|U\) * 0 0 0X 0$(#'#'#'&*&* %)o1o1o1 <o1 <o1 < o1 % o1%o1%o1%o1el #o1el #o1o1TNo1 u|U5<#=>> ?o1o1o1^o1b-Y,STBBBN!'&++/$(&*&*,, <, <, < ,  ,  , $,,"%,5/,UO,UO, u|U\) *,,,,2      r=rqo_segment_offsetskv_segment_offsetsrc tj|jj}tj|dz|jtj}tjd|jtj}tjd|jtj}tjd|jtj} ||||||| d||| |||| fS)Nr rGri)r7cudaget_device_propertiesrGmulti_processor_countrr1r) rrrrrnum_ctas work_indptrqo_tile_indices head_indices batch_indicess r;rr? s z//! +1 /6ekKk)0 O;)0 LK)0 M KK       r=rrrrrrrrcdSrOrPrrrrrrrrrrrrrrrrs r;rri s $3r=cdSrOrPrs r;rr~ s $),r=ctdd|j}t|j|j|jtj|jd|jdtjj dd|j }|j\}}}|j\}}}| rdnd}| dtj |z } | d} | d} | d} |d}|}|9t j |dd|ddz  }|t||||| }|\}}}}|h|jjdkr tjn|j}t j|t|d z|||j| t|d d}|)|r't j|||jtj }||||||||||||||| | | | ||||fS) Nfmha_varlen_cutlass_workspacerrFr rr5rqr)r%rGrLrr7r1rrrr^r r rqrrrrrrrr)rrrrrrrrrrrrrrrrrurnnz_qorrBnnz_kvrrCmask_mode_code qo_total_lenrrrrrs r;rr sK$&')918        "     F)*%FL+()%FL+ 'QQaN;///LY1!""58J3B38OOPPUUWW $ &(:L&     {&'g&6!&;&;ENN k 3z3// /  8    j#   " {z{k ,qxu}    JJ     ', 8Or=ct}|}t||SrO)rrJrr)rrs r;get_trtllm_gen_fmha_moduler s= $ & &C    Bs++--... Ir=rrr]r^rurwrxryrzr{ o_sf_scaler|r}r~r is_causalattention_sinksc||jddkr"|jddkr|jddks Jd|t|j}tj}t |j}|Dt j|jd|jd|jd|j|j}|r?|=t j|jd|jd|jt j }t|t j r!|jt j ksJ|tz}t|t j r|jt j ksJ| |z}|||||||||||| | | | | |||||||r||fS|S) a Parameters ---------- query : torch.Tensor query tensor with shape [num_tokens, num_heads, head_dim] key : torch.Tensor key tensor with shape [num_tokens, num_heads, head_dim] value : torch.Tensor value tensor with shape [num_tokens, num_heads, head_dim] workspace_buffer : torch.Tensor workspace buffer seq_lens : torch.Tensor sequence lengths max_q_len : int max query length max_kv_len : int max key/value length bmm1_scale : Union[float, torch.Tensor] scale for bmm1, scale_q * scale_k * 1.0 / (head_dim_qk ** 0.5) when using trtllm-gen backend, it can be a torch.Tensor with dtype torch.float32. bmm2_scale : Union[float, torch.Tensor] scale for bmm2, scale_v when using trtllm-gen backend, it can be a torch.Tensor with dtype torch.float32. o_sf_scale : float scale for output batch_size : int batch size window_left : int window left cum_seq_lens_q : torch.Tensor cumulative sequence lengths for query cum_seq_lens_kv : torch.Tensor cumulative sequence lengths for key/value enable_pdl : bool enable pdl is_causal : bool is causal attention_sinks : Optional[torch.Tensor] attention sinks out : Optional[torch.Tensor] output tensor, if not provided, will be allocated with shape [query.shape[0], query.shape[1], value.shape[2]] lse : Optional[torch.Tensor] lse tensor, if not provided, will be allocated with shape [query.shape[0], query.shape[1]] Returns ------- out: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] output torch.Tensor or Tuple[torch.Tensor, torch.Tensor]. If return_lse is True, the output will be a tuple of two tensors, the first is the output tensor, the second is the lse tensor. If return_lse is False, the output will be a single tensor. rr:currently only support deepseek r1 192 query and 128 valueNrr r)rr)rGrtrtllm_ragged_attentionr*r7rrrr6r8rr.r/)rrr]r^rurwrxryrzr{rr|rr}r~rrrrrrrrrs r; trtllm_ragged_attention_deepseekr sT ;q>S SYq\S%8%8U[^s=R=R=RD>S=R R' 55 )++CH"5<00H {k KN KN KN<+     ckk KN KN<-    *el++(5=0000%' *el++15=0000%++--0@0M0M0O0OON H     +.Cx r=HNDkv_cachervr o_sf_vec_sizercn |t|j}t|tr|\}}nJ|jddkr||}}n4|jddks Jd|d\}}|dkr,|dd}|dd}tj}t|j}|d ks|Jt| tr4|j tj ks Jd |J|d vs Jd |pd }|jddt|jddfz}t| trX| jjdt!|jd|jdz|zdf}| j}| j}| j} |pd }n| t!|jddt!|jd|jdz|zdf}tj|tj |j}d}tj|tj|j} nt+d| |d ksJt| tjsJt/| |tj|jdt/||tj |jd|dks|| jdz|jdkr.t+d|d| jdd|jdnt|tj s||J|Jd}d}|| | j n|j }| | ntj||} ||j tjtjfvrt+d|t/| |j||jdnt+d|t|tjr!|j tjksJ|t8z}t|tjr|j tjksJ||z}|| ||||||||||||pd|pd|| | | | |||||d kr| nt| |||jS)a Parameters ---------- query : torch.Tensor query tensor with shape [num_tokens, num_heads, head_dim] kv_cache : Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] If kv_cache is a single tensor, it should be a tensor with shape [num_pages, 1 or 2, num_kv_heads, page_size, head_dim] if :attr:`kv_layout` is "HND", or [num_pages, 1 or 2, page_size, num_kv_heads, head_dim] if :attr:`kv_layout` is "NHD". If kv_cache is a tuple of two tensors, it should be a tuple of two tensors with shape [num_pages, num_kv_heads, page_size, head_dim] if :attr:`kv_layout` is "HND", or [num_pages, page_size, num_kv_heads, head_dim] if :attr:`kv_layout` is "NHD". The first tensor is the key cache, the second tensor is the value cache. workspace_buffer : torch.Tensor. Must be initialized to 0 for its first use. workspace block_tables : torch.Tensor page_table of kv cache, [batch_size, num_pages] seq_lens : torch.Tensor A uint32 1D tensor indicating the kv sequence length of each prompt. shape: ``[batch_size]`` max_q_len : int max sequence length for query max_kv_len : int max sequence length for kv_cache bmm1_scale : Union[float, torch.Tensor] fused scale for bmm1 input. when using trtllm-gen backend, it can be a torch.Tensor with dtype torch.float32. bmm2_scale : Union[float, torch.Tensor] fused scale for bmm2 input. when using trtllm-gen backend, it can be a torch.Tensor with dtype torch.float32. batch_size : int batch size cum_seq_lens_q : torch.Tensor cumulative sequence length for query. shape: ``[batch_size + 1]`` cum_seq_lens_kv : torch.Tensor cumulative sequence length for kv_cache. shape: ``[batch_size + 1]`` window_left : int = -1 The left (inclusive) window size for the attention window, when set to ``-1``, the window size will be set to the full length of the sequence. Defaults to ``-1``. out : Optional[Union[torch.Tensor, FP4Tensor]] = None output tensor, if not provided, will be allocated with ``out_dtype``, if ``out_dtype`` is not provided, will use the type of ``query``. out_dtype : Optional[Union[torch.dtype, str]] = None output dtype, if not provided, will use the type of ``out``. For nvfp4, use string ``nvfp4``. o_sf_scale : Optional[float] = None scale for nvfp4 output tensor scale factor. o_sf_vec_size : Optional[int] = None vector size for nvfp4 output tensor scale factor. enable_pdl : Optional[bool] = None Whether to enable Programmatic Dependent Launch (PDL). See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#programmatic-dependent-launch-and-synchronization Defaults to ``None``, which means it will be enabled if the device supports PDL. kv_layout : str = "HND" Layout of kv-cache, can be "HND" or "NHD", default is "HND". sinks : Optional[List[torch.Tensor]] = None additional value per head in the denominator of the softmax. Returns ------- out: Union[torch.Tensor, FP4Tensor] output torch.Tensor or FP4Tensor. Nr rzEWhen kv_cache is a single tensor, the second dimension must be 1 or 2)rrrrnvfp4z*query must be fp8 when out_dtype is nvfp4.)Nz$only o_sf_vec_size = 16 is supportedr rqrrrz Invalid out: rout_scale_factorzQo_sf_start_index is out of the valid range of out_scale_factor. o_sf_start_index=z, out.shape[0]=z, out_scale_factor.shape[0]=)rzUnsupported out_dtype: zInvalid out_dtype: g)r)rGr6rXrunbindrrrr*rrr7 float8_e4m3fnr2r:r3scale_start_indexdatarr3rKr8r#rrSrrrr.r/)rrrrurvrwrxryrzr{r|r}r~rrrrrrrrrsrtrr fp4_out_shapefp4_out_scale_shaper o_sf_start_indexrs r;"trtllm_batch_context_with_kv_cacher skb' 55 (E"" 6# >!  ! !'WGG>!$)))W*)) (155 GWE##B++##B++)++JH"5<00HG 1ji6P6P 1{e1111 8211%%% ***,R***%+  CRC(HU[_a,H,H+JJ c9 % % 4 "Q%+a.8MI1MM#  #y "4 (C!,WII [Q--Q%+a.8MI1MM#  %{#5+>u|     ! +m5;u|TTTCC2S2233 3G#####u|,,,,, !mU[%,PUVVV     L      q #)A,.1A1G1JJJI$4IIEHYq\II-=-CA-FII K Iu{ + +  L Lr= num_headshead_dimrf scale_softmax scale_bmm1 scale_bmm2c t|jstd|jddkr"|jddkr|jddks Jdt } |jt jk} |jt jk}||n| rdnd}||nd}| | nd} | ||||| |||||| | | | r|| fS|S) a Parameters ---------- query : torch.Tensor query tensor with shape [batch_size, seq_len, num_heads, head_dim] key : torch.Tensor key tensor with shape [batch_size, seq_len, num_heads, head_dim] value : torch.Tensor value tensor with shape [batch_size, seq_len, num_heads, head_dim] out : torch.Tensor output tensor with shape [batch_size, seq_len, num_heads, head_dim] return_lse : bool whether to return the log-sum-exp of attention output num_heads : int number of heads head_dim : int head dimension seq_len : int sequence length scale_softmax : float scale for softmax scale_bmm1 : Optional[float] scale for bmm1 scale_bmm2 : Optional[float] scale for bmm2 lse : Optional[torch.Tensor] log-sum-exp of attention output Returns ------- out: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]] output torch.Tensor or Tuple[torch.Tensor, torch.Tensor]. If return_lse is True, the output will be a tuple of two tensors, the first is the output tensor, the second is the lse tensor. If return_lse is False, the output will be a single tensor. z9fmha_v2_prefill_deepseek is only supported on SM120 GPUs.rrrrNr5r) r.rGrKrrrr7rrr)rrr]r^rrrrfrrrrrris_e4m3is_bf16_outputs r;fmha_v2_prefill_deepseekrds!b u| , ,VTUUU ;q>S SYq\S%8%8U[^s=R=R=RD>S=R R' ( (FkU00GY%.0N&2 w8OC *53J)53J JJ     Cx r=)F)rFFFF)NNNNNNFrrFNrqNNNrF)NNNNNNFrrFNrqNNNrT) NNNNFNNNNNF) NNNNFNNNNNTr)rqNNNNrNN)NNFN)]rr`rr typesrtypingrrrrrr r r r7 api_loggingr jitrrrrrrrrrr&rpager quantizationrrutilsrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r<rarrrrGrLrcrrorrrrrr^r8rr9rpartial'single_prefill_with_kv_cache_return_lserrrrrrrrrrrPr=r;r(s0  !!!!!!MMMMMMMMMMMMMMMMMMMM ''''''                      54444444444444:"""( #(CC [CkC[C{ C  C  CCCC LC CCCCD@ $ %"'')) ) )[)k ) [ ) K )))"#Y)#3i)"#Y)#3i)))) !)"#)$ %)&'))))X JJJZ o3o3o3d VVVr  QcQsQQQQh(.)))) |) |) | )  )))) 5<u|U\9: :;))))D '+&*&*%)*.15#"' $'+"&"&!&) | | |el #  el #  el # ek "%,'!.  uo e_!"#$%&'()* \+ 0 '+&*&*%)*.15#"' $'+"&"& $),, |, |, |,el # , el # , el # ,ek ",%,',!., ,,, ,uo,, e_!,"#,$%,&',( ),* 5< %&+,,, ,0 '+&*&*%)*.15#"' $'+"&"&)k-k- |k- |k- |k-el # k- el # k- el # k-ek "k-%,'k-!.k- k-k-k- k-uok-k- e_!k-"#k-$%k-&'k-()k-* 5<u|U\9: :;+k-k-k-k-\+<)*; T+++' |\"L   \ .w w w w w w w w t | (-   \    [  [  [  [  [  [  [  [  |' ' ' '  ''''T /3 $"&"& $####!&! | | |    U\*+   %,  %,  uoe_e_e_e_ !" \# ( /3 $"&"& $#### $!,, |, |, |, ,  , U\*+ , , %, , %, , ,uo,e_,e_,e_,e_,  !," 5< %&#,,, ,4/3 $"&"& $####!gg |g |g |g g  g U\*+ g g %, g %, g guoge_ge_ge_ge_g !g" 5<u|U\9: :;#ggggT &/3"&"&)CC <C C <Cl C l C  CCeU\)*CeU\)*CCCCLC\CC !C"#C$el+%C& %, 'C( %, )C* 5<u|U\9: :;+CCCCL4837"&#'!%*.)TT <TEL% el(B"CCDTlT, T l T  TTeU\)*TeU\)*TTLT\TT % i/0 1Tek3./0T !T"C=#T$%T&'T( D& ')T* 5< "#+TTTTn#'"&"&OO <O O <O  O  O  OOOOOO %, O 5<u|U\9: :;OOOOOOr=