.`ij (dZddlZddlmZddlmZddlmZddlZddl Zddl m cm Z ddlmZmZmZmZmZmZmZddlmZddlmZdd lmZdd lmZdd lmZdd l m!Z!dd l"m#Z#ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*ddl+m,Z,ee-Z.dej/j0_1ej2eddZ3ej2edZ4dej5dej5fdZ6dej5de7de7fdZ8Gdde%Z9dej5dej5d e7d!e7dej5f d"Z:d#dd#d$dej5d%e7de7d&e7d'e7dej5f d(Z;d)ej5d*ej5d+ej5d,ej5fd-ZGd2d3e&Z?d4e@deAeBe7e@zffd5ZCdS)6z#Attention layer with FlexAttention.N) dataclass)cached_property)ClassVar) BlockMask_mask_mod_signature_score_mod_signature and_maskscreate_block_maskflex_attentionor_masks) VllmConfig) CacheDType) init_logger)vllm_is_batch_invariant)current_platform)cdiv)is_torch_equal_or_newer)AttentionBackend AttentionImplAttentionMetadataBuilder AttentionTypeCommonAttentionMetadatais_quantized_kv_cache) AttentionSpecTzreduce-overhead) fullgraphmode)roffsetsreturnc|j}|dd|ddz }tjtjt ||tj|S)Ndevicedtype)r$torchrepeat_interleavearangelenint32)rr$countss }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/attention/backends/flex_attention.py_offsets_to_doc_ids_tensorr-3sV ^F QRR[73B3< 'F  " S[[u{CCCV  xmultipledimc>||j||zz |z}|dkr|S|dkr|n |j|z}g}t|jdz |dz dD]7}||kr|d|g |ddg8t j||ddS)Nrr!r"constant)rvalue)shapendimrangeextendFpad)r/r0r1 differencepad_listis r,pad_to_multipler>;sagclX56(BJQ##qv|CH 16A:sQw + +$$ 88 OOQ O , , , , OOQF # # # # 5H:Q 7 7 77r.ceZdZUdZeed<ejejej gZ e e ej ed<ddgZe e eed<edefdZed edefd Zedefd Zeded fd Ze ddedededededeedff dZededfdZedefdZede efdZdS)FlexAttentionBackendTaccept_output_buffersupported_dtypesautobfloat16supported_kv_cache_dtypesrcdS)NFLEX_ATTENTIONrHr.r,get_namezFlexAttentionBackend.get_nameUsr. attn_typec6|tjtjfvS)z?FlexAttention supports both decoder and encoder-only attention.)rDECODER ENCODER_ONLY)clsrJs r,supports_attn_typez'FlexAttentionBackend.supports_attn_typeYs]2M4NOOOr.cdS)z7FlexAttention supports full attention for image tokens.TrHrNs r,supports_mm_prefixz'FlexAttentionBackend.supports_mm_prefix^s tr.FlexAttentionImplctSN)rSrHr.r, get_impl_clsz!FlexAttentionBackend.get_impl_clscs  r. num_blocks block_size num_kv_heads head_sizecache_dtype_str.cd||||fS)NrH)rWrXrYrZr[s r,get_kv_cache_shapez'FlexAttentionBackend.get_kv_cache_shapegs:z<CCr.FlexAttentionMetadataBuilderctSrU)r_rHr.r,get_builder_clsz$FlexAttentionBackend.get_builder_clsqs++r.cdSNFrH)argskwargss r,use_cascade_attentionz*FlexAttentionBackend.use_cascade_attentionusur.cgSrUrHrQs r,get_supported_head_sizesz-FlexAttentionBackend.get_supported_head_sizesys r.N)rC)__name__ __module__ __qualname__rAbool__annotations__r&float16rDfloat32rBrlistr%rEr staticmethodstrrI classmethodrOrRtyperVinttupler^rarfrhrHr.r,r@r@Ls!%$%%%   5htEK01 >DZ&999$'BA''   - -/DV(!#1 r.r")r1 ignored_valpad_valMrrcld|cxkr|ksntd||jz}||d}||jdz|jd}}|||}|j} tj||  ||} tj ||dzf|| } | d|| d||k| | d|kz} tj | tjddz } tj||}tj| d \}}|||f||| ||ff<||jd|}|S) u - Keeps the first occurrence of each non-zero value while preserving order, then left-packs those uniques and fills the rest with `pad_val`. - Returns (packed, keep_mask) with the *same shape* as `x`. - Requires that all values be in the range [0, M] - Skips ignored_val Works on CPU or GPU, no Python loops, O(B·N) time / O(B·M) memory. Example: x =[3, 1, 0, 1, 2], M=3, ignored_val=0 => [3, 1, 2, -1, -1] r"z`pad_val` must lie in [-1, M]r{r!aminr}r1T)as_tuple) ValueErrorr6movedimnumelr5reshaper$r&r(expandrrgathercumsumrr full_likenonzero)r/rr1rrx_permBNx_flatr$idx first_idxkeepdest_pos packed_flatrowssrc_colspackeds r,unique_static_unsortedrs( '    Q    8999 ,C YYsB  F <<>>V\"- -v|B/?qA ^^Aq ! !F XF ,q ( ( ( / /1 5 5C Aq1u:q888I aV<<< k !cY-=-=a-H-H&H ID|DGGEJ//Q777!;H/&'22K]4$777ND(28x2HKhtX~../  . . 6 6r3 ? ?F Mr.bhq_idxkv_idxc||kSrUrH)rrrrs r,causal_mask_modrs F?r.c (eZdZUeed<eed<eed<ejed<eed<ejed<ejed<ejed<eed <eed <ejd zed <ejd zed <ejd zed<eed<eed<eed<eed<ejed<ejed<ejed<dZeed<dZ d Z e d zed<d Z e d zed<eZeed<d Zejd zed<dZeed<dZeed<dZeed <d Ze d zed!<d Zed zed"<d Zeeeeeeffd zed#<ed$Zd%ejd&ejd'ejd(eejejejffd)Zd(efd*Zd(efd+Zd(efd,Z d(efd-Z!d.Z"d(e d zfd/Z#d(e fd0Z$d(e fd1Z%d2Z&d S)3FlexAttentionMetadatacausalnum_actual_tokens max_query_lenquery_start_loc max_seq_lenrxrw slot_mapping use_cascadecommon_prefix_lenNcu_prefix_query_lensprefix_kv_lenssuffix_kv_lenstotal_cache_tokensrXmax_possible_sequence_lengthnum_reqsr decode_offsetrrnum_input_tokens block_mask score_modlogical_mask_moddoc_idsT direct_buildr q_block_size kv_block_sizetransformed_score_modsliding_windowmm_prefix_rangectjt|j|j|jjtjS)Nr#)r&r(rrrXrwr$rselfs r,logical_block_idsz'FlexAttentionMetadata.logical_block_idsOs;| !4? 3 3#**    r.request_lookuprphysical_kv_idxrc||}||jz}||jz}|j||f}||jz|z}|dk} ||j|k} |dk} | | z| z} ||j|z } | |j|z}| ||fS)zConvert physical indices to logical indices for both query and kv. NB is_within_lower_bound: do sequences start on block_boundaries? Returns: tuple of (is_valid, logical_q_idx, logical_kv_idx) r)rXrrxrr)rrrrq_reqphysical_kv_blockphysical_kv_offsetlogical_block_idxlogical_kv_idx live_blockwithin_upper_boundwithin_lower_boundis_valid local_q_idx logical_q_idxs r,_convert_physical_to_logicalz2FlexAttentionMetadata._convert_physical_to_logicalWsu%,t>,t> 4U.final_mask_modsZ11$,WW 6X}n;%%aM>JJ r.rr&Tensor)rrs` r,get_causal_mask_modz)FlexAttentionMetadata.get_causal_mask_modysq|''' | | < #\   \        r.c t|jdtjdtjdtjdtjdtjf fd }|S)zCreates the encoder mask_mod function for FlexAttention. Since the encoder bidirectional attention doesn't run with KV cache, this function creates a mask based on the packed query sequences. rrrrrc(||kSrUrH)rrrrrs r,rzHFlexAttentionMetadata.get_bidirectional_mask_mod..final_mask_mods "%(N6,BB Br.)r-rr&r)rrrs @r,get_bidirectional_mask_modz0FlexAttentionMetadata.get_bidirectional_mask_mods~4D4HII C| C| C< CL  C \  C C C C C Cr.c :jtddtjdtjdtjdtjffd dtjdtjdtjdtjd tjf fd }jr|nS) zCreates the sliding window mask_mod function for FlexAttention. Note that the sliding window mask here is bidirectional, we need to mask it with the bidirectional/causal mask for encoder/decoder. Nz7sliding_window must be set for sliding window attentionrrrrcDtj||z jkSrU)r&absr)rrrrrs r,sliding_window_mask_modzRFlexAttentionMetadata.get_sliding_window_mask_mod..sliding_window_mask_mods!9UV^,,t/BB Br.rrc j||\}}}tj|||||dSrcrrr&r) rrrrrrrrrs r,rzIFlexAttentionMetadata.get_sliding_window_mask_mod..final_mask_modsW11$,WW 6X}n;''1m^LL r.)rrr&rr)rrrs` @r,get_sliding_window_mask_modz1FlexAttentionMetadata.get_sliding_window_mask_mods   &VWW W C| C %  C5:\ CKP< C C C C C C | | < #\  \       "&I~~2IIr.c 8jJjdtjdtjdtjdtjdtjf fd dtjdtjdtjdtjd tjf fd }|S) z:Creates the prefix LM mask_mod function for FlexAttention.Nrrcu_q_idxrrctj|tj}jpiD]>\}} ||k}|D]*\} } || k|| kz} || k|| kz} ||| z| zz}+?|S)N)r%)r& zeros_likerlritems)rrrrrrreq doc_range_lstreq_maskstartend doc_mask_q doc_mask_kvrrs r,prefix_lm_mask_modzHFlexAttentionMetadata.get_prefix_lm_mask_mod..prefix_lm_mask_mods#E<<.final_mask_modsY11$,WW 6X}n;""1a ~NN r.r)rrrrs` @@r,get_prefix_lm_mask_modz,FlexAttentionMetadata.get_prefix_lm_mask_mods|''' | | l <   L         | | < #\  \       r.c|jr|}n|}|j$|}t ||}|jr$|}t||}|SrU) rrrrrr rrr )rmask_modrrs r, get_mask_modz"FlexAttentionMetadata.get_mask_mods ; 9//11HH6688H   *&*&F&F&H&H # +BCCH   >!%!!> *<==Hr.c jdStjjdtjdtjdtjdtjdtjdtjf fd }|S) zCreates the transformed score_mod function for FlexAttention. This function wraps the user's score_mod to handle physical-to-logical index conversion, similar to how get_mask_mod works for mask functions. Nscorerrrrrc  ||\}}}tj| ||||||td S)N) physical_qinf)rr&rfloat) rrrrrrrrrruser_score_mods r,rzNFlexAttentionMetadata.get_transformed_score_mod..transformed_score_modsq11"E? 6X}n ;1a5u  r.)rr-rr&r)rrrrs` @@r,get_transformed_score_modz/FlexAttentionMetadata.get_transformed_score_mods > !44D4HII < | | <   #\   \         *%$r.c|j|jz}|dkr td|jd|jd|j|jdt |j|jf}|jr|jr|j }|jJtj |jj d|tj }||j|jz |j|jz}tj||jdz z d}||jz}|j|dddfk}||dt'||jd } | | j d|jzd } | |z} t-| |j tj} | dkd tj} |j|jf| d | d dd|j|jf|jd} t=drd| d<t?j di| S)aDirect block mask construction for standard causal attention. This method constructs the block mask directly using BlockMask.from_kv_blocks which is much more efficient than the generic create_block_mask approach. The direct path works as follows: 1. For each query token, fetch blocks from block_table using max_seq_len and exclude out of sliding window blocks if needed. (this fetches more blocks than needed for shorter sequences) 2. Group query tokens into chunks of q_block_size 3. For each group, deduplicate the blocks using unique_static_unsorted 4. Create BlockMask using the deduplicated block indices Over-estimation occurs when a group of q_block_size tokens contains multiple sequence IDs (doc_ids). In this case, we fetch ALL blocks for each sequence represented in the group, even though individual query tokens may only need a subset of those blocks based on causal masking and their position. r!z7FlexAttention currently requires the cache block size (z$) to be equal to the kv_block_size (z+). Please check your model's configuration.Nrr#)min)r0r1r")rr)NN) seq_lengths kv_num_blocks kv_indicesfull_kv_num_blocksfull_kv_indices BLOCK_SIZEr 2.9.0.dev0Fcompute_q_blocksrH)!rrXrrwrrrrrr$r&r(r5rrrclampr masked_fill_r>rrrrWrr*sumrrrrrfrom_kv_blocks) rpage_to_block_ratio used_pagesr$ token_indicesr min_kv_idx min_block_idx sliding_maskused_pages_paddedr r block_mask_kwargss r,_build_block_mask_directz.FlexAttentionMetadata._build_block_mask_direct+s,#0DOC ! # #"O""&""" % LCD!14?CCC C    64; 6&F<+++!L "1%fEJM&t|45$T\23  ]d6IA6M%NTUVVVJ&$/9M1]111d75KKL  # #\M1 5 5 5+ !2   .55  #A &$*; ;R  .1DD+  # # % %$/   "U[// $q--"-5588EE  2D4KL*:6$Z0"&#,d.@A    #< 0 0 :49 0 1'<<*;<<    r.c|jdus Jd|jdks Jd|j Jd|j Jd|j Jdt |j|_|j|j z|_ | |_ | |_|jr"|jr||_dS||_dS)NFzNot implemented yet.r)rrrrrr-rrrrXrWrrrrrrrrr#rs r, __post_init__z#FlexAttentionMetadata.__post_init__s5(((*@(((%***,B***(002H000"**,B***"**,B***1$2FGG 1T_D))++ %)%C%C%E%E"   6 6";;==DOOO"3355DOOOr.)'rirjrkrlrmrur&rrrWrrrrrrrrrrrrrrdictrprvrrrrrrrrrrr#r%rHr.r,rr%sw LLL\!!!l,,----L4''''L4''''OOO"%%%%MMM%%%< $$$cJ#'J D '''-1I#d*111,;);;;#'GU\D '''L$L#M39=/$6===!%NC$J%%%?COT#tE#s(O445<CCC  _  7  7| 7 7 u|U\5<7 8 7 7 7 7D%8@,?(J-@JJJJ@%(;%%%%N&"%+?$+F"%"%"%"%HL=)L=L=L=L=\  )     66666r.rc neZdZdedeededejffd Z d de de d e d e fd Zd e fd ZxZS)r_ kv_cache_spec layer_names vllm_configr$ct|||||j|_|j|_|j|_|j|j|_|j|j|_|j |_ |j |_ ||_ td}||_|rdnd|_|r|j nd|_dS)Nrr)super__init__ model_configparallel_config cache_configget_num_attention_heads num_heads_qget_num_kv_heads num_heads_kv get_head_sizeheaddimrXr(rrrr)rr(r)r*r$supports_small_blocks __class__s r,r.z%FlexAttentionMetadataBuilder.__init__s  [&III'4*:'4,DD    !->>t?STT(6688 '2* 7 E E"7''01D+8 !(DO<<'!+ #  ;%&9:: :'2 #0>*9)) =*)),j89 *n  -HHJJ #   '.. // (- ,O  $  X  +* & $  0/ "6!5 *> *> "z *>)= X! "!4 3# $ 21% &(-' ( 21) 0+K0D0K1 2**3 4,,5 8 r.cdSrcrH)rrdres r,rfz2FlexAttentionMetadataBuilder.use_cascade_attentionsur.)F)rirjrkrrprrr r&r$r.rurrlrrFrf __classcell__)r9s@r,r_r_sT$T#YT T  TTTTTT:! CCC6C C  CCCCJr.r_ceZdZUedzed<ejdzed<edzed<dZe ee e eeffdzed<de j dfdededed ede edzdedzd ededzd e d edzd dfdZedejd ejfdZ ddejjdejdejdejdejdedejdzdejdzdejdzd ejfdZdS)rSNr alibi_slopeslogits_soft_capr num_headsrZscalerYkv_cache_dtyperJkv_sharing_target_layer_namerc  ||_||_t||_||_| |_| t jt jfvrtd| d|tdd|_ ||_ ||_ ||_ |j td|j|jzdksJ|j|jz|_| tdt|j rtddS)NzFlexAttention does not support z attentionz0FlexAttention does not support alibi slopes yet.z3FlexAttention does not support logits soft cap yet.rz.FlexAttention does not support kv sharing yet.z6FlexAttention does not support quantized kv-cache. Yet)rLrZrrMrYrJrrMrLr>rJrrNrKnum_queries_per_kvr) rrLrZrMrYrJrrNrKrJrOres r,r.zFlexAttentionImpl.__init__s7#"5\\ (" ]79NO O O%G)GGG   #%B !%D ,,.   +%E ~ 11Q6666"&.D4E"E ' 3%&VWW W !4 5 5 %H   r.tensorcZ|jdkr|S|jdksJ|dddddddfS)zView a 3d tensor as 4D.N)r6)rRs r, view_as_4dzFlexAttentionImpl.view_as_4d6sH ;!  M{adAAAqqq!!!m$$r.layerquerykeyr4kv_cache attn_metadataoutput output_scaleoutput_block_scalec || Jd|| tdjjk} ||dS|j} d} |jjkr.j|_|jr||_d} j t|ddkr'|j _ ||_d} | rA|jr!|j r| |_ n||_ |j sjt jksJt%fd|||f\}} }|ddddd| ddf}| d | ks|d | kr(| ddddd| ddf} |ddddd| ddf}nމjt jksJ|d\}}t,jj|||||jj|j|j|d jj}|d jj}t%fd |||f\}} }|ddddd| ddf}|j J|j j \}}tC||||j}tE|| ||j#|j j$| | }|%dd dd&d}|d| ddddf'||S)aForward pass with FLexAttention. Args: query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [2, num_blocks, block_size, num_kv_heads, head_size] attn_metadata: Metadata for attention. Returns: shape = [num_tokens, num_heads * head_size] NzOutput tensor must be provided.zDfused output quantization is not yet supported for FlexAttentionImplrFTrcZ|ddddSNrr]r!rUrVpermuter/rs r,z+FlexAttentionImpl.forward..}'$//!,,44Q1a@@r.r"cZ|ddddSrarbrds r,rez+FlexAttentionImpl.forward..rfr.) enable_gqakernel_optionsr]r!rU)(r>rYrLfill_rrrrrrgetattrrrrr#rJrrMmapsizerLunbindr&ops _C_cache_opsreshape_and_cache_flashrrN_k_scale_v_scaleviewrZrget_kernel_optionsflex_attention_compiledrrMrcsqueezecopy_)rrWrXrYr4rZr[r\r]r^rirneeds_rebuild_block_mask key_tensor value_tensor key_cache value_cacheblock_mblock_nrjrEs` r,forwardzFlexAttentionImpl.forward>s0!!#D!!!  #'9'E%V &$.8  <<?? "*;#(  '4+> > >+/+>M () F)6)C)C)E)E &'+ $  7=:KT#R#R R R#0#@D %2%?%?%A%AM "'+ $ # L) Lm.B L+8+Q+Q+S+S ((+8+I+I+K+K (#) 7>]%?????.1@@@@U#// +E:| !!!QQQ 2!2 2AAA56E##&777!!"%%(999 (111.@/@.@!!!(CD +AAAqqq2D3D2Daaa,GH >]%:::::%-__Q%7%7 "I{ I " : :*#   "r4++ 7G]%?  &     /  $ J!)    kk!Q1%%--a00!!!111aaa'(..s333 r.)NNN)rirjrkrurmr&rrrr&rprvrrLrrr.rqrVnnModulerrrHr.r,rSrSs $J,%%%%T\!!!?COT#tE#s(O445<CCC)-#0#8372222 2  2 5kD( 2d 222!2'*Dj2 2222h%5<%EL%%%\%'+,026}}x}|}\ } | } , }-} t#}lT)}"L4/} }}}}}}r.rSuse_direct_buildcddi}dtdtdtfd}trd|d<d|d <d |d <|S|r ||d<||d <|S|jtjkrd nd }d}|||}|||} tjrtj} t| dr| j } n%tj rd} ntd| dkr:|td|dz|}|td| dz|} t||}t| |} ||d<| |d <|S)NFORCE_USE_FLEX_ATTENTIONT candidaterXrc|dkr|St||}|dkr|S||zdkr|Stj||}|dkr|S|S)z8Pick a kernel block size that divides the logical block.rr!)r mathgcd)rrXs r,ensure_divisiblez,get_kernel_options..ensure_divisiblesk ??  :.. >>   !Q & & HY 33 >> r.rBLOCK_MBLOCK_NF IS_DIVISIBLE @shared_memory_per_block_optiniz8Unable to determine shared memory size on this hardware.i@r!r])rurr%r&rocuda is_availableget_device_propertieshasattrrris_rocm RuntimeErrormax) rXrrrrjrpreferred_blockblock_lower_boundblock_m_candidateblock_n_candidate device_propsmax_shared_memorys r,rvrvs #D-N C S S      $&y!$&y!).~&%6$+y!$+y! % u} < <""",,_gFF,,_gFF : " " $ $  :;;==L|%DEE $0$N!!!)++ $)!!"N!:--$4$4,122G%%!%5$4,122G%%! 13DEE 13DEE$5y!$5y! r.)D__doc__r dataclassesr functoolsrtypingrr&torch._dynamo.decoratorstorch.nn.functionalr functionalr9!torch.nn.attention.flex_attentionrrrr r r r vllm.configr vllm.config.cacher vllm.loggerr*vllm.model_executor.layers.batch_invariantrvllm.platformsrvllm.utils.math_utilsrvllm.utils.torch_utilsrvllm.v1.attention.backendrrrrrrvllm.v1.kv_cache_interfacerrilogger_dynamoconfigrecompile_limitcompiler!rwrr-rur>r@rrrrr_rSrlr&rrrvrHr.r,rs~*) !!!!!!%%%%%% #"""""((((((######,+++++&&&&&&::::::544444 X  ') $*U],=(%-$GGG 8u|8s88888"/////+///fiilii i  \ iiii` 111 |1 1 1  1  1 \1111h | -2\CH<  p6p6p6p6p6p6p6 p6f `````#;