.`i"& (ddlZddlmZmZddlZddlmZddlmZddl m Z m Z ddl m Z mZddlmZdejd ed eejejejffd ZGd d e ZeGdde ZGddeeZdS)N) dataclassreplace) VllmConfig)cdiv)AttentionBackendCommonAttentionMetadata)BaseMambaAttentionMetadata!BaseMambaAttentionMetadataBuilder) AttentionSpecquery_start_loc chunk_sizereturnc>|jdks Jdt|ddks Jd|j}|t j}|dd}|dd}t|d}g}g}dgt|z} tt||D]\} \} } | | kr| } | | krr|| |zz }t|| | z }| t|| | t|dz | | <| |z } | | kr|rrt j dgttj|z|t j}t|d|ksJn"t j dg|t j}t|dkr!t j | |t jn t jd|t j}t j ||t j}|||fS) aa Build chunk-aligned, variable-length metadata used by Mamba2 SSD kernels. Given per-sequence cumulative token starts `query_start_loc` of shape [B+1] and a physical `chunk_size`, returns three tensors on the same device: - cu_chunk_seqlens: (nchunks+1,) int32 exclusive prefix-sum of logical-chunk lengths (each logical chunk never crosses a sequence or physical-chunk boundary). - last_chunk_indices: (B,) int32 index of the last logical chunk for each sequence (=-1 for empty sequences). - seq_idx_chunks: (nchunks,) int32 sequence index for each logical chunk in order. This is intentionally lightweight and CPU-side; it mirrors the metadata produced by the V1 Mamba2 meta-data builder and is exported so tests (and other callers) can avoid duplicating the logic. z!query_start_loc must be 1-D [B+1]rzquery_start_loc[0] must be 0Ndevicedtype)r)ndimintitemrtotorchint64tolistlen enumeratezipminappendtensorlist itertools accumulateint32empty)r r rqsl64startsendstotal chunk_lensseq_idx_chunkslast_chunk_indicesbseposroomtakecu_chunk_seqlenslast_chunk_indices_tseq_idx_chunks_ts z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/attention/backends/mamba2_attn.pycompute_varlen_chunk_metadatar8s*  1 $ $ $&I $ $ $ q!&&(( ) )Q . . .0N . . .  #F   u{ + +E 3B3Z   F 9    D b   ! !EJ "N%'D3v;;$6s640011   6Aq 66 Aggz!12DtQW%%D   c$ii ( ( (  ! !! $ $ $$' OOa$7 q ! 4KCAgg O < C$y+J7788 8+    #B',,..//588888 <F%+NNN v;;??  'ekJJJJ [fEK @ @ @ |N6UUU 13C CCcReZdZedefdZededfdZdS)Mamba2AttentionBackendrcdS)N MAMBA2_ATTNr>r9r7get_namezMamba2AttentionBackend.get_name[s}r9Mamba2AttentionMetadataBuilderctS)N)r@r>r9r7get_builder_clsz&Mamba2AttentionBackend.get_builder_cls_s--r9N)__name__ __module__ __qualname__ staticmethodstrr?typerBr>r9r7r;r;Zsfc\.T"BC...\...r9r;ceZdZUdZeed<dZeed<dZe j dzed<dZ e j dzed<dZ e j dzed<dS) Mamba2AttentionMetadataFprep_initial_statesrr N seq_idx_pcu_chunk_seqlen_plast_chunk_indices_p) rCrDrErKbool__annotations__r rrLrTensorrMrNr>r9r7rJrJds %%%%J&*Iu|d"))) .2u|d*11115%,-44444r9rJc eZdZeZdedeedede j ffd Z de de j de j d eee ee ee ffd Z dd e d eded efdZxZS)r@ kv_cache_spec layer_names vllm_configrct|||||j}| Jd||_dS)Nz@chunk_size needs to be set in the model config for Mamba2 models)super__init__ model_configget_mamba_chunk_sizer )selfrSrTrUrr __class__s r7rXz'Mamba2AttentionMetadataBuilder.__init__zs[  [&III -BBDD %% N&%% *r9 num_prefillsnum_computed_tokens_p_cpuquery_start_loc_p_cpurc0g}g}g}d}t|D]d}||} ||dz||z } | |jzdkrd||||t | |j|jz| z } t | | } || z }| | z} t | |j} t| D]K} ||||t |j| } || z }| | z} L| dksJ|t |dz f|||||fS)a^ Compute chunk-specific metadata for Mamba2. The code below carefully constructs the chunks such that: 1. Chunks contain tokens from a *single* sequence only. 2. For every sequence, we are guaranteed that we can retrieve the mamba state *every* chunk_size tokens. Constraint (1) dramatically simplifies the mamba2 kernels. Constraint (2) dramatically simplifies the implementation of prefix caching for mamba2 (wip). We need to take care of the interaction with chunked prefill in order to satisfy constraint (2). rr)rangerr r rrr)r[r]r^r_cu_chunk_seqlenseq_idxr- seqlen_posreq_idxthis_num_computedthis_new_tokens chunk_lenn_chunkschunks r7_compute_chunk_metadataz6Mamba2AttentionMetadataBuilder._compute_chunk_metadatas( \** @ @G 9' B G G I I %gk27799'055778 !4?2a77w'''&&z222*DO< ? ? ? ?z***);;;r9Fcommon_prefix_lencommon_attn_metadata fast_buildc||}d}d}d}d}|jdkr.|j+tj|jnd}|j} |j} |j} | } | | | z | } |j | dz d| z }| | | |\}}}tj ||j jtj}tj ||j jtj}tj ||j jtj}t!|||j|||S)NFrrr)rKr rLrMrN)_compute_common_metadatar]has_initial_states_pranyrnum_reqsnum_decode_tokenscompute_num_computed_tokenscpuquery_start_loc_cpurk as_tensorr rr%rr )r[rlrmrncommonrLrMrNrKrsr]rtnum_computed_tokens_cpur^r_rbrcr-s r7buildz$Mamba2AttentionMetadataBuilder.builds ../CDD  ##   " ".: &566;;=== H!.L & 8 %@@BBFFHH $)@<'(2) %%8,9J9L9LM#$ " <@;W;W)%<< 8OW&8 +;BkI !&+;Bk!!!  $)?"+;Bk$$$   3/!5     r9)F)rCrDrErJ metadata_clsr r"rGrrrrXrrQtuplerkrrOr{ __classcell__)r\s@r7r@r@us+L *$ *#Y * *  * * * * * *<<<<$)<<< %| << tCy$s)T#Y. / <<<<<<<r9r7rs******** """"""&&&&&&544444BD\BDBD 5<u| 34BDBDBDBDJ.....-...  5 5 5 5 58 5 5  5 R R R R R %&=>R R R R R r9