.`i"<ddlZddlmZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZddlmZddlmZmZmZmZmZe eZGdd eeZGd d e ZGd d ZedZdZGdde eZdS)N)ClassVar) CacheDType) init_logger)MLACommonBackend MLACommonImplMLACommonMetadataMLACommonMetadataBuilder)DeviceCapability)AttentionCGSupportAttentionLayer AttentionType MultipleOfis_quantized_kv_cachec4eZdZUejZeeed<dS)CutlassMLAMetadataBuilder_cudagraph_supportN)__name__ __module__ __qualname__r UNIFORM_SINGLE_TOKEN_DECODErr__annotations__~/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/attention/backends/mla/cutlass_mla.pyrrs: 6!34rrc<eZdZUejejgZeeej e d<gdZ eee e d<e deeezfdZe defdZe dedfdZe ded fd Zed edefd Zd S)CutlassMLABackendsupported_dtypes)autobfloat16fp8fp8_e4m3supported_kv_cache_dtypesreturncdgS)Nrrrr get_supported_kernel_block_sizesz2CutlassMLABackend.get_supported_kernel_block_sizes.s u rcdS)N CUTLASS_MLArrrrget_namezCutlassMLABackend.get_name2s}rCutlassMLAImplctSN)r*rrr get_impl_clszCutlassMLABackend.get_impl_cls6srrctSr,)rrrrget_builder_clsz!CutlassMLABackend.get_builder_cls:s((r capabilityc|jdkS)N )major)clsr0s rsupports_compute_capabilityz-CutlassMLABackend.supports_compute_capability>s2%%rN)rrrtorchfloat16rrrlistdtyperr"r staticmethodintrr&strr)typer-r/ classmethodr boolr5rrrrr%sJ5:]EN4ShtEK01SSS===xZ(89d33C.D\c\$/0\)T"=>)))\)&5E&$&&&[&&&rrc*eZdZdZdZdedefdZdS)SM100Workspacectj|dtj|_d|_tjtjd}|j|_ dS)Ncuda)devicer9r%zcuda:0) r6emptyuint8_workspace_buf _block_sizerCget_device_propertiesrDmulti_processor_count _sm_count)selfinitial_workspace_size propertiess r__init__zSM100Workspace.__init__Ds]#k "6   Z55el86L6LMM #9rc|jSr,)rG)rLs rget_bufzSM100Workspace.get_bufPs ""r attn_metadata num_kv_splitsc|j}|j}tj||jz||j|}|jjd|kr|j|dSdS)N)rSr) num_reqs max_query_lenops$sm100_cutlass_mla_get_workspace_sizerHrKrGshaperesize_)rLrRrS batch_size max_seq_lenworkspace_sizes r ensure_sizezSM100Workspace.ensure_sizeSs|"+ #1 A $* *  N'       $Q '. 8 8   ' ' 7 7 7 7 7 9 8rN)rrrrOrQrr;r^rrrrArACsU : : :### 8): 83 8 8 8 8 8 8rrAir%ceZdZUdZeed<dededededeedzd edzd e d edzd e d e dzddffd Z de j de j de j de j de j de j dedede e j e j ffdZde j e e j e j fzde j dedede e j e j dzff dZxZS)r*Tcan_return_lse_for_decode num_heads head_sizescale num_kv_heads alibi_slopesNsliding_windowkv_cache_dtypelogits_soft_cap attn_typekv_sharing_target_layer_namer#c tj||||||||| | f dti| |||g} t| rt d| t jkrt dtj dd} | r=t dt| t| |_ nd|_ t|_dS)Nq_pad_num_headszcCutlassMLAImpl does not support one of the following: alibi_slopes, sliding_window, logits_soft_capzaEncoder self-attention and encoder/decoder cross-attention are not implemented for CutlassMLAImplFORCE_NUM_KV_SPLITSzForcing num_kv_splits to %d)superrO MAX_HEADSanyNotImplementedErrorr DECODERosenvirongetlogger debug_oncer;_num_kv_splitsg_sm100_workspace _workspace)rLrarbrcrdrerfrgrhrirjmla_argsunsupported_featuresforce_num_kv_splits __class__s rrOzCutlassMLAImpl.__init__js            ( &  !-noN # $ $ %@   - - -%! !jnn-BDII  %   ;SAT=U=U V V V"%&9":":D  "$D ,rq_nopeq_pekv_c_and_k_pe_cacheseq_lens page_table workspacesm_scalerSc .|jdksJd|j|jdksJd|j|jdks"Jd|j|j\} } } |j\} } }| | kr| | ksJ|j\}}}d}d}| |ksJ||ksJ|||zksJd}| |ksJd|d | t|jd ksJ|j\}}|| ksJ|d ks Jd ||d|z zd ksJ|jt jt jt jfvsJd |jd|j|jcxkr |jksnJ|jt j ksJd|jd|jt j ksJd|jdt|j r t jn|j}| | ||f|}|j r(t j| |ft j|jnt j}t%j|||||||||| | |kr%|j r|ddd| fn|}|ddd| f}||fS)Nz$q_nope must be a 3D tensor, but got z"q_pe must be a 3D tensor, but got z3kv_c_and_k_pe_cache must be a 3D tensor, but got {}i@r%z H must be <= z , but got rz&block num must be greater than 0, got z6q_nope.dtype needs to be fp16 or bf16 or e4m3 but got .z)seq_lens.dtype needs to be int32 but got z+page_table.dtype needs to be int32 but got )r9)r9rD)ndimformatrYlenr9r6r7r float8_e4m3fnint32rrg new_emptyneed_to_return_lse_for_decoderEfloat32rDTensorrWsm100_cutlass_mla_decode)rLrrrrrrrrSB_qHD_q_nopeB_q_2H_2D_q_pe_ PAGE_SIZED_ckvD_latentD_roperp B_block_table block_numr9outlses r_sm100_cutlass_mla_decodez(CutlassMLAImpl._sm100_cutlass_mla_decodesc{a!U !U!UyA~~~ODIOO~~~"'1,,, A H H#(  -,, "<Q!ZsFu 1888,179e8####6))))) I~~~GyGGAGG~~~:#$$))))#-#3 y####1}}}RyRR}}}C)O,1111| u~u?RSSSS TV\ T T TTSS|tzFFFF-@-FFFFFFF~,,, I I I I-,,5;... M*:J M M M/.. %T%899 ENN  Y9GG1 EKi( fm T T T T $            y== $ BK#aaa!e**Caaa!e*CCxrqrRlayerc |dksJ|jJt|tur|\}}n&t j||j|jgd\}}|j ||j | ||||jj |jj |j|j|j \}}||jr|ndfS)Nrrn)dim)numeldecoder=tupler6split kv_lora_rankqk_rope_head_dimr{r^ryrr block_tablerQrcr) rLrrrRrrrors r_forward_decodezCutlassMLAImpl._forward_decodes#((**Q....#/// 77e  LFDD ;D%t'<=2LFD ##M43FGGG//     )  , O # # % % J     3$<F33$GGr)rrrr`r?rr;floatr8r<rOr6rrrrr r __classcell__)rs@rr*r*gs&*t***8,8,8, 8,  8, 5kD( 8,d 8,8,8,8,'*Dj8, 8,8,8,8,8,8,tQ QlQ#\ Q , Q L Q<QQQ u|U\) *QQQQf H <% el :; ; H#\ H) H  H u|U\D00 1 H H H H H H H Hrr*) rttypingrr6vllm._custom_ops _custom_opsrWvllm.config.cacher vllm.loggerr2vllm.model_executor.layers.attention.mla_attentionrrrr vllm.platforms.interfacer vllm.v1.attention.backendr r r rrrrwrrrArzrpr*rrrrs  ((((((###### 655555 X   89J K&&&&&(&&&<88888888>#N#455  pHpHpHpHpH]#45pHpHpHpHpHr