.`iQddlZddlmZddlmZmZejdejdejdejdejdejd ejf d Zejdejdejdejdejdejd ejd ejd ejd ejdejfdZ ejdejdejdejdejdejd ejd ejfdZ ejdejdejdejdejdejd ejd ejdejfdZ Gddej j ZejZ d%dejdejdejdejdedejdzdeejejffdZejdejdejfdZ d&dejdejdejd!ejd"ejd#ejdedejfd$ZdS)'N) rearrange)tltritonbhdeBLOCKCBLOCKc tjd} | | z}| | z}tjd}||z}||z|z}||z| z}||z| z}|| z}||z}|| z}|| z}|| z}||z}|| z}||z|z|ztjd| dddf|zztjd|dddfz}||z|ztjd| dddf|zztjd|dddfz}||z|ztjd| dddf| zztjd| dddfz}||z|z|ztjd| dddf| zztjd| dddfz}||z} tj| }!|}"tjd| |"| zz}#tj|||#dddfz|kdtj}$tj| | gtj}%t|"dzD]B}&tjd| |&| zz}'|#dddf|'dddfz }(|!|(z})tj|(dk|) td})tj |)}*tj|||'dddfz|kdtj}+tj|||'dddfz|kdtj},tj |$|+|*z}-|%tj |-|,z }%|| |zz }|| | zz }Dtj ||%|j j||#dddfz|kdS)Nrmaskotherdtypez-infr)r program_idarangeloadtofloat32zerosrangewherefloatexpdotstorer element_ty).QKVOutSrrnrr r NUM_BLOCKr offoff_bh off_block off_cblockoff_h qk_offsetv_offseto_offset block_offsetqk_block_offsetv_block_offseto_block_offset cblock_offsetq_cblock_offseto_cblock_offset Q_block_ptrK_trans_block_ptr V_block_ptr O_block_ptr S_block_ptrsiq_indexqqkvjkv_indexdiffs_indexdecayk_transvqks. }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/model_executor/layers/lightning_attn.py_fwd_diag_kernelrK s& -  C I FiIq!!J QJE QIzA~HzA~Hu$L"Q&O!A%N!A%N'M#a'O#a'O       )Av  qqq$w '! +  , )Aq//$' "  #     )Av  tQQQw '! + , )Aq//!!!T' "  #     )Av  qqq$w '! + , )Aq//$' "  #       )Av  qqq$w '! +  , )Aq//$' "  #e)K  AAi6""QZ/G  ,D1A"AA"ESQQQTT   A (FA;bj 1 1 1C1q5\\""9Q''!f*4qqq$w(47"33d((419whf >>w' qqq 11A5    "RZ..  G D 11A5    "RZ.. VAw  % ' rvb!}} VaZ'vz! H { +,, GAAAtG, ,q 0D_FBLOCKE_FBLOCK NUM_FBLOCK NUM_CBLOCKctjd}tjd}||z}|| z}||z}||z}||z|z}||z|z}||z|z}|| z|z|z}||z|ztjd|dddf|zztjd| dddfz}||z|ztjd|dddf|zztjd| dddfz}||z|ztjd| dddf|zztjd| dddfz}||| zztjd|z}tjd|}tj| | gtj}|| dz kr || dz | zz n| } tj| ||z| z }!t tj| ||}"|||"z |zz }t|"D]}#d|#z |!z}$tj||!|zz |dddf|$kd}%tj||!|zz |dddf|$kd}&tj|}'|'dddf}'|tj |%|'z|&z }|||zz }|||zz }||z }tj || |j j dS)Nrr rrr)rrrrrcdivminrrrr rrr!)(r#r$K_decayKVrrr'rr r r(rMrNrOr rPr*r+r-r1k_block_offsetr3kv_block_offsetk_offsetr/ kv_offsetr9r: KV_block_ptr k_decay_ptrrCkvsplit_n left_shift num_blocksrB left_boundrGrHk_decays( rJ_fwd_kv_parallelrbs*]1  F a  I QJEu$L"A%N!A%N!ma'OzA~HzA~H"Q&*I     )Av  tQQQw '! + , )Ax D )  *     )Av  qqq$w '! + , )Ax qqq )  *     )Ax D )A - . )Ax qqq )  *EEM)BIa,@,@@KyF##H 8X&bj 9 9 9B.7)a--G-Ga9q=E)))UG&))F2W??@@@@@rLc tjd} | |z} | | z|z|z}||ztjd| dddf|zztjd| dddfz}|| z}tj|}| |z|z}||ztjd| dddf|zztjd| dddfz}tj|tj}t | D]}t|||zz |}tj|tj |z}tj|tj}tj |||j j ||z|z}|||zz }tj ||dS)Nr) rrrrrrrrSrr rr!)r&rU KV_HISTORYrrr'rr r r(rMrNr*r-rYrZs_ptrsr=kv_history_offsetKV_HISTORY_block_ptrkv_prer> block_size block_decaykv_curs rJ_fwd_kv_reducerls"]1  F QJE"Q&*I   )Ax D )A - . )Ax qqq ) *YF A Q   )Ax D )A - . )Ax qqq ) *W) * * - -bj 9 9F9    QY.. fadd2:...;<< &&))"*55 vyy);)FGGHHHv%.A H !6*****rLc*tjd}||z}tjd}|| z}|| z}tjd}|| z}|| z}|| z}||z}||z|z||z|zz}||z|z||z|zz|z}|| z|z|z||z|zz|z}||ztjd| dddf|zztjd|dddfz}||ztjd| dddf|zztjd| dddfz}||ztjd|dddf|zztjd| dddfz}||z}tj|}tjd| } tj|tj}!|tjd| z}"tj||"dddf|kdtj}#tj|tj || z| dddfzz}$tj|#|!|$z}%tj||"dddf|kdtj}&|&|%z}'tj||'|j j |"dddf|kdS)Nrr rrr) rrrrrrrrr rr!)(r"r%r&rUrrr'rr r r(rNr rPr*r-off_ncoff_noff_coff_en_offsetc_offsete_offsetr1q_offsetr0rYr8r;rZr<r=c_arrayr\r?r@q_decay qkv_none_diagqkv_diagrAs( rJ_fwd_none_diag_kernelr{9sZ(]1  F QJE ]1  F j E Z E M!  Eu}Hv~HxHh&LzA~H!4 99HzA~H!4 99HDH"Q&*UQY]:XEI H ryF++AAAtG4q8829Q??4QRQRQR7;SS   )Av  qqq$w '! + , )Ax qqq ) * Y1aD1A55 !X8N8NtUVUVUVw8WW e)K  Ai6""G    ! !"* - -BRYq&111G  '!!!T'"2Q"6cBBBEEbjQQAfadd2:&&&%&.7111d7;K*KLMMGF1bMMG+Mw{D)9A)=SIIILLRZXXH ] "CHSVVK-8994@PST@TrLc$eZdZedZdS) _attentioncR|}|}|}|}tj}|ddkrt dd|j\}}} } |jd} tj||| | f|j|j} d} tj | | }d}| |z}| |zdks Jd tj d| |j d z}tj | | | d dz z}||z|z|f}t||||| |||| | | | || d }| |z}| |zdksJ| |z}| |zdksJd }| |z}| |zdks Jd tj|||| | ftj|j}||z|f}t!|||||||| | | | ||||||||z|f}t#||||||| | | | ||| ||z||zf}t%||| ||||| | | | ||||||||||| |_| tj||dgdfS)Nrz(Flash attention currently only supportedzfor compute capability >= 80rdevice z"BLOCK must be a multiple of CBLOCK)rr )r r(r @)r r(rMrNrOr rP)r r(rMrN)r r(rNr rPrn)dim) contiguoustorchcudaget_device_capability RuntimeErrorshapeemptyrrrrRrrreshaperKrrbrlr{save_for_backwardr cat unsqueeze)ctxr@krHr= kv_history capabilityrrr'rr or r(r rParrayragridrOrMrNr\s rJforwardz_attention.forwards LLNN LLNN LLNN LLNNZ5577 a=1  :.  W 1a GBK KAq! AGAH E E EK5)) f_ v~"""$H""" Qah777!;)QB%%--2*>*>">?@@A !:.     "  ?:~"""" ?:~""""f_ v~"""$H"""[!Q 1a0 ah W W WAy!   !!!    *Az"t    Ay:-.d#  !    $ aAq"--- %)R!5!5a!8!89qAAAAArLN)__name__ __module__ __qualname__ staticmethodrrLrJr}r}s7~B~B\~B~B~BrLr}rr@rrHedrirreturnc|jd}|jd}|dkr|dddd}|dkrdnd|zdksJd|ddfd t|zdzD}|d|kr||t |} d} |At j|jd|jd||ft j|j }n&| }t| dz D]H} || } || dz}|d | |f} |d | |f}t| ||||\}}| |z} I| |fS) a$ Apply lightning attention algorithm to compute attention efficiently. Args: q: Query tensor of shape [batch, heads, seq_len, dim] k: Key tensor of shape [batch, heads, seq_len, dim] v: Value tensor of shape [batch, heads, seq_len, dim_v] ed: Decay rate tensor of shape [heads] block_size: Size of blocks for block-sparse attention kv_history: Optional key-value history from previous computations Returns: output: Attention output kv: Updated key-value history rr rrz Dimension d (z) must be divisible by m ()cg|]}|zSrr).0r>ms rJ z'lightning_attention..4s , , ,Q1q5 , , ,rLNr.) rrviewrappendlenrrrrclonerlightning_attention_)r@rrHrrirrr arrr'outputr>r=q1k1rr\rs @rJlightning_attentionrs0  A  A vvxx1}} WWQAq ! !CxxRA q5A:::HqHHAHHH::: , , , ,%Q ++ , , ,C 2w!|| 1  CA F[ WQZQ *%-    %%''2244 1q5\\ F AJ sAaCx[ sAaCx[$RQJ??2! 2:rLD BLOCK_SIZEctjd}tjd}tjd}tj||ztj}|dkrdS|}|}tj||z}tjd|}tjd|||zz}|dddf| z|dddf| zz}||z|| zz}||z|| zz}||z|| zz}|| z|| zz}||k}||k}tj||z|z|d}tj||z|z|d} tj||z|z|d}!| dddf|!dddfz}"|dddf|dddfz}#tj| }||z|z}$tj|$|#d}%|"||%zz}"|dddftj|"z}&tj|&d}&tj |$|"|# tj ||z|z|&| dS) z Kernel for linear attention decoding with KV cache. This kernel computes attention for a single token using the KV cache. rr rnrNrr)axisr) rrrrint64rrrsumr )'q_ptrk_ptrv_ptr kv_cache_ptr slope_rateslot_idx output_ptrr qkv_b_stride qkv_h_stridecache_b_stridecache_h_stridecache_d0_stridecache_d1_striderpid_bpid_hpid_dslot_idbatch_idhead_idratio qk_d_offsets v_d_offsetscache_d_offsetsrvrXr/ cache_offsetqk_maskv_maskr@rrHkv_outerkv_maskkv_ptr kv_cache_oldrs' rJ_linear_attn_decode_kernelrMs. M!  E M!  E M!  Egh&''**2844G"}}HG GJ& ' 'E9Q??L)Az**UZ-??KQQQW/+dAAAg2F2XX ,&<)??H,&<)??H,&<)??H^+g.FFLQG 1_F  </gSIIIA  </gSIIIA  ;.V3GGGADzAdAAAgJ&Haaagaaa0G FE6NNE L (? :F76s;;;L%,..Hqqq$wZ]]2: & & 1F VF # # #FHVXG,,,,HZ( "[ 0&vFFFFFFrLr kv_cachesrrcH|j\}}} } |j||d| fksJ|j||d| fksJtj|} ||| |zf} |d} |d}|d}|d}|d}|d}t | ||||||| | | ||||||t | d} | dS)a Perform linear attention decoding using Triton kernels. Args: q: Query tensor of shape [B, H, 1, D] k: Key tensor of shape [B, H, 1, D] v: Value tensor of shape [B, H, 1, D] kv_caches: Key-value cache tensor slope_rate: Decay rate tensor slot_idx: Slot indices for batches BLOCK_SIZE: Size of blocks for processing Returns: output: Attention output tensor r rrn)rzb h n d -> b n (h d))rr empty_likestriderrsqueezer)r@rrHrrrrBH_rrrrrrrrrs rJlinear_decode_forward_tritonrsT0JAq!Q 7q!Ql " " " " 7q!Ql " " " " a F q!z/ "D88A;;L88A;;L%%a((N%%a((N&&q))O&&q))Ot$    &v5 6 6F >>!   ' ' ) ))rL)rN)r)reinopsrvllm.triton_utilsrrjit constexprrKrbrlr{autogradFunctionr}applyrTensorinttuplerrrrrLrJrs ((((((((}  | } |} |} |} <} L}}} }@iA  | iA  | iA |iA |iA <iAliAliA iA LiA  !iAiAiA iAX=+ | =+  | =+ |=+ |=+ <=+l=+l=+=+=+ =+@O  | O  | O |O |O <OlO LO OOO Od@B@B@B@B@B(@B@B@BH"'&* 77 |7 |7 |7  7  7  t# 7 5< %&7777tNG |NG NGNGNG NGp@*@* |@* |@* |@*| @*  @* l @*@* \@*@*@*@*@*@*rL