
    .`ij              *          d dl Z d dlmZ d dlmZmZ  ej        d          rdndZ ej                    rdndZ	 ej
                    d	k    Z e j         ej                              Zej        d d ej        ej        fd
ej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        f(d            Zej        dedej        dej        dej        dej        dej        dej        fd            Z e j                    	 	 	 	 	 	 	 d&d!ed"e j        d#e j        d$efd%            ZdS )'    N)current_platform)tltritonP      @         )      xstride_k_cache_blnum_queries_per_kvIN_PRECISIONBLOCK_MBLOCK_DMODELBLOCK_DMODEL_PADDED
BLOCK_SIZEPHYSICAL_BLOCK_SIZEBLOCK_NSLIDING_WINDOWnum_unroll_cachenum_unroll_requestSKIP_DECODE	USE_SINKSUSE_FP8	MAX_Q_LENMAX_CTX_LENFP8_MINFP8_MAXc8           
         t          j        d          }8t          j        d          }9t          j        d          }:|9|&z  };t          j        ||8z             }<t          j        ||8z             }=t          j        ||8z   dz             }>|>|=z
  }?|<|?z
  }@|1r|?dk    rd S |(|:z  }At          j        d|+          }Bt          j        d|-          }Ct          j        d|*          }D|:|(z  t          j        d|(          z   }E|=|Ed d d f         z   |z  |9|z  z   |Dd d d f         |z  z   }Ft          j        t          j        d|*          |)k     dd                              t           j                  }Gt          j        | |Fz   |Gd d d f         |Ed d d f         |?k     z  d          }H|2sQt          j        |(gt          d          t           j	                  }It          j
        |(gt           j	                  }Jnt          j        |t          j        |(g|9t           j                  z   |E|?k     t          d                                        t           j	                  }It          j        |It          d          k    dd          }Jt          j
        |(|*gt           j	                  }Kt          j        d|@|+|/	          D ]}L|L|Bz   }M|M|,z  }Nt          j        ||8|z  z   |N|z  z                                 t           j                  }O|M|,z  }P|Od d d f         |z  |;|z  z   |Dd d d f         |z  |z  z   |Pd d d f         | z  z   |Dd d d f         |z  |!z  z   }Q|Od d d f         |"z  |;|#z  z   |Dd d d f         |$z  z   |Pd d d f         |%z  z   }R|L|+z   |@k    s|)|*k    r9t          j        ||Qz   |Gd d d f         |L|Bd d d f         z   |@k     z  d          }Snt          j        ||Qz             }S|Sj                                        rM|S                    t           j	                  t          j        |          z                      |Hj                  }Tn|S}T|t          j        |H|T|'
          z  }Ut          j        |L|Bd d d f         z   |@k     |Ut          d                    }U|.dk    rDt          j        |@|Ed d d f         z   |L|Bd d d f         z   z
  |.k     |Ut          d                    }Ut          j        |It          j        |Ud                    }Vt          j        |U|Vd d d f         z
            }Wt          j        |Vd d d f         t          d          k    d|W          }Wt          j        |Wd          }Xt          j        |I|Vz
            }Yt          j        |It          d          k    d|Y          }Y|K|Yd d d f         z  }K|L|+z   |@k    s|)|*k    r9t          j        ||Rz   |Gd d d f         |L|Bd d d f         z   |@k     z  d          }Znt          j        ||Rz             }Z|Zj                                        rM|Z                    t           j	                  t          j        |	          z                      |Hj                  }[n|Z}[|W                    |[j                  }Wt          j        |W|[|K|'          }K|J|Yz  |Xz   }J|V}I|Cd d d f         |z  |;|z  z   |Dd d d f         |z  z   }Q|Cd d d f         |z  |;|z  z   |Dd d d f         |z  z   }R||Qz   }\||Rz   }]t          j        |A|?k     dd          }^t          j        d|^|:dz   z  |(z  |-|0	          D ]r}Lt          j        |L|-          }Lt          j        |\|=|Lz   |z  z   |Gd d d f         |L|Cd d d f         z   |?k     z  d          }Tt          j
        |(|-gt           j	                  }Ut          j        |H|T|U|'          }U|U|z  }Ut          j        |Ed d d f         |L|Cd d d f         z   k    |Ut          d                    }U|.dk    rAt          j        |Ed d d f         |L|Cd d d f         z   z
  |.k     |Ut          d                    }Ut          j        |It          j        |Ud                    }Vt          j        |U|Vd d d f         z
            }Wt          j        |Vd d d f         t          d          k    d|W          }Wt          j        |Wd          }Xt          j        |I|Vz
            }Yt          j        |It          d          k    d|Y          }Y|K|Yd d d f         z  }Kt          j        |]|=|Lz   |z  z   |Gd d d f         |L|Cd d d f         z   |?k     z  d          }[|W                    |[j                  }Wt          j        |W|[|K|'          }K|J|Yz  |Xz   }J|V}It|K|Jd d d f         dz   z  }K|=|Ed d d f         z   |z  |9|z  z   |Dd d d f         |z  z   }_||_z   }`|3r-|Kt          j        |
          z  }Kt          j        |K|6|7          }Kt          j        |`|K|Gd d d f         |Ed d d f         |?k     z             d S )Nr                 maskother-infdtype      ?)loop_unroll_factor)input_precision)axisaccr-   g|=r&   )r   
program_idloadarangewheretoint1fullfloatfloat32zerosint64ranger*   is_fp8dotmaximummaxexpsummultiple_ofclampstore)aQKVK_cacheV_cachesink_ptrB_Locsm_scalek_scalev_scaleout_scale_invB_Start_LocB_Seqlenr   Outstride_b_loc_bstride_b_loc_s
stride_qbs	stride_qh	stride_qd
stride_kbs	stride_kh	stride_kd
stride_vbs	stride_vh	stride_vd
stride_obs	stride_oh	stride_odstride_k_cache_bsstride_k_cache_hstride_k_cache_dr   stride_k_cache_xstride_v_cache_bsstride_v_cache_hstride_v_cache_dstride_v_cache_blr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    	cur_batchcur_headstart_mcur_kv_headcur_batch_seq_lencur_batch_in_all_start_indexcur_batch_in_all_stop_indexcur_batch_query_lencur_batch_ctx_lenblock_start_loc	offs_bs_noffs_noffs_doffs_moff_qdim_maskqm_il_ir0   start_ntoken_indicesbn_logical_indicesbninternal_offsetsoff_koff_vk_loadkqkm_ijpl_ijalphav_loadvk_ptrsv_ptrs
block_maskoff_oout_ptrssa                                                                                                    x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/attention/ops/prefix_prefill.py_fwd_kernelr   $   s   v a  I}QHmAG00K9 455#%7;+B#C#C "$'+	*AA*E"F"F58TT),?? *a// 'O 	!Z((IYq'""FYq-..Fw1g!6!66F 
&qqq$w	7:E
Y
	
qqq/I
%	& 
 x	!%899LH!QOORR
 H 		E	dAAAg&D/4G"GH	 	 	A  	6gwivbjAAAhy
333grwy("(CCCC..--
 
 
 "2:"

	 	
 hsU6]]*C55
(G01
D
D
DC 8	j=M   o o  )+*.AA WI..1Cn1TT
 

"RX,, 	 )+>> tQQQwK++,,-aaag!#'778 tQQQw'*;;< aaag"&66	7 	 qqq$wK++,,-T111Wo 001 qqq$w'*;;< 	 j #444222W%aaag&iaaa004EEG	  FF WWu_--F<   	2:&&)9)99==agFFAAA q!\BBBBXyqqq))->>E&MM
 
 A "VAAAtG_49TSTSTSTWCU9UV !f	 B z#rvbq11122F2QQQW%&&HT!!!T']eFmm3S!<<vaa   sTz""f-sE::E!!!T'N" j #444222W%dAAAg&i4004EEG	  FF WWu_--F<   	2:&&)9)99==agFFAAADDMMfQsLAAAEkD  	tQQQw*$
	
!	"
D/I
%	& 
 	qqq$w*$
	
!	"
qqq/I
%	& 

 YFYF /,??AFFJ 8	gk"W,-	   1 1 .'22G2W<
JJ!!!T'"&qqq/)-@@B	
 
 
 Xw(
;;;VAqb,???
hXfQQQWo'F47O*CDb%PV--XXAqqq$w7VD!!!G_#<=Nf B z#rvbq11122F2QQQW%&&HT!!!T']eFmm3S!<<vaa   sTz""f-sE::E!!!T'N" G2W<
JJ$'"&D/)-@@B	
 
 
 DDMMfQsLAAAEkD 
QQQW%
&C 
&qqq$w	7:E
Y
	
qqq/I
%	& 

 U{H .BGM***hsGW--H#HT111W-4CV1VW    F    c-           
         t          j        d          }-t          j        d          }.t          j        d          }/|.|&z  }0t          j        |
|-z             }1t          j        |	|-z             }2t          j        |	|-z   dz             }3|3|2z
  }4|1|4z
  }5|,r|4dk    rd S |(|/z  }6t          j        d|+          }7t          j        d|*          }8|/|(z  t          j        d|(          z   }9|2|9d d d f         z   |z  |.|z  z   |8d d d f         |z  z   }:t          j        t          j        d|*          |)k     dd                              t           j                  };t          j        | |:z   |;d d d f         |9d d d f         |1|5z
  k     z  d          }<t          j        |(gt           j                  t          d          z
  }=t          j        |(gt           j                  }>t          j        |(|*gt           j                  }?t          j        ||.z             }@t          j        d|(          |6z   |5z   }Ad}Bt          d|5|+          D ]}Ct          j        |C|+          }Ct          j        ||-|z  z   |C|7z   |z  |z  z   |C|7z   |5k     d                              t           j                  }D|Dd d d f         |z  |0|z  z   |8d d d f         |z  |z  z   |C|7d d d f         z   |z  | z  z   |8d d d f         |z  |!z  z   }E|Dd d d f         |"z  |0|#z  z   |8d d d f         |$z  z   |C|7d d d f         z   |z  |%z  z   }Ft          j        ||Ez   |;d d d f         |C|7d d d f         z   |5k     z  d          }G|Gj                                        rM|G                    t           j                  t          j        |          z                      |<j                  }Hn|G}Ht          j        |(|+gt           j                  }It          j        |<|H|I|'          }It          j        |C|7d d d f         z   |5k     |It          d	                    }I|I|z  }It          j        d|+          d d d f         |Bz   |Ad d d f         z
  |@z  }Jt          j        |Jdk    |Ad d d f         |1k     z  |Jt          d	                    }J|I|Jz  }I|B|+z  }Bt          j        |Id          }Kt          j        |=|K          }Lt           j                            |I|Ld d d f         z
            }Mt          j        |Md          }Nt           j                            |=|Lz
            }O|O|>z  |Nz   }P|O}Q|?|Qd d d f         z  }?t          j        ||Fz   |;d d d f         |C|7d d d f         z   |5k     z  d          }R|Rj                                        rM|R                    t           j                  t          j        |          z                      |<j                  }Sn|R}S|M                    |Sj                  }Mt          j        |M|S|?d
          }?|P}>|L}=|7d d d f         |z  |0|z  z   |8d d d f         |z  z   }E|7d d d f         |z  |0|z  z   |8d d d f         |z  z   }F||Ez   }T||Fz   }Ut          j        |6|1|5z
  k     dd          }Vt          j        ||.z             }@t          j        d|(          |6z   |5z   }A|5}Bt          d|V|/dz   z  |(z  |+          D ]g}Ct          j        |C|+          }Ct          j        |T|2|Cz   |z  z   |;d d d f         |C|7d d d f         z   |1|5z
  k     z  d          }Ht          j        |(|+gt           j                  }It          j        |<|H|Id
          }I|I|z  }It          j        |9d d d f         |C|7d d d f         z   k    |It          d	                    }It          j        d|+          d d d f         |Bz   |Ad d d f         z
  |@z  }Jt          j        |Jdk    |Ad d d f         |1k     z  |Jt          d	                    }J|I|Jz  }I|B|+z  }Bt          j        |Id          }Kt          j        |=|K          }Lt           j                            |I|Ld d d f         z
            }Mt          j        |Md          }Nt           j                            |=|Lz
            }O|O|>z  |Nz   }P|O}Q|?|Qd d d f         z  }?t          j        |U|2|Cz   |z  z   |;d d d f         |C|7d d d f         z   |1|5z
  k     z  d          }S|M                    |Sj                  }Mt          j        |M|S|?d
          }?|P}>|L}=i|?|>d d d f         z  }?|2|9d d d f         z   |z  |.|z  z   |8d d d f         |z  z   }W||Wz   }Xt          j        |X|?|;d d d f         |9d d d f         |1|5z
  k     z             d S )Nr   r"   r#   r$   r%   r)   infr/   r(   ieeer1   )r   r2   r3   r4   r5   r6   r7   r;   r:   r9   r=   rD   r<   r*   r>   r?   rA   r@   mathrB   rC   rF   )YrG   rH   rI   rJ   rK   rM   rN   rO   rP   rR   rS   Alibi_slopes
block_sizer   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   r   rf   rg   rh   ri   rj   r   r   r   r   r   r   r   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   rv   rw   rx   ry   rz   r{   r|   r}   r0   alibi_slopealibi_start_qalibi_start_kr~   r   r   r   r   r   r   alibir   m_i_newr   r   r   l_i_new	acc_scaler   r   r   r   r   r   r   sY                                                                                            r   _fwd_kernel_alibir   b  s   b a  I}QHmAG00K
 9 455#%7;+B#C#C "$'+	*AA*E"F"F58TT),?? *a//'O Yq'""FYq-..Fw1g!6!66F	%qqq$w	7:E
Y
	
qqq/I
%	& 
 x	!%899LH!QOORR
 H 		E	dAAAg!!!T'?.1BBBD		 	 	A (G9BJ
/
/
/%,,
>C
(G9BJ
/
/
/C
(G01
D
D
DC',122KIa))O;>OOMM-w77 S S.'22W.()& Z/>AB F"&77
 
 
 "RX,, 	 tQQQwK++,,-aaag!#'778 &qqq/)Z7;LLM aaag"&66	7 	 qqq$wK++,,-T111Wo 001 4(J69JJK 	 eO!!!T'"waaa'@DU&UV
 
 
 <   	2:&&)9)99==agFFAAAXw(
;;;VAqb,???XvdAAAg&*;;Rv
 
 	h Ia!!$'*]:]111d7=SS aZM!!!T'25FFG&MM
 

 	e  vb!}}*S$''GKKWQQQW--..va|| C'M**#+$ 	Iaaag&&eO$'"w4'@DU&UV
 
 

 <   	2:&&)9)99==agFFAAADDMMfQsF;;; 	tQQQw*$
	
!	"
D/I
%	& 
 	qqq$w*$
	
!	"
qqq/I
%	& 

 YFYF/,=@Q,QQSTVWXXJ ',122KIa))O;>OOM%M
 J'A+6@'JJ 6 6.'22G2W<
JJ!!!T'"&qqq/)->AR-RRT	
 
 
 Xw(
;;;VAqb&999
hXfQQQWo'F47O*CDb%PV--XX Ia!!$'*]:]111d7=SS aZM!!!T'25FFG&MM
 

 	e  vb!}}*S$''GKKWQQQW--..va|| C'M**#+$ 	Iaaag&&G2W<
JJ$'"&D/)->AR-RRT	
 
 
 DDMMfQsF;;;
AAAtG
C 
&qqq$w	7:E
Y
	
qqq/I
%	& 

 U{HHdAAAg!!!T'?.1BBBD    Fr   Fkv_cache_dtyperO   rP   is_block_table_ptrc           !        +, | j         t          j        u }t          r|rdnd }d|v r|j         t          j        t          j                    fv sJ |j         t          j        t          j                    fv sJ |dv rt          j                    }n#|dk    rt          j        }nt          d|          |	                    |          }|	                    |          }|j         t          j        k    s|j         t          j        k    r|dk    rt          d          | j
        d         |j
        d         |j
        d         }}}||k    r||k    sJ t          j        |          }|d	|d
z  z  }|	j
        d         | j
        d         c+,| j
        d         |j
        d         z  }+dz   t          |          k    sJ ||dk    rd}|r|                                }|                    d          |z  }|                                }|dk    } t          j        | ||z
  |z  |                              t          j                  }!n|                    t          j                  }!|||
J d            |
J d            |r
t(          dz  nt(          }"+,t          j        |"          f}#t-          |#         g | ||||||||||	||j
        d         |j
        d         ||                    d          |                    d          |                     d          |                     d          |                     d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          R |||"|||"|t.          dd	 d S |
dn|
}
i }$t          j                    ri }$|j
        d         }%|%dk    o|%|%dz
  z  dk    }&|&rd}'d}(nd}'d}(d})+,fd}*t3          |*         | ||||||!||||d	|z  nd	||	|j
        d         ||!                    d          |!                    d          |                     d          |                     d          |                     d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          |                    d          fi d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d |)d!|%d"|d#|d$|d%|d&|d'|d(|d ud)|'d*|(d+dd,dd-dd.dd/|d u|$ d S )0Nr   fp8)r   fp8_e4m3fp8_e5m2zUnsupported FP8 dtype:autozLkv_cache_dtype='auto' unsupported for            FP8 KV Cache prefill kernelr+   g      ?r   r"   z%Sinks arg is not supported with alibiz#FP8 output not supported with alibir#      r	   )	r   r   r   r   r   r   r   	num_warps
num_stagesr   r       c                 @    t          j        | d                   fS )Nr   )r   cdiv)METAbatchheadmax_input_lens    r   <lambda>z'context_attention_fwd.<locals>.<lambda>$  s    E4]DO)T)TU r   rc   rd   re   r   rf   rg   rh   ri   rj   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r*   torchr:   	IS_TURINGuint8r   	fp8_dtypefloat8_e5m2
ValueErrorviewshaper   next_power_of_2lenelement_sizestridedata_ptrr5   r6   int32
BASE_BLOCKr   r   	NUM_WARPSis_rocmr   )-r{   r   r   or   k_cachev_cacheb_locb_start_loc	b_seq_lenmax_seq_lenr   rO   rP   alibi_slopessliding_windowrN   skip_decodefp8_out_scalesinksr   q_dtype_is_f32r   target_dtypeLqLkLv	Lk_paddedr   kv_element_sizeblock_byte_stride	base_addrr&   processed_b_locBLOCKgridextra_kargsreal_block_sizeis_pow2r   r   TRITON_BLOCK_SIZEgrid_fnr   r   s-              `                               @@r   context_attention_fwdr   z  s@	   0 W-N 'C>C66tL }.>.H.J.J KKKKK}.>.H.J.J KKKKK000+577LLz)) ,LL5~FFF,,|,,,,|,, 	$$=EK''f$$)
 
 	
 agbk172;BB88b &r**I"c'?/!$agajKE4qwqz119K(((((( 1!4!4 0!..00#NN1--?$$&&	qy+59$)::E
 

"U[// 	  ((5;//}}E}}}$$&K$$$ $2A
aztV[>>?$ 0	
0	
0	
 0	
 	0	

 0	
 0	
 0	
 0	
 0	
 0	
 0	
 0	
 M!0	
 M!0	
 0	
  LLOO!0	
" LLOO#0	
$ HHQKK%0	
& HHQKK'0	
( HHQKK)0	
* HHQKK+0	
, HHQKK-0	
. HHQKK/0	
0 HHQKK10	
2 HHQKK30	
4 HHQKK50	
6 HHQKK70	
8 HHQKK90	
: HHQKK;0	
< NN1=0	
> NN1?0	
@ NN1A0	
B NN1C0	
D NN1E0	
F NN1G0	
H NN1I0	
J NN1K0	
L NN1M0	
 0	
N  2% )#_0	
 0	
 0	
 0	
b 	"*!!KK!! mA&O!T/A:M'NRS'SG  
 UUUUUUG			,8mca	q!!q!!												;8 8 8< "..+++=8> !***?8@ !***A8B "..+++C8D !***E8F "..+++G8H !***I8J !***K8L "..+++M8N %$O8P ,OQ8R .-S8T "\U8V RW8X &IY8Z &~[8\  K]8^ T))_8` a8b c8d e8f 1g8h !i8j 1k8l t##
o8 8 8r Fr   )NNNFNNF)r   vllm.platformsr   vllm.triton_utilsr   r   has_device_capabilityr   r   r   get_device_capabilityr   finfor   float8_infojitminrA   	constexprr   intr   inference_modestrTensorboolr    r   r   <module>r      s*    + + + + + + ( ( ( ( ( ( ( ( ;$:2>>FSSB
)!)++2AA	 3244>	ek4*46677$ l   !'O'Oqz z 
|zB |CzN OzP ,QzR \SzT ,UzV WzX YzZ [z\ \]z^ L_z` lazb czd ezf |gzh \izj |kzl mzn \ozp \qz z z zz	 TN OTP ,QTR \STT ,UTV WTX \YTZ [T T T Tn   
$+c c
 c \c \c* +c c c c c cr   