
    .`i1              F          d dl Z d dlmZ d dlmZ d dlmZmZ  ee          Z	 e j
         ej                              Zej        d             Zej        d             Zej        dej        dej        fd	            Zej        ej        ej        fd
ej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        d ej        d!ej        d"ej        d#ej        d$ej        d%ej        dej        d&ej        d'ej        d(ej        d)ej        d*ej        fDd+            Zej        d
ej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        dej        d ej        d!ej        d"ej        d#ej        d$ej        d%ej        dej        d&ej        d'ej        d,ej        dej        dej        f<d-            Zej        ej        ej        fd
ej        dej        dej        dej        dej        dej        dej        dej        d,ej        d(ej        d)ej        d*ej        fd.            Zd/ed0ed1efd2Zd/ed0ed3ed4ed1ef
d5Z	 	 	 	 	 	 	 	 	 	 	 d8d7ZdS )9    N)init_logger)current_platform)tltritonc                     | |z   dz
  |z  S )N    )xys     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/attention/ops/triton_unified_attention.pycdiv_fnr      s    EAI!    c                 z    | |z  }t          j        |          }t          j        |           }|||z
  z  ||z   z  S )N)r   exp)Sr
   Sdivp1p2s        r   apply_softcapr      s>    q5D	B	BR=BG$$r   BLOCK_Quse_q_block_modec                     d}|}||k     r?||z   dz  }t          j        | |z             }|r||z  |z   n|}	|	|k    r|dz   }n|}||k     ?|dz
  S )Nr      r   )r   load)
query_start_len_ptr
target_idxnum_seqsr   r   leftrightmidvalmid_vals
             r   find_seq_idxr#   !   s     DE
,,e|!g)C/00*:C#.3&&j  7DDE ,, !8Or   num_query_headsnum_queries_per_kvblock_table_stridequery_stride_0query_stride_1output_stride_0output_stride_1qq_bias_stride_0
BLOCK_SIZE	TILE_SIZE	HEAD_SIZEHEAD_SIZE_PADDEDUSE_ALIBI_SLOPESUSE_ALIBI_SQRTUSE_QQ_BIASUSE_SOFTCAP	USE_SINKSSLIDING_WINDOWUSE_MM_PREFIXMAX_MM_RANGESstride_k_cache_0stride_k_cache_1stride_k_cache_2stride_k_cache_3stride_v_cache_0stride_v_cache_1stride_v_cache_2stride_v_cache_3r   BLOCK_MUSE_FP8FP8_MINFP8_MAXc2           	         t          j        d          }2t          j        d          }3t          |+|2|-|,d          }4t          j        |+|4z             |,z  |4z   }5|2|5z
  }6t          j        |+|4z             }7t          j        |+|4z   dz             }8|8|7z
  }9|6|,z  |9k    rd S t          j        d|.          }:t          j        d|          };t          j        d|          }<|6|,z  |:|z  z   }=|7|=z   }>|3|z  |:|z  z   }?|>d d d f         |z  |?d d d f         |z  z   |;d d d f         z   }@t          j        |;|k     dd                              t           j                  }At          j        |=|9k     dd                              t           j                  }Bt          j        |?|k     dd                              t           j                  }Ct          j        ||@z   |Ad d d f         |Bd d d f         z  |Cd d d f         z  d          }D|4|z  }E|s0t          j        |.gt          d          t           j
                  }FnEt          j        ||?z   |Ct          d                                        t           j
                  }Ft          j        |.gdt           j
                  }Gt          j        |.|gt           j
                  }Ht          j        ||4z             }I|I|9z
  }J|rt          j        ||?z   |Cd          }K|r||=d d d f         |z  z   }L|J|6|,z  z   |.dz
  |z  z   dz   }M| rt          j        |M|I          }Mnt          j        |M|I          }Mt          |M|          }Nd}O|N}P|dk    rk| si|6|,z  }Qt          j        |Q|.dz
  |z  z   |9dz
            }R|J|Qz   |z
  dz   }S|J|Rz   }Tt          j        d|S|z            }Ot          j        |T|z  dz   |N          }Pt          |O|P          D ]X}U|U|z  |<z   }V|V|Mk     }Wt          j        ||Ez   |V|z  z                                 t           j                  }X|Xd d d f         |'z  |3|)z  z   |;d d d f         |*z  z   |V|z  d d d f         |(z  z   }Y|Xd d d f         |#z  |3|%z  z   |;d d d f         |&z  z   |V|z  d d d f         |$z  z   }Zt          j        ||Zz   |Ad d d f         |Wd d d f         z  d          }[|[j                                        ri|Dj                                        r|[}\nO|[                    t           j
                  t          j        |
          z                      |Dj                  }\n|[}\t          j        ||Yz   |Ad d d f         |Wd d d f         z  d          }]|]j                                        ri|Dj                                        r|]}^nO|]                    t           j
                  t          j        |          z                      |Dj                  }^n|]}^|J|=d d d f         z   }_|Vd d d f         |_k    }`|dk    r|`|_|Vz
  |k     z  }`| rt          |!          D ]}at          j        |"|4|!z  d	z  z   |ad	z  z             }bt          j        |"|4|!z  d	z  z   |ad	z  z   dz             }c|b|ck     }d|_|bk    |_|ck    z  |dz  }e|Vd d d f         |bk    |Vd d d f         |ck    z  |dz  }f|`|e|fz  z  }`t          j        |.|ft           j
        
          }g|g|	t          j        |D|\          z  z  }g|rt)          |g|          }gt          j        |Cd d d f         |Bd d d f         z  |`z  |gt          d                    }g|rw|r^|V|J|=d d d f         z   z
  }ht          j        |hdk    t          j        |h                     t           j
                             d          }in|V|Jz
  }i|g|Kd d d f         |iz  z  }g|rD|V|Jz
  }j|jdk    o|j|k     }kt          j        |L|jd d d f         z   |kd d d f         d          }l|g|lz  }gt          j        |Ft          j        |gd                    }mt          j        |mt          d          k    |md          }mt          j        |g|md d d f         z
            }nt          j        |nd          }ot          j        |F|mz
            }p|H|pd d d f         z  }H|G|pz  |oz   }G|m}F|r/|6|,z  }Qt          j        |J|Qz   |Vd d d f         z
  |k     |^d          }^|Ht          j        |n                    |^j                  |^          z  }HZ|H|Gd d d f         z  }H|/r-|Ht          j        |          z  }Ht          j        |H|0|1          }H|>d d d f         |z  |?d d d f         |z  z   |;d d d f         z   }qt          j        | |qz   |H|Ad d d f         |Bd d d f         z  |Cd d d f         z             d S )Nr   r   T        maskother-infdtype      ?r   shaperK   axisrG   )r   
program_idr#   r   arangewheretoint1fullfloatfloat32zerosmaximumminimumr   rangeint64rK   is_fp8dotr   sqrtmaxr   sumclampstore)r
output_ptr	query_ptrkey_cache_ptrvalue_cache_ptrsink_ptrblock_tables_ptrseq_lens_ptralibi_slopes_ptrqq_bias_ptrscalek_scalev_scale	out_scalesoftcapr$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   mm_prefix_range_ptrr8   r9   r:   r;   r<   r=   r>   r?   r   r   r   r@   rA   rB   rC   q_block_global_idxkv_head_idxseq_idxq_block_start_idxq_block_local_idxcur_batch_in_all_start_indexcur_batch_in_all_stop_indexcur_batch_query_lenoffs_moffs_doffs_t	query_posquery_offset_0query_offset_1query_offsetdim_maskquery_mask_0query_mask_1Qblock_table_offsetMLaccseq_lencontext_lenalibi_slopeqq_bias_row_ptrsmax_seq_prefix_len	num_tiles
tile_starttile_endqpos_loqpos_hifirst_allowed_keylast_allowed_keyj
seq_offset	tile_maskphysical_block_idxv_offsetk_offsetK_loadKV_loadVquery_abs_posseq_maskirange_start	range_endis_valid
q_in_range
k_in_ranger   relative_posalibi_offsetkey_rel_posis_query_keyqq_biasm_jPl_jalphaoutput_offsetsr                                                                                                                     r   kernel_unified_attention_2dr   8   s/   j q))-""K/7D G  3g =>>'IGS*->>#%7+>+H#I#I "$'*=*G!*K"L"L58TT7"&999Yq'""FYq*++FYq)$$F!G+f8J.JJI1I=N #55AS8SSNqqq$w.0
D
!N
2	3
qqq/	  x*Aq1144RW==H8I(;;QBBEEbgNNL8N_<aCCFFrwOOL 	L dAAAgaaag!66aaag9NN	 	 	A !#55 GWIuV}}BJ???G~%--
 
 
 "2:"

	 	
 		3bj111A
(G-.bj
A
A
AC glW,--G //K  
g~-L
 
 

  
)AAAtG,/??? 	 	
g
%	&Q;-
-	. 	   E  Z(:GDD  Z(:GDD
 *I66I JH-#g-*w{'999!#
 
 ('1NBQF&0Z#4	#ABB
:/9<A9MM :x(( X( X(]V+
!33	W11J*4LL
 

"RX,, 	
 qqq$w'*::,,-T111Wo 001 J&403CCD 	 tQQQw'*::,,-QQQWo 001 J&aaa03CCD 	 H$!!!T'"YtQQQw%77
 
 
 <   	w~~ KYYrz**RWW-=-==AA!'JJA h&$'"Yqqq$w%77
 
 
 <   	w~~ KYYrz**RWW-=-==AA!'JJA $i4&88dAAAg&-7 AMJ$>.#PQH  	4=)) 4 4 g''M*AA*EEAM  G''M*AA*EEAMPQQ 	 '2"k1$	13   aaa(K7!$'*i79 
 J33 HGY/rzBBB	URVAq\\!! 	*a))AHD!LD$99HDav
 
  
	5 8)[9QQQW;M-MN!x A%W|m//
;;<<<     *K7QQQW%44A 
	${2K&!+N>N0NLg ;tQQQw#77!$'*  G
 LA jBF11---.. hsU6]]*C55 F1s111d7|#$$ fQQ q3w E!!!T'N" IO 	''1Gw&AAAtG)<<NPQSV A
 	rvadd17mmQ''' !!!T'

C .BGI&&&hsGW-- 	qqq$w/1
D
!O
3	4
qqq/	  H]"dAAAgaaag!66aaag9NN     r   NUM_SEGMENTS_PER_SEQc/           	         t          j        d          }/t          j        d          }0t          j        d          }1t          |'|/|)|(d          }2t          j        |'|2z             |(z  |2z   }3|/|3z
  }4t          j        |'|2z             }5t          j        |'|2z   dz             }6|6|5z
  }7|4|(z  |7k    rd S t          j        ||2z             }8|+}9t	          |8|9|z            }:|1|:z  |z  |8k    rd S t          j        d|*          };t          j        d|          }<t          j        d|          }=|4|(z  |;|z  z   }>|5|>z   }?|0|z  |;|z  z   }@|?d d d f         |z  |@d d d f         |z  z   |<d d d f         z   }At          j        |<|k     dd                              t           j                  }Bt          j        |>|7k     dd                              t           j                  }Ct          j        |@|k     dd                              t           j                  }Dt          j        ||Az   |Bd d d f         |Cd d d f         z  |Dd d d f         z  d          }E|2|z  }F|r||1dk    rFt          j        ||@z   |Dt          d                                        t           j
                  }Gn_t          j        |*gt          d          t           j
                  }Gn/t          j        |*gt          d          t           j
                  }Gt          j        |*gd	t           j
                  }Ht          j        |*|gt           j
                  }I|8|7z
  }J|rt          j        |	|@z   |Dd          }K|r|
|>d d d f         |z  z   }L|J|4|(z  z   |*dz
  |z  z   dz   }Mt          j        |M|8          }Mt	          |M|          }Nd}O|N}P|dk    rk|,si|4|(z  }Qt          j        |Q|*dz
  |z  z   |7dz
            }R|J|Qz   |z
  dz   }S|J|Rz   }Tt          j        d|S|z            }Ot          j        |T|z  dz   |N          }Pt          t!          |1|:z  |O          t#          |1dz   |:z  |P                    D ]X}U|U|z  |=z   }V|V|Mk     }Wt          j        ||Fz   |V|z  z                                 t           j                  }X|Xd d d f         |#z  |0|%z  z   |<d d d f         |&z  z   |V|z  d d d f         |$z  z   }Y|Xd d d f         |z  |0|!z  z   |<d d d f         |"z  z   |V|z  d d d f         | z  z   }Zt          j        ||Zz   |Bd d d f         |Wd d d f         z  d          }[|[j                                        ri|Ej                                        r|[}\nO|[                    t           j
                  t          j        |          z                      |Ej                  }\n|[}\t          j        ||Yz   |Bd d d f         |Wd d d f         z  d          }]|]j                                        ri|Ej                                        r|]}^nO|]                    t           j
                  t          j        |          z                      |Ej                  }^n|]}^|J|>d d d f         z   }_|Vd d d f         |_k    }`|dk    r|`|_|Vz
  |k     z  }`|,rt          |-          D ]}at          j        |.|2|-z  dz  z   |adz  z             }bt          j        |.|2|-z  dz  z   |adz  z   dz             }c|b|ck     }d|_|bk    |_|ck    z  |dz  }e|Vd d d f         |bk    |Vd d d f         |ck    z  |dz  }f|`|e|fz  z  }`t          j        |*|ft           j
        
          }g|g|t          j        |E|\          z  z  }g|rt-          |g|          }gt          j        |Dd d d f         |Cd d d f         z  |`z  |gt          d                    }g|rw|r^|V|J|>d d d f         z   z
  }ht          j        |hdk    t          j        |h                     t           j
                             d          }in|V|Jz
  }i|g|Kd d d f         |iz  z  }g|rD|V|Jz
  }j|jdk    o|j|k     }kt          j        |L|jd d d f         z   |kd d d f         d          }l|g|lz  }gt          j        |Gt          j        |gd                    }mt          j        |mt          d          k    |md          }mt          j        |g|md d d f         z
            }nt          j        |nd          }ot          j        |G|mz
            }p|I|pd d d f         z  }I|H|pz  |oz   }H|m}G|r/|4|(z  }Qt          j        |J|Qz   |Vd d d f         z
  |k     |^d          }^|It          j        |n                    |^j                  |^          z  }IZ|?d d d f                             t           j                  ||+z  |z  z  |@d d d f         |+|z  z  z   |1|z  z   t          j        d|          d d d f         z   }qt          j        | |qz   |I|Bd d d f         |Cd d d f         z  |Dd d d f         z             |?                    t           j                  ||+z  z  |@|+z  z   |1z   }rt          j        ||rz   |G|C|Dz             t          j        ||rz   |H|C|Dz             d S )Nr   r   r   TrE   rF   rI   rJ   rL   rM   rO   rQ   )r   rR   r#   r   r   rS   rT   rU   rV   rX   rY   rW   rZ   r\   r[   r]   rb   minr^   rK   r_   r`   r   ra   r   rc   re   )ssegm_output_ptrsegm_max_ptrsegm_expsum_ptrrg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rs   r$   r%   r&   r'   r(   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r8   r9   r:   r;   r<   r=   r>   r?   r   r   r   r@   r   r6   r7   rt   ru   rv   segm_idxrw   rx   ry   rz   r{   r|   r   num_segmentstiles_per_segmentr}   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   segm_output_offsetsegm_offsetss                                                                                                                      r   kernel_unified_attention_3dr     s'   f q))-""K}QH/7D G  3g =>>'IGS*->>#%7+>+H#I#I "$'*=*G!*K"L"L58TT7"&999 glW,--G (L	)ABB##i/7::Yq'""FYq*++FYq)$$F!G+f8J.JJI1I=N #55AS8SSNqqq$w.0
D
!N
2	3
qqq/	  x*Aq1144RW==H8I(;;QBBEEbgNNL8N_<aCCFFrwOOL 	L dAAAgaaag!66aaag9NN	 	 	A !#55 
@q==>)!Fmm   brzb""	 A 	5==
CCCAAGWIuV}}BJ???
	3bj111A
(G-.bj
A
A
AC //K  
g~-L
 
 

  
)AAAtG,/??? 	 	
g
%	&Q;-
-	. 	  $6@@
 *I66I JH-#g-*w{'999!#
 
 ('1NBQF&0Z#4	#ABB
:/9<A9MM H((*55X\..99  Z( Z( ]V+
!33	W11J*4LL
 

"RX,, 	
 qqq$w'*::,,-T111Wo 001 J&403CCD 	 tQQQw'*::,,-QQQWo 001 J&aaa03CCD 	 H$!!!T'"YtQQQw%77
 
 
 <   	w~~ KYYrz**RWW-=-==AA!'JJA h&$'"Yqqq$w%77
 
 
 <   	w~~ KYYrz**RWW-=-==AA!'JJA $i4&88dAAAg&-7 AMJ$>.#PQH  	4=)) 4 4 g''M*AA*EEAM  G''M*AA*EEAMPQQ 	 '2"k1$	13   aaa(K7!$'*i79 
 J33 HGY/rzBBB	URVAq\\!! 	*a))AHD!LD$99HDav
 
  
	5 8)[9QQQW;M-MN!x A%W|m//
;;<<<     *K7QQQW%44A 
	${2K&!+N>N0NLg ;tQQQw#77!$'*  G
 LA jBF11---.. hsU6]]*C55 F1s111d7|#$$ fQQ q3w E!!!T'N" IO 	''1Gw&AAAtG)<<NPQSV A
 	rvadd17mmQ''' 	qqq$w""28,,114DD	F
D
!%9<L%L
M	N %
%	& )A'
(
(qqq
1		2  H,,dAAAgaaag!66aaag9NN    	"(##9M'MN
/
/	0
	 
 H\K'1LMMMMH_{*AL<4OPPPPPPr   c                 T   t          j        d          }t          j        d          }t          ||||d          }t          j        ||z             }|}t	          |||z            }t	          |||z            }t          j        d|          t          j        |g|t           j                  k     }t          j        t          j        d|          |k     dd          	                    t           j
                  }|	                    t           j                  ||z  z  ||z  z   t          j        d|          z   }t          j        ||z   |t          d                    }t          j        |          }t          j        ||z   |d          } | t          j        ||z
            z  } t          j        |           }!|	                    t           j                  ||z  |z  z  |||z  z  z   t          j        d|          d d d f         |z  z   t          j        d|          d d d f         z   }"t          j        ||"z   |d d d f         |d d d f         z  d          }#|#t          j        ||z
            d d d f         z  }#t          j        |#d          }$t          j        |!dk    d|$|!z            }%|r-|%t          j        |          z  }%t          j        |%||          }%||z  ||	z  z   t          j        d|          z   }&t          j        | |&z   |%|	           d S )
Nr   r   FrJ   rI   rF   rE   rO   rQ   )r   rR   r#   r   r   rS   rW   int32rT   rU   rV   r^   rX   rb   r   rc   rd   re   )'rf   r   r   r   rl   r   r$   out_scale_invr)   r*   r&   r-   r.   r/   r   r   r   rA   rB   rC   query_token_idxquery_head_idxrw   r   r   r   act_num_segments	segm_maskr   r   segm_maxoverall_maxsegm_expsumoverall_expsumr   segm_outputacc_sumr   r   s'                                          r   reduce_segmentsr     si   0 mA&&O]1%%N_h G
 glW,--G (L	)ABB w(9I(EFF	!122RW	 06 6 6 I x	!%566BAqIILLRWUUH 	28$$:N(NO
/
/	0
)A+
,
,	- 
 w|k1	vWWWH&""K '/K7isSSSKx+'= > >>KVK((N 	28$$114DD	F
03CC
D	E )A+
,
,QQQW
58H
H	I )A'
(
(qqq
1		2  ',,qqq$w(47"33  K
 26([011!!!T'::Kf[q)))G
(>S(#w/G
H
HC .BGM***hsGW-- 	/)
?
*	+
)A'
(
(	) 
 HZ-'8<<<<<<r   	head_sizesliding_windowreturnc                     |dk    o| dv S )a	  Detect Gemma3 models via unique (head_size, sliding_window) signature.

    Gemma3 models are the only ones using sliding_window=1024 with
    head_size 128 (27B) or 256 (1B, 4B, 12B). Other SWA models use
    different window sizes (Mistral=4096, Phi-3=2047).
    i   )      r	   )r   r   s     r   _is_gemma3_attentionr   R  s     T!=i:&==r   element_size
is_prefillc                 B    t          | |          rdS |rdS |dk    rdndS )zSelect tile size with Gemma3-specific optimization.

    For Gemma3, use 32 for both prefill and decode to better utilize
    the larger head dimension (128/256). For other models, use
    the default vLLM behavior.
        r      )r   )r   r   r   r   s       r   _get_tile_sizer   \  s>     I~66 r  r""22*r   Fc                 J   |	s
J d            |
J d            |&|j         d         | j         d         k    s
J d            d}d}|2|j        dk    rd}|j         d         }nt          d	|j                    |d u}|d u}|j         d         }t          |          } | j         d         }!|j         d
         }"|!|"z  }#| j         d
         }$|#dk    rdnt	          j        |#          }%|%|#z  }&| j         d         |&z  | z   }'|
d         dk    rd|
d         z   nd}(t          |$|(|                                 d          })t          |$|(|                                 d          }*||||||dk    s| |k    rt          |'|"f         dCi d|d| d|d|d|d|d|d|d|d|d|d|d|d|z  ndd|d|!d|#d|	                    d          d| 	                    d          d | 	                    d          d!|	                    d          d"|	                    d          d#|r|	                    d          ndd$|d%|)d&|$d't	          j        |$          d(|d)|d*|d+|dk    d,|d ud-|d.|d/|d0d|
d         z   d1|	                    d          d2|	                    d          d3|	                    d
          d4|	                    d          d5|	                    d          d6|	                    d          d7|	                    d
          d8|	                    d          d9|d:|&d;| d<|%d=|d u d S t          |'|"|f         dCi d>|d?|d@|d| d|d|d|d|d|d|d|d|d|d|d|d|!d|#d|	                    d          d| 	                    d          d | 	                    d          d#|r|	                    d          ndd$|d%|*d&|$d't	          j        |$          d(|d)|d*|d+|dk    d,|d ud-|d.|d/|d0d|
d         z   d1|	                    d          d2|	                    d          d3|	                    d
          d4|	                    d          d5|	                    d          d6|	                    d          d7|	                    d
          d8|	                    d          d9|d:|&d;| d<|%dA| t          | j         d         |!f         dCi d|d>|d?|d@|d|d;| d|!dB|d|z  ndd!|	                    d          d"|	                    d          d|	                    d          d%|*d&|$d't	          j        |$          d9|d:|&dA|d=|d u d S )DNz"Only causal attention is supportedzQ scales not supportedr   r   z"Sinks must be num_query_heads sizeF   Tz#Unsupported mm_prefix_range shape: r   r   )r   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rL   rs   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r6   r7   rt   r5   r8   r9   r:   r;   r<   r=   r>   r?   r   r   r   r@   rA   r   r   r   r   r   r	   )rN   ndim
ValueErrorlenr   next_power_of_2r   r   r   strider   r   )+qkvoutcu_seqlens_qmax_seqlen_q	seqused_kmax_seqlen_ksoftmax_scalecausalwindow_sizeblock_tablers   	q_descale	k_descale	v_descaleseq_threshold_3Dnum_par_softmax_segmentssoftmax_segm_outputsoftmax_segm_maxsoftmax_segm_expsumalibi_slopesoutput_scaler   sinksmm_prefix_rangeuse_alibi_sqrtuse_mm_prefixmax_mm_rangesuse_alibi_slopesuse_qq_bias
block_sizer   r$   num_kv_headsr%   r   r@   r   total_num_q_blockssliding_window_valTILE_SIZE_PREFILLTILE_SIZE_DECODEs+                                              r   unified_attentionr  r  s	   > 7777766{1~+++-Q+++MM"1$$ M+1!4MMMo6KMM   $4/%KJ9~~HgajO71:L(L8
I !B&&F,BCU,V,V  ++G w.9 0;1~/B/B[^++&		   &		   	 #+&#&!&&&#"	
 6	
 6	
 6	
 s6	
 a6	
 !6	
 A6	
 U6	
 )[6	
 #6	
 *\6	
  6	
  -6	
  I!6	
" I#6	
$ +7*Ba,&&%6	
& G'6	
( ,O)6	
*  21+6	
,  +11!444-6	
. 88A;;;/6	
0 88A;;;16	
2  JJqMMM36	
4  JJqMMM56	
6 3>DW^^A...176	
8 "z96	
: (';6	
<  i=6	
> $3I>>>?6	
@ .-A6	
B *>C6	
D $E6	
F !1G6	
H D((I6	
J (-K6	
L (-M6	
N !0O6	
P A..Q6	
R XXa[[[S6	
T XXa[[[U6	
V XXa[[[W6	
X XXa[[[Y6	
Z XXa[[[[6	
\ XXa[[[]6	
^ XXa[[[_6	
` XXa[[[a6	
b !-c6	
d Ge6	
f Xg6	
h Gi6	
j !,,k6	
 6	
 6	
 6	
p 	$/GH	
 2	
 2	
 2	
 0/2	
 *)	2	

 0/2	
 a2	
 !2	
 A2	
 U2	
 )[2	
 #2	
 *\2	
  2	
  -2	
 I2	
  I!2	
" G#2	
$ ,O%2	
&  21'2	
(  +11!444)2	
* 88A;;;+2	
, 88A;;;-2	
. 3>DW^^A...1/2	
0 "z12	
2 '&32	
4  i52	
6 $3I>>>72	
8 .-92	
: *>;2	
< $=2	
> !1?2	
@ D((A2	
B (-C2	
D (-E2	
F !0G2	
H A..I2	
J XXa[[[K2	
L XXa[[[M2	
N XXa[[[O2	
P XXa[[[Q2	
R XXa[[[S2	
T XXa[[[U2	
V XXa[[[W2	
X XXa[[[Y2	
Z !-[2	
\ G]2	
^ X_2	
` Ga2	
b ":!9c2	
 2	
f 	_56 	
 	
 	
s	
//	
 *)	
 0/		

 #	
 X	
 ,O	
 /;.F!l**C	
  JJqMMM	
  JJqMMM	
  +11!444	
 '&	
  i	
 $3I>>>	
 !-	
  G!	
" ":!9#	
$ !,,%	
 	
 	
 	
r   )NNNNNNNNNNF)torchvllm.loggerr   vllm.platformsr   vllm.triton_utilsr   r   __name__loggerfinfo	fp8_dtypefloat8_infojitr   r   	constexprr#   r   rb   r^   r   r   r   r   intboolr   r   r  r	   r   r   <module>r     sy    # # # # # # + + + + + + ( ( ( ( ( ( ( (	X		ek4*46677    % % %  \	
 l   , d (O'OeX X \X  !X" #X$ H%X& H'X( X)X* X+X, h-X. /X0 |1X2 |3X4 l5X6 l7X8 L9X: ;X< =X> |?X@ LAXB <CXD <EXH hIXJ hKXL hMXN lOXP hQXR hSXT hUXV lWXZ \[X\ h]X^ \_X` \aXb \cXd \eX X X Xv
 `Q" \#`Q$ %`Q& '`Q( H)`Q* H+`Q, h-`Q. /`Q0 |1`Q2 |3`Q4 l5`Q6 l7`Q8 L9`Q: ;`Q< =`Q> |?`Q@ LA`QB hC`QD hE`QF hG`QH lI`QJ hK`QL hM`QN hO`QP lQ`QT \U`QV hW`QX \Y`QZ ,[`Q\ <]`Q^ <_`Q `Q `Q `QF * (O'O+V= V= \V= XV= XV= V= |V= |V= lV=" \#V=$ ,%V=& \'V=( \)V=* \+V= V= V= V=r>C > > > > > >+++ + 	+
 	+ + + +N !
;e
 e
 e
 e
 e
 e
r   