
    .`iL                        d Z ddlZddlmZ ddlmZ ddlmZmZ  ej	                    Z
 ej        e          Z ej        ej                   ej        d          k     re                    d           ej        d             Zej        d	ej        d
ej        dej        dej        dej        dej        dej        dej        dej        fd            Zd Zej        d	ej        dej        d
ej        dej        dej        dej        dej        dej        dej        dej        dej        dej        fd            Zd Zej        dej        dej        dej        fd            Zd Z	 d dZ	 d dZ	 	 d!dZdS )"zF
Memory-efficient attention for decoding.
It supports page size >= 1.
    N)version)current_platform)tltritonz3.2.0zUThe following error message 'operation scheduled before its operands' can be ignored.c                 <    dt          j        d| z            z  dz
  S )N      )r   sigmoid)xs    /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/attention/ops/triton_decode_attention.pytanhr   4   s"     rz!a%   1$$    kv_group_numBLOCK_DMODELBLOCK_DVBLOCK_NNUM_KV_SPLITS	PAGE_SIZE	logit_capLkLvc           	      ^   t          j        d          }t          j        d          }t          j        d          }||z  }t          j        d|          }t          j        d|          }||k     } ||k     }!t          j        ||z             }"|}#||z  ||	z  z   |z   }$t          j        | |$z   | d          }%t          j        |"|          }&|&|z  }'t          j        |'|&z   |"          }(t          d           })d}*t          j        |gt           j                  }+|(|'k    rt          |'|(|          D ]},|,t          j        d|          z   }-t          j        |||#z  z   |-|z  z   |-|(k     d          }.|.|z  |-|z  z   }/|/d d d f         |
z  ||z  z   |d d d f         z   }0t          j        ||0z   |-d d d f         |(k     | d d d f         z  d          }1t          j
        |%d d d f         |1z  d          }2|2|z  }2|dk    r|t          |2|z            z  }2t          j        |-|(k     |2t          d                    }2|/d d d f         |z  ||z  z   |d d d f         z   }3t          j        ||3z   |-d d d f         |(k     |!d d d f         z  d          }4t          j        t          j        |2d          |)          }5t          j        |)|5z
            }6t          j        |2|5z
            }7|+|6z  }+|+t          j
        |7d d d f         |4z  d          z  }+|*|6z  t          j
        |7d          z   }*|5})||z  ||z  z   ||z  z   |z   }8t          j        ||8z   |+|*z  |!	           ||z  ||z  z   ||z  z   |z   }9t          j        ||9z   |)t          j        |*          z              d S d S )
Nr   r	   r           maskotherinfdtype-infr   )r   
program_idarangeloadcdivminimumfloatzerosfloat32rangesumr   wheremaximummaxexpstorelog):QK_BufferV_Buffersm_scaleReq_to_tokensB_SeqlenAtt_Outstride_req_to_tokens_b
stride_qbs	stride_qhstride_buf_kbsstride_buf_khstride_buf_vbsstride_buf_vhstride_mid_obstride_mid_ohstride_mid_osr   r   r   r   r   r   r   r   r   	cur_batchcur_headsplit_kv_idcur_kv_headoffs_doffs_dvmask_dmask_dvcur_batch_seq_lencur_batch_req_idxoff_qqkv_len_per_splitsplit_kv_startsplit_kv_ende_maxe_sumaccstart_noffs_nkv_page_numberkv_loc
offs_buf_kkqk
offs_buf_vvn_e_maxre_scalep
offs_mid_ooffs_mid_o_1s:                                                             r   _fwd_kernel_stage1rc   :   s   : a  I}QH-""Kl*KYq,''Fi8$$Gb[FlG9 455!
"X	%99FBE
E	c222Aw0-@@%3N:n/??ARSSL5\\MEE
(H:RZ
0
0
0Cn$$^\7CC .	 .	GryG444FW(+<<=I%& l*  N $i/&92DDFqqq$w.0-.qqq/" 
 :%QQQWo4aaaI  A
 $'
Q**B(NB1}}b9n!5!55&</U6]]CCB qqq$w.0-.$'"# 
 :%QQQWo4qqq9IJ  A jA66Gvego..HrG|$$A8OC26!AAAtG*q.!,,,CH$rva||3EEE %&'M)*  	 	j %K	
 	
 	
 	
 %&'M)*  	 	l"BF5MM!	
 	
 	
 	
 	
K %$r   c
                    t           sdnd}
|}|j        d         }|j        d         }| j        d         | j        d         }}|||f}| j        d         |j        d         z  }d}|dk    rt           rdnd}t          j        |          }t          j        |          }t	          |         | |||||||                    d          |                     d          |                     d          |                    d	          |                    d          |                    d	          |                    d          |                    d          |                    d          |                    d          f||||
|||	|d||d
 d S )N@      r   r	      r   )r   r   r   r   r   r   r   	num_warps
num_stagesr   r   )is_hip_shaper   next_power_of_2rc   stride)rN   k_bufferv_bufferatt_outr6   r7   num_kv_splitsr5   	page_sizer   BLOCKr   r   r   batchhead_numgridr   rk   r   r   s                        r   _decode_att_m_fwdrz      s    $BB1E!M		B		Bgaj!'!*8E8]+D71:!33LIq 'AAa	)"--L%b))Ht	Q		qqq#$ "!#9     r   
q_head_num	BLOCK_DPEBLOCK_Hc           	         t          j        d          }t          j        d          }|t          j        ||          z  }t          j        d          } ||k    r|n|}!||!z  t          j        d|          z   }"|"|dz   |!z  k     }#|#|"|k     z  }#t          j        d|          }$t          j        d|          }%|$|k     }&|%|k     }'t          j        ||z             }(|})||z  |"d d d f         |	z  z   |$d d d f         z   }*t          j        | |*z   |#d d d f         |&d d d f         z  d          }+|dk    rq|t          j        d|          z   },|,|k     }-||z  |"d d d f         |	z  z   |,d d d f         z   }.t          j        | |.z   |#d d d f         |-d d d f         z  d          }/t          j        |(|          }0|0| z  }1t          j        |1|0z   |(          }2t          j        |gt           j                  t          d          z
  }3t          j        |gt           j                  }4t          j        ||gt           j                  }5|2|1k    rt          |1|2|          D ]}6|6t          j        d|          z   }7t          j        |||)z  z   |7|z  z   |7|2k     d          }8|8|z  |7|z  z   }9|9d d d f         |
z  ||z  z   |$d d d f         z   }:t          j        ||:z   |7d d d f         |2k     |&d d d f         z  d          };t          j
        |+|;                    |+j                            }<|dk    r|9d d d f         |
z  ||z  z   |,d d d f         z   }=t          j        ||=z   |7d d d f         |2k     |-d d d f         z  d          }>|<t          j
        |/|>                    |/j                            z  }<|<|z  }<|dk    r|t          |<|z            z  }<t          j        |#d d d f         |7d d d f         |2k     z  |<t          d                    }<|9d d d f         |z  ||z  z   |%d d d f         z   }?t          j        ||?z   |7d d d f         |2k     |'d d d f         z  d          }@t          j        t          j        |<d          |3          }At          j        |3|Az
            }Bt          j        |<|Ad d d f         z
            }C|5|Bd d d f         z  }5|5t          j
        |C                    |@j                  |@          z  }5|4|Bz  t          j        |Cd          z   }4|A}3||z  |"d d d f         |z  z   | |z  z   |%d d d f         z   }Dt          j        ||Dz   |5|4d d d f         z  |#d d d f         |'d d d f         z  	           ||z  |"|z  z   | |z  z   |z   }Et          j        ||Ez   |3t          j        |4          z   |#	           d S d S )
Nr   r	   r   r   r   r   r   r    r!   )r   r"   r%   r#   r$   r&   r(   r)   r'   r*   dottor   r   r,   r-   r.   r/   r+   r0   r1   )Fr2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   r   r{   r   r|   r   r   r}   r   r   r   r   r   rC   cur_head_idrF   rE   VALID_BLOCK_HrD   mask_hrG   rH   rI   rJ   rK   rL   offs_qrN   offs_dpemask_dpeoff_qpeqperO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   offs_buf_kpekper\   r]   r^   r_   r`   ra   rb   sF                                                                         r   _fwd_grouped_kernel_stage1r      s"   @ a  I-""Kw!?!??K-""K-9G-C-C''M]*RYq'-B-BBHqM99Fx*,-FYq,''Fi8$$Gb[FlG9 455!#hqqq$w&7)&CCfTSTSTSTWoUF
F
&D/fT111Wo!FcRRRA1}}")Ay"9"99b=
"Xaaag%6%BBXdTUTUTUgEVV 	 gKvaaag8D!!!G3DES
 
 
 w0-@@%3N:n/??ARSSLHgYbj111E%LL@EHgYbj111E
(GX&bj
9
9
9Cn$$^\7CC <	 <	GryG444FW(+<<=I%& l*  N $i/&92DDFtQQQw.0-.D/" 
 :%T111Wo44I  A
 144==))B1}}47On4!M12qqq$w'( 
 g|+ qqq/L8Xaaag=NO  
 bfS#&&"3"3444(NB1}}b9n!5!55qqq$w6$'?\#ABBf B
 qqq$w.0-.$'"# 
 :%QQQWo4qqq9IJ  A jA66Gvego..HrGAAAtG,,--A8AAAtG$$C26!$$qw--+++CH$rva||3EEE %qqq$w-/0M)* dAAAg 	 	j %4. D/gdAAAg&67	
 	
 	
 	
 %&'M)*  	 	l"BF5MM!	
 	
 	
 	
 	
 	
g %$r   c
                    d}
|j         d         }|j         d         }t          r|dk    rd}
|dk    rd}d}n!|dk    rd}d}nt          j        |          }d	}t          j        |          }| j         d	         | j         d
         }}| j         d
         |j         d         z  }d}|}|t          j        |t          ||                    |f}i }d}t          rd
ddd}d
}t          |         | |||||||                    d	          |                     d	          |                     d
          |                    d          |                    d          |                    d          |                    d          |                    d	          |                    d
          |                    d          f||||||
||||	d|||d| d S )N    rg   i@     i   re   i      r   r	   rh   r   waves_per_eumatrix_instr_nonkdimkpackrj   ri   )r   r{   r   r|   r   r   r}   r   r   r   rk   rl   r   r   )rn   rm   r   ro   r%   minr   rp   )rN   rq   rr   rs   r6   r7   rt   r5   ru   r   rv   r   r   r   r|   r   rw   rx   r   r}   r   ry   extra_kargsrl   s                           r   _decode_grouped_att_m_fwdr     s    E		B		B  299	Syy			s		-b11	%b))Hgaj!'!*8E71:!33LG!MHc'<8899D KJ  ()"qQQ
t$	Q		qqq#!$ "!#?! !@ A! ! ! ! !r   c                    t          j        d          }t          j        d          }t          j        ||z             }t          j        d|          }||k     }d}t	          d           }t          j        |gt           j                  }||z  ||z  z   |z   }||z  ||z  z   |z   }t          d|
          D ]}t          j        ||
          }||z  }t          j	        ||z   |          }||k    rt          j        | |z   ||z  z   |d          }t          j        | |z   ||z  z             }t          j
        ||          }t          j        ||z
            }||z  }t          j        ||z
            }|||z  z  }||z  |z   }|}t          j        |||z  z   ||z  z   |z   ||z  |           |t          j        |          z   } t          j        |||	z  z   |z   |            d S )Nr   r	   r   r   r   r   r!   )r   r"   r$   r#   r'   r(   r)   r*   r%   r&   r-   r/   r0   r1   )!Mid_Oolser7   r@   rA   rB   
stride_obs	stride_ohstride_lse_bsr   r   r   rC   rD   rK   rG   rI   rS   rR   rT   offs_v
offs_logicrE   rO   rP   rQ   tvtlogicr^   	old_scale	exp_logiclse_vals!                                    r   _fwd_kernel_stage2r     s=     a  I}QH9 455Yq(##Fb[FE5\\ME
(H:RZ
0
0
0C&M)AAFJF]*X-EEJJQ..  7#4mDD)K7z.3C"CEVWW.((}!<<6QT  B WUZ/+2MMNNFj//Guw//I9Cv/00I9r>!CI%	1EEH	I
""X	%99FBe   
 bfUmm#GHi-''(2    r   c                    |j         d         |j         d         }}|j         d         }	t          j        |	          }
|}i }t          rdddd}||f}t	          |         | ||||                     d          |                     d          |                     d          |                    d          |                    d          |                    d          f
||
|	ddd| d S )	Nr   r	   rg   ri   r   r   r   )r   r   r   rk   rl   )rn   r   ro   rm   r   rp   )logitsrN   r   r   rr   	b_seq_lenrt   rw   rx   r   r   r   r   ry   s                 r   _decode_softmax_reducev_fwdr   -  s	    gaj!'!*8E		B%b))H!MK R ()"qQQ8Dt	aaa		

1 $   !    r   r   c                 `    t          | |||||||	|
|
  
         t          || |||||           d S N)rz   r   rN   rq   rr   r   r   req_to_tokenr   attn_logitsrt   r5   ru   r   s               r   decode_attention_fwd_normalr   W  sa     	    Q3)]    r   c                 `    t          | |||||||	|
|
  
         t          || |||||           d S r   )r   r   r   s               r   decode_attention_fwd_groupedr   v  sa     	    Q3)]    r   r	   c                     ||j         d         k    sJ | j         d         |j         d         z  }|dk    rt          | |||||||||	|
|           d S t          | |||||||||	|
|           d S )Nr   r	   rh   )rn   r   r   )rN   rq   rr   r   r   r   r   r   rt   r5   ru   r   r   s                r   decode_attention_fwdr     s     K-a0000071:!33Lq#	
 	
 	
 	
 	
  	%	
 	
 	
 	
 	
r   )r   )r	   r   )__doc__logging	packagingr   vllm.platformsr   vllm.triton_utilsr   r   is_rocmrm   	getLogger__name__loggerparse__version__warningjitr   	constexprrc   rz   r   r   r   r   r   r   r    r   r   <module>r      s  6 
        + + + + + + ( ( ( ( ( ( ( (
"

"
$
$		8	$	$ 7=#$$}w}W'='===
NN	   % % %
 |
$ ,%|
& ,'|
( l)|
* \+|
, <-|
. |/|
0 |1|
2 	3|
4 	5|
 |
 |
 |
~; ; ;| \
$ ,%\
& '\
( ,)\
* |+\
, l-\
. \/\
0 \1\
2 <3\
4 |5\
6 |7\
8 	9\
: 	;\
 \
 \
 \
~S S Sl ; <; l; 	; ; ; ;|' ' 'l    V    T 0
 0
 0
 0
 0
 0
r   