
    .`i                         d dl mZmZ d dlZd dlZd dlmZmZ d dl	m
Z
 d dlmZ d dlmZ  G d d          Z G d	 d
          Z G d d          Z G d d          Zej        dej        fd            ZdS )    )IterableSequenceN)tltriton)next_power_of_2)is_uva_available)get_cuda_view_from_cpu_tensorc                   :    e Zd Zdeee         z  dej        fdZdS )	UvaBuffersizedtypec                     t                      st          d          t          j        ||dd          | _        | j                                        | _        t          | j                  | _        d S )NzUVA is not availablecpuTr   device
pin_memory)	r   RuntimeErrortorchzerosr   numpynpr	   uva)selfr   r   s      s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/worker/gpu/buffer_utils.py__init__zUvaBuffer.__init__   sa    !! 	75666;t54PPP(..""0::    N)__name__
__module____qualname__intr   r   r   r    r   r   r   r      sA        ;S8C=0 ; ; ; ; ; ; ;r   r   c                       e Zd Z	 ddeee         z  dej        defdZdej        e	j
        z  ez  dej        fdZ	 ddej        e	j
        z  d
ej        d	z  dej        fdZd	S )UvaBufferPool   r   r   max_concurrencyc                     | _         | _        || _        fdt          |          D             | _        d| _        d S )Nc                 0    g | ]}t                    S r!   )r   ).0_r   r   s     r   
<listcomp>z*UvaBufferPool.__init__.<locals>.<listcomp>#   s#    QQQQ)D%00QQQr   r   )r   r   r%   range	_uva_bufs_currr   r   r   r%   s    `` r   r   zUvaBufferPool.__init__   sP     	
. RQQQQ%:P:PQQQ


r   xreturnc                     | j         dz   | j        z  | _         | j        | j                  }t          |t          j                  r|j        n|j        }t          |          }||d |<   |j	        d |         S )N   )
r-   r%   r,   
isinstancer   Tensorr   r   lenr   )r   r/   bufdstns        r   copy_to_uvazUvaBufferPool.copy_to_uva'   sk    j1n(<<
nTZ(#Au|44@cgg#&FFBQBwrr{r   Noutc                     |                      |          }||                                S |                    |d          S )NT)non_blocking)r9   clonecopy_)r   r/   r:   r   s       r   copy_to_gpuzUvaBufferPool.copy_to_gpu1   s@    
 q!!;99;;yy4y000r   r$   N)r   r   r   r    r   r   r   r   r4   r   ndarraylistr9   r?   r!   r   r   r#   r#      s        
  !	 HSM! { 	   U\BJ6= %,     $(
1 
1<"*$
1 \D 
1 
	
1 
1 
1 
1 
1 
1r   r#   c                   d    e Zd Z	 d
deee         z  dej        defdZddedz  dej        fd	Z	dS )UvaBackedTensorr$   r   r   r%   c                    || _         || _        t          j        ||dd          | _        | j                                        | _        t          |||          | _        | j        	                    | j                  | _
        d S )Nr   Fr   )r   r%   r   r   r   r   r   r#   poolr9   gpur.   s       r   r   zUvaBackedTensor.__init__?   st     
. ;t55QQQ(.."" "$??	9((11r   Nr8   r0   c                 z    | j                             || j        d |         n| j                  | _        | j        S rA   )rG   r9   r   rH   )r   r8   s     r   r9   zUvaBackedTensor.copy_to_uvaP   s4    9((!47SSxr   r@   rA   )
r   r   r   r    r   r   r   r   r4   r9   r!   r   r   rE   rE   >   s        
  !	2 2HSM!2 {2 	2 2 2 2" S4Z 5<      r   rE   c                       e Zd Z	 	 ddeee         z  dej        dej        dedef
dZ	d	ed
ede
e         e
e         z  ddfdZd	ededdfdZddZddZdS )StagedWriteTensorr$   Fr   r   r   r%   uva_instead_of_gpuc                    t           j        t           j        t           j        g}||vrt	          d| d|           t          |t                    r|n|d         | _        || _        || _	        |st          j
        |||          | _        n&t          ||          | _        | j        j        | _        g | _        g | _        g | _        g | _        t'          | j        t           j        |          | _        t'          | j        t           j        |          | _        t-          | j                  }t'          |||          | _        t'          | j        t           j        |          | _        d S )NzUnsupported dtype z: should be one of r   )r   r   r   r%   )r   int32int64float32
ValueErrorr3   r    num_rowsr   r%   r   rH   r   _uva_bufr   _staged_write_indices_staged_write_starts_staged_write_contents_staged_write_cu_lensr#   write_indiceswrite_startsr   write_contentswrite_cu_lens)r   r   r   r   r%   rL   supported_dtypes	init_sizes           r   r   zStagedWriteTensor.__init__W   sx    "KemD(((QUQQ?OQQ   !+4 5 5B47
.! 	){4uVDDDDHH &dE22DM}(DH02"/1!9;#02"*Mo
 
 
 *Mo
 
 
 $DM22	+UO
 
 
 +Mo
 
 
r   indexstartr/   r0   Nc                 "   |dk    sJ |dk    sJ |sd S | j                             |           | j                            |           | j                            |           | j                            t          | j                             d S Nr   )rU   appendrV   rW   extendrX   r5   )r   r_   r`   r/   s       r   stage_writezStagedWriteTensor.stage_write   s     zzzzzzzz 	F"))%000!((///#**1---"))#d.I*J*JKKKKKr   c                 
   |dk    sJ | j                             |           | j                            d           | j                            |           | j                            t          | j                             d S rb   )rU   rc   rV   rW   rX   r5   )r   r_   r/   s      r   stage_write_elemz"StagedWriteTensor.stage_write_elem   s|    zzzz"))%000!((+++#**1---"))#d.I*J*JKKKKKr   c           	         t          | j                  }|dk    rd S | j                            | j                  }| j                            | j                  }| j                            | j                  }t          | j                  }t          | j
        j        t                    sJ || j
        j        k    rNt          |          }t          || j        | j                  | _
        t"          j                                         | j
                            | j                  }t)          |f         | j        | j                            d          ||||d           |                                  d S )Nr   rN   i   )
BLOCK_SIZE)r5   rU   rY   r9   rZ   rV   r\   rX   rW   r3   r[   r   r    r   r#   r   r%   r   cudasynchronize_apply_write_kernelrH   strideclear_staged_writes)r   r8   indices_uva
starts_uvacu_lens_uvadiff_lennew_sizecontents_uvas           r   apply_writezStagedWriteTensor.apply_write   sh   *++66F(44T5OPP&2243LMM
(44T5OPP t233$-2C88888d)...&x00H"/
D<P# # #D J""$$$*66t7RSS 	QD!HHOOA	
 	
 	
 	
 	  """""r   c                     | j                                          | j                                         | j                                         | j                                         d S rA   )rU   clearrV   rW   rX   )r   s    r   rn   z%StagedWriteTensor.clear_staged_writes   s\    "((***!'')))#))+++"((*****r   )r$   F)r0   N)r   r   r   r    r   r   r   r   boolr   r   floatre   rg   ru   rn   r!   r   r   rK   rK   V   s         !#(+
 +
HSM!+
 {+
 	+

 +
 !+
 +
 +
 +
ZLL L C=8E?*	L
 
L L L LLc Lc Ld L L L L%# %# %# %#N+ + + + + +r   rK   ri   c                    t          j        d          }t          j        ||z             }t          j        ||z             }	|dk    rt          j        ||z   dz
            nd}
t          j        ||z             }||
z
  }t          d||          D ]_}|t          j        d|          z   }||k     }t          j        ||
z   |z   |          }t          j        | ||z  z   |	z   |z   ||           `d S )Nr   r2   )mask)r   
program_idloadr+   arangestore)
output_ptroutput_stridewrite_indices_ptrwrite_starts_ptrwrite_contents_ptrwrite_cu_lens_ptrri   pidrow_idx	start_idxcu_startcu_endcontent_leniblockr{   contents                    r   rl   rl      s!    -

Cg'#-..G(3.//I7:Qwwrw(3.2333AHW&,--F8#K1k:.. 
 
BIa,,,{"',x7%?dKKK
=009<uDgTX	
 	
 	
 	
 	
	
 
r   )collections.abcr   r   r   r   r   vllm.triton_utilsr   r   vllm.utils.math_utilsr   vllm.utils.platform_utilsr   vllm.utils.torch_utilsr	   r   r#   rE   rK   jit	constexprrl   r!   r   r   <module>r      sd   / . . . . . . .      ( ( ( ( ( ( ( ( 1 1 1 1 1 1 6 6 6 6 6 6 @ @ @ @ @ @; ; ; ; ; ; ; ;$1 $1 $1 $1 $1 $1 $1 $1N       0o+ o+ o+ o+ o+ o+ o+ o+d 
 
 
 
 
 
 
r   