
    -`ir/                        d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZmZ d dlmZ d dlmZ d dlmZm Z   ee!          Z" e j#        d           G d d                      Z$ G d d          Z%e j#         G d d                      Z&e j#         G d d                      Z' G d d          Z(dS )    N)Counter)Callable)	ExitStack)Any)patch)compilation_counter)$validate_cudagraph_capturing_enabled)CUDAGraphMode
VllmConfig)set_graph_pool_id)BatchDescriptorget_forward_context)init_logger)current_platform)current_streamweak_ref_tensorsT)frozenc                   8    e Zd ZU eed<   eed<   eed<   eed<   dS )CUDAGraphStatnum_unpadded_tokensnum_padded_tokensnum_paddingsruntime_modeN)__name__
__module____qualname__int__annotations__str     o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/compilation/cuda_graph.pyr   r      sB         r!   r   c                       e Zd ZdZg dZdedee         dz  ddfdZddZ	d	e
ddfd
ZdefdZej        fdedef         ddfdZdS )CUDAGraphLoggingz#Aggregate and log cudagraph metrics)zUnpadded TokenszPadded TokenszNum PaddingszRuntime ModeCountcg_modecg_capture_sizesNreturnc                     |                                   t          |          | _        t          |pg           | _        d| j         d| j         d| _        d S )Nz(**CUDAGraph Config Settings:**

- Mode: z
- Capture sizes: z

**CUDAGraph Stats:**

)resetr   r&   r'   settings_header)selfr&   r'   s      r"   __init__zCUDAGraphLogging.__init__-   sk     	

7|| #$4$: ; ;'|' ' $ 5' ' ' 	r!   c                     g | _         d S N)statsr,   s    r"   r*   zCUDAGraphLogging.reset;   s    *,


r!   cudagraph_statc                 :    | j                             |           d S r/   )r0   append)r,   r2   s     r"   observezCUDAGraphLogging.observe>   s    
.)))))r!   c           
         t          | j                  }g }t          |                                d d          D ]g\  }}|                    t          |j                  t          |j                  t          |j                  |j	        t          |          g           hg }t          | j                  D ]Q\  }}t          |          }|D ]%}	t          |t          |	|                             }&|                    |           Rd t          | j        |          D             }
dd                    |
          z   dz   }dd                    d	 |D                       z   d
z   }g }|D ]J}	d t          |	|          D             }|                    dd                    |          z   dz              K| j        |z   |z   d                    |          z   dz   S )Nc                     | d         S )N   r    )items    r"   <lambda>z8CUDAGraphLogging.generate_metric_table.<locals>.<lambda>G   s
    47 r!   T)keyreversec                 >    g | ]\  }}|                     |          S r    )ljust).0hws      r"   
<listcomp>z:CUDAGraphLogging.generate_metric_table.<locals>.<listcomp>[   s3     
 
 
1aAGGAJJ
 
 
r!   z| z | z |
|c              3   &   K   | ]}d |dz   z  V  dS )-   Nr    )r?   rA   s     r"   	<genexpr>z9CUDAGraphLogging.generate_metric_table.<locals>.<genexpr>`   s*      (K(K1A(K(K(K(K(K(Kr!   z|
c                 X    g | ]'\  }}t          |                              |          (S r    )r   r>   )r?   valwidths      r"   rB   z:CUDAGraphLogging.generate_metric_table.<locals>.<listcomp>e   s;       *4#uCu%%  r!   z |
)r   r0   sorteditemsr4   r   r   r   r   r   	enumerateCOLUMN_HEADERSlenmaxzipjoinr+   )r,   stats_countsrowsstatcount
col_widthsiheader_text	max_widthrowtable_header_listtable_headertable_separator	data_rowsformatted_rows                  r"   generate_metric_tablez&CUDAGraphLogging.generate_metric_tableA   s?   tz** !  &:&:D
 
 
 	 	KD% KK011.//)**%JJ    
'(;<< 	) 	)NA{K((I 8 8	3s1v;;77		i((((
 
#&t':J#G#G
 
 
 ejj):;;;fD(K(K
(K(K(K K KKeS 	 	F 	FC 8;C8L8L  M TEJJ}$=$==DEEEE   ii	""# 		
r!   log_fn.c                 z    | j         sd S  ||                                            |                                  d S r/   )r0   rb   r*   )r,   rc   s     r"   logzCUDAGraphLogging.logr   s>    z 	Ft))++,,,

r!   )r(   N)r   r   r   __doc__rO   r
   listr   r-   r*   r   r5   r   rb   loggerinfor   r   re   r    r!   r"   r$   r$   "   s        --  N
$
8<S	D8H
	
 
 
 
- - - -*m * * * * */
s /
 /
 /
 /
b 06{  (38, t      r!   r$   c                   v    e Zd ZU eed<   dZej        j        dz  ed<   dZ	e
dz  ed<   dZee         dz  ed<   dS )CUDAGraphEntrybatch_descriptorN	cudagraphoutputinput_addresses)r   r   r   r   r   rm   torchcuda	CUDAGraphrn   r   ro   rg   r   r    r!   r"   rk   rk   y   sg         %%%%-1Iuz#d*111FC$J )-OT#Y%,,,,,r!   rk   c                   :    e Zd ZU dZeed<   dZeed<   dZeed<   dS )CUDAGraphOptionsTdebug_log_enableF
gc_disableweak_ref_outputN)r   r   r   ru   boolr   rv   rw   r    r!   r"   rt   rt      sC         !d!!!J OT     r!   rt   c                       e Zd ZdZ	 ddedef         dedededz  ddf
d	Z	d
e
defdZdedef         fdZdedededz  fdZdS )CUDAGraphWrappera9  Wraps a runnable to add CUDA graph capturing and replaying ability. And
    provide attribute access to the underlying `runnable` via `__getattr__`.

    The workflow of this wrapper in the cudagraph dispatching is as follows:
    1. At initialization, a runtime mode is assigned to the wrapper (FULL or
    PIECEWISE).
    2. At runtime, the wrapper receives a runtime_mode and a
    batch_descriptor(key) from the forward context and blindly trust them
    for cudagraph dispatching.
    3. If runtime_mode is NONE or runtime_mode does not match the mode of the
    wrapper, just call the runnable directly.
    4. Otherwise, i.e., the runtime_mode matches the mode of the wrapper,
    the wrapper will perform cudagraph capture(if key does not exist, create
    a new entry and cache it) or replay (if key exists in the cache).

    Note: CUDAGraphWrapper does not store persistent buffers or copy any
    runtime inputs into that buffers for replay. We assume implementing them
    is done outside of the wrapper. That is because we do not make any
    assumption on the dynamic shape (batch size) of the runtime inputs, as a
    trade-off for staying orthogonal to compilation logic. Nevertheless,
    tracing and checking the input addresses to be consistent during replay is
    guaranteed when VLLM_LOGGING_LEVEL == "DEBUG".
    Nrunnable.vllm_configr   cudagraph_optionsr(   c                    || _         || _        || _        |j        | _        d| _        t
          j        dk    | _        | j        t          j	        k    sJ t          j                    | _        |t                      }|| _        i | _        d S )NFDEBUG)r{   r|   r   compilation_configfirst_run_finishedenvsVLLM_LOGGING_LEVELis_debugging_moder
   NONEr   get_global_graph_pool
graph_poolrt   r}   concrete_cudagraph_entries)r,   r{   r|   r   r}   s        r"   r-   zCUDAGraphWrapper.__init__   s     !&("-"@"'!%!8G!C  M$66666 +@BB$ 0 2 2!2 RT'''r!   r;   c                     t          | j        |          rt          | j        |          S t          d| d| j                   )Nz
Attribute z2 not exists in the runnable of cudagraph wrapper: )hasattrr{   getattrAttributeError)r,   r;   s     r"   __getattr__zCUDAGraphWrapper.__getattr__   sZ    4=#&& 	/4=#...2 2 2"&-2 2
 
 	
r!   c                     | j         S r/   )r{   r1   s    r"   unwrapzCUDAGraphWrapper.unwrap   s
    }r!   argskwargsc                 F   t                      }|j        }|j        }|t          j        k    s|| j        k    r | j        |i |S |J || j        vrt          |          | j        |<   | j        |         }|j	        | j
        j        r+t                              d| j        j        |j                   t                       d |D             }||_        t"          j                                        }t)                      5 }	| j
        j        rH|	                    t/          dd                      |	                    t/          dd                      | j        t3          | j                   n t3          t5          j                               t"          j                            || j        t;                                5   | j        |i |}
| j
        j        rt?          |
          }
d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t?          |
          |_         ||_	        tB          xj"        d	z  c_"        |
S | j#        r,d
 |D             }||j        k    sJ d|j         d|             |j	        $                                 |j         S )N)rl   z Capturing a cudagraph on (%s,%s)c                 j    g | ]0}t          |t          j                  |                                1S r    
isinstancerp   Tensordata_ptrr?   xs     r"   rB   z-CUDAGraphWrapper.__call__.<locals>.<listcomp>   sA       !"jEL.I.I

  r!   z
gc.collectc                      d S r/   r    r    r!   r"   r:   z+CUDAGraphWrapper.__call__.<locals>.<lambda>  s    D r!   ztorch.cuda.empty_cachec                      d S r/   r    r    r!   r"   r:   z+CUDAGraphWrapper.__call__.<locals>.<lambda>  s    PT r!   )poolstreamr8   c                 j    g | ]0}t          |t          j                  |                                1S r    r   r   s     r"   rB   z-CUDAGraphWrapper.__call__.<locals>.<listcomp>+  sA     # # #!"jEL.I.I#

# # #r!   zEInput addresses for cudagraphs are different during replay. Expected z, got )%r   rl   cudagraph_runtime_moder
   r   r   r{   r   rk   rm   r}   ru   rh   debugnamer	   ro   rp   rq   rr   r   rv   enter_contextr   r   r   r   graph_pool_handlegraphr   rw   r   rn   r   num_cudagraph_capturedr   replay)r,   r   r   forward_contextrl   r   entryro   rm   stackrn   new_input_addressess               r"   __call__zCUDAGraphWrapper.__call__   s   -//*;!0!G #m&888%)::: !4=$1&111+++4#BBB@N!1A A AD+,<= /0@A?"%6 	
 6%**   1222 &*  O %4E!
,,..I :)4 W ''lLL(I(IJJJ''.F(U(UVVV?.%do6666%&6&H&J&JKKKZ%%)++ &   : : +T]D;F;;F-= : "2&!9!9: : : : : : : : : : : : : : :!: : : : : : : : : : : : : : :D ,F33EL'EO66!;66
 M! 		# #&*# # # '%*????-+0+@- -*- - @?? 	   |s7   CH)G<0H<H 	 HH 	HHHr/   )r   r   r   rf   r   r   r   r
   rt   r-   r   r   r   r   r    r!   r"   rz   rz      s         : 6:T T38$T  T $	T
 ,d2T 
T T T T<
s 
s 
 
 
 
c*    fc fS fS4Z f f f f f fr!   rz   ))dataclassescollectionsr   collections.abcr   
contextlibr   typingr   unittest.mockr   rp   	vllm.envsr   vllm.compilation.counterr   vllm.compilation.monitorr	   vllm.configr
   r   6vllm.distributed.device_communicators.pynccl_allocatorr   vllm.forward_contextr   r   vllm.loggerr   vllm.platformsr   vllm.utils.torch_utilsr   r   r   rh   	dataclassr   r$   rk   rt   rz   r    r!   r"   <module>r      s[             $ $ $ $ $ $                                8 8 8 8 8 8 I I I I I I 1 1 1 1 1 1 1 1 T T T T T T E E E E E E E E # # # # # # + + + + + + C C C C C C C C	X		 d###       $#T T T T T T T Tn - - - - - - - - ! ! ! ! ! ! ! !j j j j j j j j j jr!   