
    .`i                      b   U d dl Z d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ  e	e          Zi aeed<   daeed	<   g aeed
                  ed<    G d d
          ZdefdZdefdZd Z eej                  Z eej                  Z eej                  Z eej                   Z! eej"                  Z# eej$                  Z% eej&                  Z' eej(                  Z)d Z*d Z+	 ddedej,        j-        dej,        j-        dee         de j.        de/dee         fdZ0dS )    N)Optional)forward_context)ForwardContext)init_logger)current_stream_THREAD_ID_TO_CONTEXT   _NUM_UBATCHESUBatchContext_CURRENT_CONTEXTSc                      e Zd ZdZ	 d dedej        j        dej        j        dede	j
        de	j        d	e	j        d
ej        dej        defdZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdS )!r   zT
    Context manager for micro-batching synchronization using threading events.
    defaultidcomm_streamcompute_streamr   ready_barriercpu_wait_eventcpu_signal_eventgpu_comm_done_eventgpu_compute_done_eventschedulec                     || _         || _        || _        || _        || _        || _        || _        || _        || _        |	| _	        |
| _
        d | _        d S N)r   r   r   r   r   r   r   r   r   r   r   	recv_hook)selfr   r   r   r   r   r   r   r   r   r   s              l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/vllm/v1/worker/ubatching.py__init__zUBatchContext.__init__   se     &,.*, 0,#6 &<#     c                 V   | j         t          t          j                    <   | t          | j         <   | j                                         | j                                         | j                                         | 	                                 | 
                    | j                   | S r   )r   r   	threading	get_identr   r   waitr   clear_restore_contextupdate_streamr   r   s    r   	__enter__zUBatchContext.__enter__4   s    7;wi1334%)$'"!!!  """!!###4.///r   c                     d t           | j        <   t          t          j                    = |                                  | j                                         | j        	                                 dS )NF)
r   r   r   r    r!   maybe_run_recv_hookr   setr   r#   )r   exc_typeexc_valexc_tbs       r   __exit__zUBatchContext.__exit__B   sa    %)$'"!)"5"7"78  """!!###!!###ur   c                 (    | j         t           _        d S r   )r   _forward_contextr&   s    r   r$   zUBatchContext._restore_contextK   s    +/+?(((r   c                     || _         t                      | j         k    r&t          j                            | j                    d S d S r   )r   torchcuda
set_stream)r   streams     r   r%   zUBatchContext.update_streamN   sE    $t222J!!$"566666 32r   c                 D    | j                             | j                   d S r   )r   recordr   r&   s    r   _signal_comm_donezUBatchContext._signal_comm_doneS   s"     ''(899999r   c                 D    | j                             | j                   d S r   )r   r7   r   r&   s    r   _signal_compute_donez"UBatchContext._signal_compute_doneV   s"    #**4+>?????r   c                 D    | j                             | j                   d S r   )r   
wait_eventr   r&   s    r   _wait_compute_donez UBatchContext._wait_compute_doneY   s"    ##D$?@@@@@r   c                 D    | j                             | j                   d S r   )r   r<   r   r&   s    r   _wait_comm_donezUBatchContext._wait_comm_done\   s"    &&t'?@@@@@r   c                 Z   t           j        | j         k    sJ t                      | j        k    sJ | j                                        rJ | j                                         | j                                         | j                                         | 	                                 d S r   )
r   r0   r   r   is_setr   r*   r"   r#   r$   r&   s    r   
_cpu_yieldzUBatchContext._cpu_yield_   s     /43GGGGG4#66666&--/////!!###  """!!###r   c                 :    |                      | j                   d S r   )r%   r   r&   s    r   switch_to_commzUBatchContext.switch_to_comml   s    4+,,,,,r   c                 :    |                      | j                   d S r   )r%   r   r&   s    r   switch_to_computezUBatchContext.switch_to_computeo   s    4./////r   c                     |                                   |                     | j                   |                                  d S r   )r:   r%   r   r=   r&   s    r   switch_to_comm_syncz!UBatchContext.switch_to_comm_syncr   sC    !!###4+,,,!!!!!r   c                     |                                   |                     | j                   |                                  d S r   )r8   r%   r   r?   r&   s    r   switch_to_compute_syncz$UBatchContext.switch_to_compute_syncw   sC       4.///r   c                 N    | j         |                                   d | _         d S d S r   )r   r&   s    r   r)   z!UBatchContext.maybe_run_recv_hook|   s-    >%NN!DNNN &%r   c                     t                      | _         |                                  |                     | j                    d S r   )r   rB   r%   r&   s    r   yield_zUBatchContext.yield_   s=    ,..4./////r   c                    t                      | j        k    sJ |                                  |                                  | j         | j        k    sJ |                     | j                   |                                  d S r   )r   r   r:   rB   r%   r   r=   r&   s    r   %yield_and_switch_from_compute_to_commz3UBatchContext.yield_and_switch_from_compute_to_comm   s    4#66666!!###"d&999994+,,,!!!!!r   c                    t                      | j        k    sJ |                                  |                                  | j         | j        k    sJ |                     | j                   |                                  d S r   )r   r   r8   rB   r%   r   r?   r&   s    r   %yield_and_switch_from_comm_to_computez3UBatchContext.yield_and_switch_from_comm_to_compute   s    4#33333   "d&666664.///r   Nr   )__name__
__module____qualname____doc__intr2   r3   Streamr   r    BarrierEventstrr   r'   r.   r$   r%   r8   r:   r=   r?   rB   rD   rF   rH   rJ   r)   rM   rO   rQ    r   r   r   r      s         "  Z& 
)	
 ( !( " $/ #[ !&    4    @ @ @7 7 7
: : :@ @ @A A AA A A     - - -0 0 0" " "
  
" " "
0 0 0
" " "    r   returnc                  2    t          t                    dk    S Nr   )lenr   r\   r   r   dbo_enabledra      s    $%%))r   c                  r    t          t                    dk    rdS t          t          j                             S r_   )r`   r   r    r!   r\   r   r   dbo_current_ubatch_idrc      s/    
 !!Q&&q !4!6!677r   c                       fd}|S )Nc                      t          t                    dk    r9t          t          j                             }t          |         } |g| R i | d S d S r_   )r`   r   r    r!   r   )argskwargsctx_idxctxfuncs       r   wrapperz*_register_ubatch_function.<locals>.wrapper   sb    $%%))+I,?,A,ABG#G,CD&t&&&v&&&&& *)r   r\   )rj   rk   s   ` r   _register_ubatch_functionrl      s#    ' ' ' ' ' Nr   c                     t          t                    dk    r?t          t          j                             }t          |dz   t
          z           }| |_        d S d S )Nr      )r`   r   r    r!   r   r
   r   )r   rh   next_ctxs      r   dbo_register_recv_hookrp      sR    
 !!A%%'	(;(=(=>$gk]%BC& &%r   c                    t          t                    dk    rqt          t          j                             }t          |         }t
          j                            |j                  5   | |i |cd d d            S # 1 swxY w Y   d S d S r_   )	r`   r   r    r!   r   r2   r3   r5   r   )rj   rf   rg   rh   ri   s        r   dbo_get_previous_eventrr      s    
 !!A%%'	(;(=(=>(Zs122 	) 	)4(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	)	 &%s   (A==BBr   num_micro_batchesr   r   forward_contextsr   r   c                 $   | dk    s
J d            | a t          t                    | k     r3t                              d g| t          t                    z
  z             	 d t	          |           D             }d t	          |           D             }d t	          |           D             }g }	t	          |           D ]T}
t          |
||||
         |||
         ||
dz   | z           ||
         ||
         |
  
        }|	                    |           U|	S )Nrn   z(num_micro_batches must be greater than 1c                 4    g | ]}t          j                    S r\   )r    rZ   .0_s     r   
<listcomp>z(make_ubatch_contexts.<locals>.<listcomp>   s     FFF)/##FFFr   c                 4    g | ]}t          j                    S r\   r2   rZ   rw   s     r   rz   z(make_ubatch_contexts.<locals>.<listcomp>   s    LLLaEKMMLLLr   c                 4    g | ]}t          j                    S r\   r|   rw   s     r   rz   z(make_ubatch_contexts.<locals>.<listcomp>   s    OOOu{}}OOOr   )
r   r   r   r   r   r   r   r   r   r   )r
   r`   r   extendranger   append)rs   r   r   rt   r   r   
cpu_eventsgpu_comm_done_eventsgpu_compute_done_eventsctxsiri   s               r   make_ubatch_contextsr      sM    q   "L   %M
 111  $+<sCT?U?U+U!VWWW GFU3D-E-EFFFJLL59J3K3KLLLOOe<M6N6NOOOD$%%  )#,Q/'%a='Q2C(CD 4Q 7#:1#=
 
 
 	CKr   rR   )1r    typingr   r2   vllmr   vllm.forward_contextr   vllm.loggerr   vllm.utils.torch_utilsr   rS   loggerr   dict__annotations__r
   rW   r   listr   boolra   rc   rl   r)   dbo_maybe_run_recv_hookrM   	dbo_yieldrO   )dbo_yield_and_switch_from_compute_to_commrQ   )dbo_yield_and_switch_from_comm_to_computerD   dbo_switch_to_commrF   dbo_switch_to_computerH   dbo_switch_to_comm_syncrJ   dbo_switch_to_compute_syncrp   rr   r3   rX   rY   r[   r   r\   r   r   <module>r      s                           / / / / / / # # # # # # 1 1 1 1 1 1	X		  t      s   57 412 7 7 7       D*T * * * *8s 8 8 8 8   43M4UVV %%m&:;;	,E,E7- - ) -F,E7- - ) /.}/KLL 11-2QRR 33M4UVV 66(  
' ' ') ) ) ' ''J%' "' >*	'
 $' ' 
-' ' ' ' ' 'r   