
    &`ieL                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZmZ d dlmZmZmZmZmZmZ d dlZd dlZd dlmc mZ d dlmZ d dlm Z  d d	l!m"Z" d d
l#m$Z$ d dl%m&Z&m'Z' d dl(m)Z)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0Z0e G d d                      Z1e G d d                      Z2e G d d                      Z3 e3            Z4 e5d          Z6dZ7e4dddfdedede3dee8         dee9         dee8         fdZ:d Z;e4fde3fdZ< ej=        d             G d! d"                      Z>d#e"fd$Z?d%e8d#e$fd&Z@d' ZA	 d=d(e8d)e8d*ee8ee8         df         d#e9fd+ZBej=         G d, d-                      ZCd.e5d/e8fd0ZDd1ee8ee5ee8         f         f         d2e5fd3ZEd>d4eFd5e9fd6ZGe*e)ddd7fd8e5d9e5d:eee8e,e-f                  d5e9d;e9f
d<ZHdS )?    N)defaultdict)ThreadPoolExecutor)deepcopy)	dataclassfield)CallableDictListOptionalTupleUnion)
GcsChannel)	GcsClient)ActorHandle)StateAPIManager)
list_taskslist_workers)DEFAULT_LIMITDEFAULT_RPC_TIMEOUTListApiOptionsPredicateTypeSupportedFilterType)StateDataSourceClientc                   $    e Zd ZU eed<   eed<   dS )StateAPIMetriclatency_secresult_sizeN)__name__
__module____qualname__float__annotations__int     u/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/state_api_test_utils.pyr   r   %   s*         r%   r   c                   F    e Zd ZU eed<   eed<    ee          Zeed<   dS )StateAPICallSpecapi	verify_cbdefault_factorykwargsN)	r   r   r    r   r"   r   dictr-   r	   r$   r%   r&   r(   r(   +   sD         	MMM5...FD.....r%   r(   c                   P    e Zd ZU dZeed<   dZeed<    ed           Ze	ed<   dS )StateAPIStatsr   pending_callstotal_callsc                  *    t          t                    S N)r   listr$   r%   r&   <lambda>zStateAPIStats.<lambda>6   s    D0A0A r%   r+   callsN)
r   r   r    r1   r#   r"   r2   r   r7   r	   r$   r%   r&   r0   r0   2   sS         M3K%(A(ABBBE4BBBBBr%   r0   g    .AiX  Fr*   state_api_fnstate_stats
key_suffixprint_resulterr_msgc           
         d|vr
t           |d<   d|d<   d}	 |xj        dz  c_        |xj        dz  c_        t          j                    } |di |}t          j                    }	|rt          j        |           t          |	|z
  t          |                    }
|r|j         d| }n|j        }|j	        |         
                    |
            | |          sJ dt          |           d|             nL# t          $ r?}t          j                     	 J d	|j         d
| dt          |           d            d}~ww xY w	 |xj        dz  c_        n# |xj        dz  c_        w xY w|S )aV  Invoke a State API

    Args:
        - verify_cb: Callback that takes in the response from `state_api_fn` and
            returns a boolean, indicating the correctness of the results.
        - state_api_fn: Function of the state API
        - state_stats: Stats
        - kwargs: Keyword arguments to be forwarded to the `state_api_fn`
    timeoutFraise_on_missing_outputN   _z$Calling State API failed. len(res)=(z): zCalling (z) failed with .r$   )STATE_LIST_TIMEOUTr2   r1   timeperf_counterpprintr   lenr   r7   append	Exception	traceback	print_excrepr)r*   r8   r9   r:   r;   r<   r-   rest_startt_endmetrickeyes                r&   invoke_state_apirT   ?   s   $ .y ).F$%
C'1$!!Q&!!#%%l$$V$$!## 	M#S:: 	(!*99Z99CC'C#%%f---y
 
 	I 	IH#c((HHwHH	I 	I 
 	I  O O O	ONl+NNfNNDGGNNN	O 	OO	I 	!!Q&!!!!!Q&!!!!!!Js*   C"C: 9E :
E:D>>EE E*c                  D      fd}t          j        |dd           d S )Nc                  H    d} t          |           D ]}t          i  dS )N
   T)rangerT   )NUM_API_CALL_SAMPLESrA   argsr-   s     r&   verifyz"invoke_state_api_n.<locals>.verifyx   s<    !+,, 	. 	.Ad-f----tr%   i     )retry_interval_msr>   )
test_utilswait_for_condition)rZ   r-   r[   s   `` r&   invoke_state_api_nr`   w   sB          !&D"MMMMMMr%   c                    t          |           } i }| j                                        D ]\  }}d| d}| d}t          |d           }|j        ||<   |j        ||<   t          j        d |D                       }d| d}t          j        |          ||<   d| d}t          j	        |d	          ||<   d
| d}t          j	        |d          ||<   d| d}t          j	        |d          ||<   t          d | j                                        D                       }	| j        dk    r
|	| j        z  nd|d<   |S )aQ  Aggregate stats of state API calls

    Return:
        This returns a dict of below fields:
            - max_{api_key_name}_latency_sec:
                Max latency of call to {api_key_name}
            - {api_key_name}_result_size_with_max_latency:
                The size of the result (or the number of bytes for get_log API)
                for the max latency invocation
            - avg/p99/p95/p50_{api_key_name}_latency_sec:
                The percentile latency stats
            - avg_state_api_latency_sec:
                The average latency of all the state apis tracked
    max__latency_sec_result_size_with_max_latencyc                     | j         S r4   r   )rQ   s    r&   r6   z(aggregate_perf_results.<locals>.<lambda>   s	    1C r%   )rR   c                     g | ]	}|j         
S r$   rf   ).0rQ   s     r&   
<listcomp>z*aggregate_perf_results.<locals>.<listcomp>   s     J J J!3 J J Jr%   avg_p99_c   p95__   p50_2   c              3   .   K   | ]}|D ]}|j         V  d S r4   rf   )rh   metric_samplesrQ   s      r&   	<genexpr>z)aggregate_perf_results.<locals>.<genexpr>   sP          $     	             r%   r   avg_state_api_latency_sec)r   r7   itemsmaxr   r   nparrayaverage
percentilesumvaluesr2   )
r9   perf_resultapi_key_namemetricslatency_keysize_keyrQ   latency_listrR   all_state_api_latencys
             r&   aggregate_perf_resultsr      s     ;''KK!,!2!8!8!:!: ; ;g 8\777"AAAW"C"CDDD#)#5K  & 2Hx J J' J J JKK/\///:l33C 0\///=r::C 0\///=r::C 0\///=r::C    )/6688        "a'' 
!8	8	8 +, r%   )num_cpusc                   t    e Zd Z	 	 	 	 ddee         dededed	ed
dfdZd Zd Z	d Z
d Zd Zd Zd Zd ZdS )StateAPIGeneratorActor      @      4@TFapiscall_interval_sprint_interval_swait_after_stopr;   returnNc                    || _         || _        || _        || _        t	          j        | j        j                  | _        || _	        d| _
        d| _        d| _        d| _        d| _        d| _        t!                      | _        dS )ar  An actor that periodically issues state API

        Args:
            - apis: List of StateAPICallSpec
            - call_interval_s: State apis in the `apis` will be issued
                every `call_interval_s` seconds.
            - print_interval_s: How frequent state api stats will be dumped.
            - wait_after_stop: When true, call to `ray.get(actor.stop.remote())`
                will wait for all pending state APIs to return.
                Setting it to `False` might miss some long-running state apis calls.
            - print_result: True if result of each API call is printed. Default False.
        NF)_apis_call_interval_s_print_interval_s_wait_after_cancellogging	getLogger	__class__r   _logger_print_result_tasks
_fut_queue	_executor_loop	_stopping_stoppedr0   _stats)selfr   r   r   r   r;   s         r&   __init__zStateAPIGeneratorActor.__init__   s}    * 
 /!1"1()@AA) 
#oor%   c                 F  K   t          j                    | _        t          j                                        | _        d |                                 |                                 | 	                                fD             | _
        t          j        | j
          d {V  d S )Nc                 6    g | ]}t          j        |          S r$   )asyncioensure_future)rh   awts     r&   ri   z0StateAPIGeneratorActor.start.<locals>.<listcomp>   s3     
 
 
 !#&&
 
 
r%   )r   Queuer   
concurrentfuturesr   r   _run_generator_run_result_waiter_run_stats_reporterr   gatherr   s    r&   startzStateAPIGeneratorActor.start   s      !-//#+>>@@
 
 ##%%''))((**
 
 
 ndk**********r%   c                 v      fd}t          j                                         j        |          }|S )Nc            
          	 j                             dj         d d           t          fj        j        dS # t          $ r?} j                             j         d dt          |                       Y d } ~ d S d } ~ ww xY w)Nzcalling rB   ))r9   r;   z) failed with: )	r   debugr   rT   r   r   rJ   warningrM   )rS   fnr-   r   r*   s    r&   run_fnz+StateAPIGeneratorActor.call.<locals>.run_fn   s    ""#Ebk#E#EF#E#E#EFFF' !%!%!3	 
       $$%V%Vf%V%VTRSWW%V%VWWWttttts   A A 
B4BB)r   get_running_looprun_in_executorr   )r   r   r*   r-   r   futs   ````  r&   callzStateAPIGeneratorActor.call   sU    	 	 	 	 	 	 	 	 &((88PP
r%   c                 b  K   | j         s| j                            t          j        t	          | j                                       	 t          j        | j                   d {V  n=# t          j	        $ r+ | j                            d| j        j
         d           Y nw xY w| j         d S d S )Nz3_run_stats_reporter cancelled, waiting for all api zcalls to return...)r   r   inforG   r   r   r   sleepr   CancelledErrorr1   r   s    r&   r   z*StateAPIGeneratorActor._run_stats_reporter  s      - 
	 Lfm,B4;,O,OPPQQQmD$:;;;;;;;;;;)   !!Y+/;+DY Y Y     - 
	 
	 
	 
	 
	s   	A) )7B#"B#c                 N  K   	 | j         sj| j        D ]:} | j        |j        |j        fi |j        }| j                            |           ;t          j	        | j
                   d {V  | j         hd S d S # t          j        $ r | j                            d           Y d S w xY w)Nz)_run_generator cancelled, now stopping...)r   r   r   r)   r*   r-   r   
put_nowaitr   r   r   r   r   r   )r   api_specr   s      r&   r   z%StateAPIGeneratorActor._run_generator  s      	n ; $
 4 4H#$)HL(2DXXXXCO..s3333mD$9::::::::: n ; ; ; ; ; % 	 	 	LIJJJFF	s   A/A7 7)B$#B$c                   K   	 | j         s0| j                                         d {V }| d {V  | j         .d S d S # t          j        $ r | j                            d| j                                         d           | j                                        sV| j        	                                }| j
        r	| d {V  n|                                 | j                                        VY d S w xY w)Nz)_run_result_waiter cancelled, cancelling z pending futures...)r   r   getr   r   r   r   qsizeempty
get_nowaitr   cancel)r   r   s     r&   r   z)StateAPIGeneratorActor._run_result_waiter   sB     	n  O//11111111							 n      % 	 	 	L%DO<Q<Q<S<S % % %   o++-- !o0022* !IIIIIIII JJLLL o++-- ! FF	s   5= B3C43C4c                 *    t          | j                  S r4   )r   r   r   s    r&   	get_statsz StateAPIGeneratorActor.get_stats4  s    %dk222r%   c                     d S r4   r$   r   s    r&   readyzStateAPIGeneratorActor.ready8  s    r%   c                     d| _         | j                            dt          | j                   d           | j        D ]}|                                 | j                            | j                   d| _	        d S )NTzcalling stop, canceling z tasks)wait)
r   r   r   rH   r   r   r   shutdownr   r   )r   tasks     r&   stopzStateAPIGeneratorActor.stop;  s~    Nc$+6F6FNNNOOOK 	 	DKKMMMM 	T%<===r%   )r   r   TF)r   r   r    r
   r(   r!   boolr   r   r   r   r   r   r   r   r   r$   r%   r&   r   r      s        
 "%"& $"#& #&#$#& #&  	#&
 #& #& 
#& #& #& #&J+ + +  $      (3 3 3      r%   r   r   c                  T   t           j        j        j        j        }t                              d| di          j        | i |}t          d           t          j	        |j
                                                   t          d           |j                                         |S )Nznode:gMbP?)	resourcesz*Waiting for state api actor to be ready...zState api actor is ready now.)ray_privateworkerglobal_workernode_ip_addressr   optionsremoteprintr   r   r   )rZ   r-   current_node_ipactors       r&   %periodic_invoke_state_apis_with_actorr   I  s    l)7GO"**,?,,e4 +  d E 

6777GEK  !!!	
)***	KLr%   gcs_addressc                     t          |           }t          | d          }|                                 t          |                                |          }t          |t          d                    S )N)addressT)r   aiostate_api_test_utils)thread_name_prefix)thread_pool_executor)r   r   connectr   channelr   r   )r   
gcs_clientgcs_channelstate_api_data_source_clients       r&   get_state_api_managerr   V  s    ;///J$???K#8z$ $  $/5
 
 
   r%   c                  
   t          ddgdd          } g }g }| D ]y}|                    d          }|                    d          }|                    d          }|d	k    r|                    ||z
             |r|                    ||z
             z|                                 |                                 d
 }t	          d            ||           t	          d           t	          d            ||           d S )NT)worker_type=WORKERi'  F)detailfilterslimitr?   worker_launch_time_msworker_launched_time_msstart_time_msr   c                    t          dt          t          |           t          |           z  d           d           t          dt          | t	          t          |           dz                     d           d           t          dt          | t	          t          |           dz                     d           d           t          dt          | t	          t          |           d	z                     d           d           t          d
t          | t	          t          |           dz                     d           d           d S )NzAvg:    z mszP25: g      ?zP50: g      ?zP95: gffffff?zP99: gGz?)r   roundr|   rH   r#   )	latenciess    r&   print_latenciesz6summarize_worker_startup_time.<locals>.print_latenciesz  s-   DeC	NNS^^;Q??DDDEEEJeIc#i..4*?&@&@A1EEJJJKKKIeIc#i..3*>&?&?@!DDIIIJJJJeIc#i..4*?&@&@A1EEJJJKKKJeIc#i..4*?&@&@A1EEJJJKKKKKr%   zTime to launch workersz=======================zTime to initialize workers)r   r   rI   sortr   )workerstime_to_launchtime_to_initializer   launch_timelaunched_time
start_timer   s           r&   summarize_worker_startup_timer  e  sK   /0 %	  G N B Bjj!899

#<==ZZ00
1!!-+"=>>> 	B%%j=&@AAAL L L 

"###ON###	
#$$$	
&'''O&'''''r%   name
error_typeerror_messagec                 T   t          dd| fgd          }t          |          dk    s
J |            |d         }|d         dk    s
J |            |d	         |k    s
J |            |?t          |t                    r|g}|D ]$}||                    dd
          v s
J |            %dS )z
    Check if a task with 'name' has failed with the exact error type 'error_type'
    and 'error_message' in the error message.
    r  r   T)r   r   r@   r   stateFAILEDr  Nr  )r   rH   
isinstancestrr   )r  r  r  taskstmsgs         r&   verify_failed_taskr    s     d 34TBBBEu::???E???aAW:!!!1!!!\?j(((!((( mS)) 	,*OM  	: 	:C!%%6666666664r%   c                   "    e Zd Zd Zd ZddZdS )PidActorc                     i | _         d S r4   name_to_pidr   s    r&   r   zPidActor.__init__  s    r%   c                     | j         S r4   r  r   s    r&   get_pidszPidActor.get_pids  s    r%   Nc                     ||f| j         |<   d S r4   r  )r   r  pidr  s       r&   
report_pidzPidActor.report_pid  s    "%ur%   r4   )r   r   r    r   r  r  r$   r%   r&   r  r    sF               . . . . . .r%   r  	actor_pid	task_namec                 T   t          j        |           sdS 	 t          j        |                                           }||v r|                    d          rdS t          j        |                                           }|r'||d         v r|d                             d          rdS dS )a6  
    Check whether the actor task `task_name` is running on the actor process
    with pid `actor_pid`.

    Args:
      actor_pid: The pid of the actor process.
      task_name: The name of the actor task.

    Returns:
      True if the actor task is running, False otherwise.

    Limitation:
        If the actor task name is set using options.name and is a substring of
        the actor name, this function may return true even if the task is not
        running on the actor process. To resolve this issue, we can possibly
        pass in the actor name.
    Fzray::Tr   )psutil
pid_existsProcessr  
startswithcmdline)r  r  r  r!  s       r&   _is_actor_task_runningr"    s    $ Y'' u$J >)$$))++DDT__W55tnY''//11G  9
**wqz/D/DW/M/M*t5r%   	task_pidsexpect_num_tasksc                    t          |           |k    s
J |             |                                 D ]\  }}t          ddd|fg          }t          |          dk    sJ d| d            |d         }|\  }}t          j        d	v r||d         |k    s
J |            pt          ||          rCd|d         vs
J d            |d         dk    s
J |            ||d         |k    s
J |            |)|d         dv sJ | d|d          d|d                      |d         |k    sJ d| d|d          d|             dS )a/  
    Check if the tasks in task_pids are in RUNNING state if pid exists
    and running the task.
    If the pid is missing or the task is not running the task, check if the task
    is marked FAILED or FINISHED.

    Args:
        task_pids: A dict of task name to (pid, expected terminal state).

    Tr  r   )r   r   r@   zOne unique task with zI should be found. Use `options(name=<task_name>)` when creating the task.r   )win32darwinNr  z	ray::IDLEz8One should not name it 'IDLE' since it's reserved in RayRUNNING)r	  FINISHEDz: task_idz = zexpect z but z for )rH   rv   r   sysplatformr"  )r#  r$  r  pid_and_stater  r   r  expected_states           r&   "verify_tasks_running_or_terminatedr/    s    y>>----y---$-OO$5$5 !M !M 	=$&#y1I0JKKK5zzQFI F F F  Qx+^
 <...)G}666666!#y11 	M4<///I 0//=I---t---)G}666666 %G} )     FF4	?FFtG}FF    M^333L^LL$w-LLdLL 4333 4r%   result_dictr   c                     t                      }|r|                                 }n|                                 }|D ]}||v sJ 	|D ]}||v sJ 	 | di | dS )zH
    Verify the schema of the result_dict is the same as the state.
    Nr$   )setcolumnsbase_columns)r  r0  r   state_fields_columnsks        r&   verify_schemar7  "  s     55 4$}}$1133!    K ) )(((((( 
EKr%   Tr>   r   r   exclude_driverc                 4    |sg }t          || |d||          S )Ng      ?)r   r>   r   server_timeout_multiplierr   r8  )r   )r>   r   r   r   r8  s        r&   create_api_optionsr;  6  s:      "%%   r%   r4   )F)Ir   concurrent.futuresr   r   rG   r+  rE   rK   collectionsr   r   copyr   dataclassesr   r   typingr   r	   r
   r   r   r   numpyrx   r   ray._common.test_utils_commonr^   ray._private.gcs_utilsr   ray._rayletr   	ray.actorr   ray.dashboard.state_aggregatorr   ray.util.stater   r   ray.util.state.commonr   r   r   r   r   ray.util.state.state_managerr   r  r   r(   r0   GLOBAL_STATE_STATSr#   STATE_LIST_LIMITrD   r  r   rT   r`   r   r   r   r   r   r  r  r  r"  r/  r.   r7  r;  r$   r%   r&   <module>rM     sU          



      # # # # # # 1 1 1 1 1 1       ( ( ( ( ( ( ( ( ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?     



 + + + + + + + + + - - - - - - ! ! ! ! ! ! ! ! ! ! ! !      4 3 3 3 3 3 3 3              ? > > > > >         
 / / / / / / / / C C C C C C C C #]__ 3s88   "4 $#(!5 555 5 	5
 4.5 c]5 5 5 5pN N N 9K 9 9 9 9 9 9x QH H H H H H H HV
k 
 
 
 
s      (  (  (H NR 
/4S$s)T5I/J	   ( . . . . . . . .Cc Cc C C C CL1CsHSM12231GJ1 1 1 1h d D    * 'DH  %],??@A 	
      r%   