
    &`i                    b   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZmZmZ d dlmZmZ d dlmZ d dlmZmZmZmZm Z m!Z!m"Z" d dl#m$Z$ d dl%Z%d dl&Z&d dl'Z'd dl(m)c m*Z* d dl+Z'd dl+m)c m,Z, d dl-Z'd dl.m/c m0Z1 d dl2m3Z3m4Z4 d d	l5m6Z6 d d
l7m8Z8 d dl9m:Z: d dl;m<Z< d dl=m>Z> d dl?m@Z@ d dlAmBZBmCZCmDZDmEZE d dlFmGZGmHZHmIZI d dlJmKZKmLZLmMZM d dlNmOZO d dlPmQZQmRZR d dlSZS ejT        eU          ZVe
jW        dk    rdndZXejY        Z                    ejY        [                    ejY        [                    e\                              Z]ejY        ^                    e]deXz             Z_	 d dl`maZa d dlbmcZcmdZd n# eeeff$ r
 dZadZcd ZdY nw xY wd Zgd Zhd Zid Zjd Zk	 	 	 	 dd!eld"emd#emd$e el         d%enf
d&Zo	 	 	 	 	 	 	 	 	 	 	 	 	 dd!eld"emd(e em         d)emd*e el         d+e el         d$e el         d,e en         d-e eem                  d.end%enfd/Zpd0 Zqdd1Zrdd2Zsdd3Ztd4 Zudd5emd6evfd7Zwdd9Zxdd:Zydd;Zzdd=eld>ed?elfd@Z{	 dd=eld>ed?eldAe"elelf         fdBZ|dd>efdCZ}dD Z~ddFZddHZ	 	 	 ddIed6emdJemdKendLef
dMZe G dN dO                      Ze G dP dQ                      Z	 ddRee         dSedTe el         dAeg enf         fdUZdVdGd dWdXZdY ZdZ Zd[ Ze'j        d\             Zd] Zdd_Zdd`Zda Zdb ZdddZde Z	 	 	 	 ddfemd6evdge el         dAee         fdhZ	 	 	 	 ddfemd6evdge el         dAeeel                  fdiZ	 	 	 	 ddfemd6evdge el         fdjZ	 	 	 ddfemd6evdge el         dAeel         fdkZdl ZdAenfdmZdAenfdnZdo Zdp Zdqeel         dredAefdsZdteel         dAeelee         f         fduZdteel         dredAeelee         f         fdvZdwe@dredAeelee         f         fdxZdyeldzeld{eldAeev         fd|Zd} Zd~ Zd Z G d deL          Z G d deM          Zd Zd Z	 	 	 ddemdemdevfdZd Zd Z G d d          Z G d de          Z e'j        d            G d de                      Z e'j        d            G d de                      Z e'j        d            G d de                      Z e'j        d            G d de                      Z	 	 	 	 	 	 	 ddZdddAelfdZedelfd            Zd Zd ZddZddZd Zd Zd ZddZddZd ZddZd Z	 	 ddredelde el         fdZd Zd ad ad Zd Zd Zd Zd Zd ZdeldAdfdZdeldelfdZddddZ ee̠                                          ZdS )    N)defaultdict)contextmanagerredirect_stderrredirect_stdout)	dataclassfield)datetime)AnyCallableDictListOptionalSetTuple)quote)build_addressparse_addresswait_for_condition)get_or_create_event_loop)ray_constants)memory_summary)generate_self_signed_tls_certs)
RayContext)Config	GcsClientGcsClientOptionsGlobalStateAccessor)gcs_pb2gcs_service_pb2node_manager_pb2)EmptyQueue_QueueActor)NodeAffinitySchedulingStrategy)	get_actorlist_actorswin32z.exe z.core/src/ray/thirdparty/redis/src/redis-server)Metric)Sampletext_string_to_metric_familiesc                       t          d          )Nz`prometheus_client` not found)ModuleNotFoundError)argskwargss     k/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/test_utils.pyr,   r,   F   s    !"ABBB    c                     t          j        | j        d         d dd          }t          |          }|                                 |S )Ngcs_addressTF)allow_cluster_id_nilfetch_cluster_id_if_nil)r   createaddress_infor   connect)ray_contextgcs_optionsglobal_state_accessors      r1   make_global_state_accessorr=   J   sU    ") /! %	  K 0<<!!###  r2   c                  H    t           j                            d          dk    S )NTEST_EXTERNAL_REDIS1)osenvironget r2   r1   external_redis_test_enabledrE   V   s    :>>/00C77r2   c                  \    t          t          j                            dd                    S )NTEST_EXTERNAL_REDIS_REPLICASr@   intrA   rB   rC   rD   r2   r1   redis_replicasrJ   Z   s!    rz~~<cBBCCCr2   c                  \    t          t          j                            dd                    S )N%TEST_EXTERNAL_REDIS_SENTINEL_REPLICAS2rH   rD   r2   r1   redis_sentinel_replicasrN   ^   s!    rz~~EsKKLLLr2   c                 L   	 dd l }n# t          $ r Y dS w xY wi }|rnddlm} ddd} |j                    r |j                    |d<    |j                    r |j                    |d<    |j                    r |j                    |d<    |j        d	t          |           fi |S )
Nr   T)r   required)sslssl_cert_reqsssl_ca_certsssl_certfilessl_keyfile	localhost)	redis	Exceptionray._rayletr   REDIS_CA_CERTREDIS_CLIENT_CERTREDIS_CLIENT_KEYRedisstr)port
enable_tlsrW   paramsr   s        r1   get_redis_clirb   b   s     	   tt F 	>&&&&&&
;;6!! 	<%9V%9%;%;F>"#6#%% 	@%=V%=%?%?F>""6"$$ 	>$;F$;$=$=F=!5;{CII88888s    
Fsession_dir_pathr_   redis_master_portpasswordr`   c                    t           j                            | dt          j                    j        z   dz             }g }|r|                    d|            n|                    d|            |                    d| d           |                    d           |                    d           |                    d           |r|                    d	|            |r|                    d
|            t          j                    r)|                    dt          j                                t          j	                    r)|                    dt          j	                                t          j
                    r)|                    dt          j
                                |                    d           |                    d           |r|                    d|            t          |d          5 }	|	                    d                    |                     d d d            n# 1 swxY w Y   t          |dg}
t          j        j                            |
t$          j        d          }|S )Nzredis-sentinel-z.confzport z&sentinel monitor redis-test 127.0.0.1 z 1z0sentinel down-after-milliseconds redis-test 1000z)sentinel failover-timeout redis-test 5000z$sentinel parallel-syncs redis-test 1zsentinel auth-pass redis-test z	tls-port ztls-ca-cert-file ztls-cert-file ztls-key-file ztls-auth-clients noz'sentinel tls-auth-clients redis-test nozdir w
z
--sentinelF)
fate_share)rA   pathjoinuuiduuid4hexappendr   rZ   r[   r\   openwriteREDIS_EXECUTABLEray_privateservicesstart_ray_processr   PROCESS_TYPE_REDIS_SERVER)rc   r_   rd   re   r`   db_dir	free_portconfig_fileconfig_linesfcommandprocess_infos               r1   start_redis_sentinel_instancer   z   s    ',,+djll.>>H K L ,/I//0000NDNN+++ VARVVVWWW:   CDDD>??? IGXGGHHH G...///!! 	N LF4H4J4J L LMMM#%% 	O M1I1K1K M MNNN"$$ 	M K0G0I0I K KLLL1222EFFF -O6OO,,,	k3		 )1			,''((() ) ) ) ) ) ) ) ) ) ) ) ) ) )  l;G<(::/ ;  L
 s   )H;;H?H?   redis_max_clientsnum_retriesstdout_filestderr_fileri   port_denylistlisten_to_localhost_onlyc           	         t           j                            t                    sJ t          g}|rd|v rt	          d          |d|gz  }t                      dk    r|dddd| gz  }|
r&|d	t          |          d
ddt          |          gz  }n|dt          |          d
dgz  }|	r|ddgz  }t           j                            | dt          j	                    j
        z   dz             }|d|gz  }|
rt          j                    r|dt          j                    gz  }t          j                    r|dt          j                    gz  }t          j                    r|dt          j                    gz  }||ddgz  }|g dz  }t          j        dk    r|g dz  }||dt          |          gz  }t"          j        j                            |t*          j        |||          }d}t                      dk    rQddl}	 	 t1          ||
          }|(d t3          d          D             } |j        d g|R   njt6                              |                    d!dt          |                               t6                              |                    d"|                     |                    d#          }n# |j        j        |j        j        f$ rq}dd$l m!} t6                              d%| d&           |j"        #                                tI          d'| d(|j"        j%                    |d)           Y d}~nd}~ww xY wLt6                              d*| d+|j"        j&                    ||fS ),aq  Start a single Redis server.

    Notes:
        We will initially try to start the Redis instance at the given port,
        and then try at most `num_retries - 1` times to start the Redis
        instance at successive random ports.

    Args:
        session_dir_path: Path to the session directory of
            this Ray cluster.
        port: Try to start a Redis server at this port.
        redis_max_clients: If this is provided, Ray will attempt to configure
            Redis with this maxclients number.
        num_retries: The number of times to attempt to start Redis at
            successive ports.
        stdout_file: A file handle opened for writing to redirect stdout to. If
            no redirection should happen, then this should be None.
        stderr_file: A file handle opened for writing to redirect stderr to. If
            no redirection should happen, then this should be None.
        password: Prevents external clients without the password
            from connecting to Redis if provided.
        port_denylist: A set of denylist ports that shouldn't
            be used when allocating a new port.
        listen_to_localhost_only: Redis server only listens to
            localhost (127.0.0.1) if it's true,
            otherwise it listens to all network interfaces.
        enable_tls: Enable the TLS/SSL in Redis or not

    Returns:
        A tuple of the port used by Redis and ProcessInfo for the process that
            was started. If a port is passed in, then the returned port value
            is the same.

    Raises:
        Exception: An exception is raised if Redis could not be started.
     z'Spaces not permitted in redis password.z--requirepass   z--cluster-enabledyesz--cluster-config-fileznode-z
--tls-portz
--loglevelwarningz--portz--bindz	127.0.0.1zredis-z.pidz	--pidfilez--tls-ca-cert-filez--tls-cert-filez--tls-key-fileNz--tls-replication)z--tls-auth-clientsnoz--tls-clusterr   r(   )z--saver)   z--appendonlyr   z--dir)r   r   ri   r   Tc                 ,    g | ]}t          |          S rD   )r^   ).0is     r1   
<listcomp>z(start_redis_instance.<locals>.<listcomp>"  s    :::SVV:::r2   i @  addslotsmeet	replicatemyid)sleepz5Waiting for redis to be up. Check failed with error: z. Will retry in 0.1sz#Redis process exited unexpectedly: z. Exit code: 皙?zRedis started with node_id z	 and pid )'rA   rj   isfilerr   
ValueErrorrJ   r^   rk   rl   rm   rn   r   rZ   r[   r\   sysplatformrs   rt   ru   rv   r   rw   rW   rb   rangeclusterloggerinfo
exceptionsConnectionErrorResponseErrortimer   processpollrX   
returncodepid)rc   r_   r   r   r   r   re   ri   r   r   r`   
replica_of	leader_idrx   ry   r}   pidfiler~   node_idrW   	redis_clislotser   s                           r1   start_redis_instancer      s=   l 7>>*+++++   G /(??FGGGOX..!'0GQUXX 
BII	NN
 	
 	Hc$iiyAA +Hk**gll+X
8H-H6-QRRGW%%G 	H!! 	F,f.B.D.DEEG#%% 	G)6+C+E+EFFG"$$ 	E(&*A*C*CDDG!+U33GGGGG
|w7777GS[[))<(::/ ;  L G!	)$
;;	%::U5\\:::E%I%j9599999KK	 1 1&+s: W WXXXKK	 1 1+y I IJJJ#++F33 0 .    '&&&&&)A ) ) )  
  ',,..:#Hl H H&2&:&EH H  
 c







#	< KKRgRR8L8PRR   L  s   B9J; ;MA'MMc                     d}	 t          j        |           }|                                t           j        k    rd}n# t           j        $ r d}Y nw xY w|S )zCheck if the process with this PID is alive or not.

    Args:
        pid: The pid to check.

    Returns:
        This returns false if the process is dead. Otherwise, it returns true.
    TF)psutilProcessstatusSTATUS_ZOMBIENoSuchProcess)r   aliveprocs      r1   
_pid_aliver   C  se     E~c"";;==F000E   Ls   8= AAc                    t          j        t          j                    t          j        j                  }t          j        d d          }	 |d d          t          _        	 t          |r|nt          j                  5  t          |r|nt          j                  5   |              d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |
                                 n# |
                                 w xY wn# t          $ rE}|j        r4|j                                        }t          j        |j        ||          Y d }~nPd }~wt"          $ r@}|j                                        }t          j        d|||j        d                   d }~ww xY w|t          _        |r<t          j        j                            |j                                                   n|r;t          j        j                            |j                                                   n# |t          _        |r<t          j        j                            |j                                                   w |r<t          j        j                            |j                                                   w w xY w|j                                        S )Nencodingr   r   )ioTextIOWrapperBytesIOr   stdoutr   argvr   stderrr   flush
SystemExitcodebuffergetvalue
subprocessCalledProcessErrorrX   r/   rq   )mainr   capture_stdoutcapture_stderrstreamold_argvexoutputs           r1   _check_call_windowsr   V  s    bjllSZ5HIIIFx{H>7	 >!IszJJ  $~%MVV3:NN  DFFF                             LLNNNNFLLNNNNN G G G7 	G]++--F/vFFF	G 	G 	G 	G 	G  I I I''))+AtVRWQZHHHI  	>J##FM$:$:$<$<==== 	>J##FM$:$:$<$<===	  	>J##FM$:$:$<$<==== 	>J##FM$:$:$<$<====	>=!!###s   D !C* >C
B3'C
3B7	7C
:B7	;C
>C* 
CC* CC* D *D  D H& 
F;E	H& F;FFH& &B
J0c                    ddl m} t          j        dk    rt	          || ||          }nd }d }|rt
          j        }|r|rt
          j        }n|rt
          j        }t          j        | ||          }|	                                \  }}	|j
        rt          j        |j
        | ||	          d                    d ||	fD                       }|S )Nr   )r   r(   )r   r   )r   r   r2   c                     g | ]}||S NrD   )r   ss     r1   r   z)check_call_subprocess.<locals>.<listcomp>  s    HHH!-1---r2   )ray.scripts.scriptsr   r   r   r   r   PIPESTDOUTPopencommunicater   r   rk   )
r   r   r   ray_mainresultstdout_redirstderr_redirr   r   r   s
             r1   check_call_subprocessr   s  s     544444
|w$d>.
 
 
  	+%?L 	+n 	+%,LL 	+%?L\,OOO++--? 	W/vvVVVHHvv&6HHHIIMr2   c                 0    t          dg| z   ||           d S )Nrs   )r   )r/   r   r   s      r1   check_call_rayr     s     5'D...IIIIIr2   c                 f     t           j                   fdt          fd           d S )Naddressc                                           t          j          j        j                                         t          j        t          j                  S )N)	namespacetimeout)	internal_kv_getdashboard_consts#DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX	head_noder   encoder   KV_NAMESPACE_DASHBOARDGCS_RPC_TIMEOUT_SECONDS)r   
gcs_clients   r1   get_dashboard_agent_addresszGwait_for_dashboard_agent_available.<locals>.get_dashboard_agent_address  sO    ))C`WEVE^``ggii#:$< * 
 
 	
r2   c                                    d uS r   rD   )r   s   r1   <lambda>z4wait_for_dashboard_agent_available.<locals>.<lambda>  s    ::<<DH r2   )r   r   r   )r   r   r   s   `@@r1   "wait_for_dashboard_agent_availabler     sU    7?333J
 
 
 
 
 
 HHHHIIIIIr2   r   r   c                     t          j                     }t          j                     |z
  |k     r?t          |           sd S t          j        d           t          j                     |z
  |k     ?t          d|  d          )Nr   $Timed out while waiting for process z	 to exit.)r   r   r   TimeoutError)r   r   
start_times      r1   wait_for_pid_to_exitr     sw    J
)++

"W
,
,# 	F
3 )++

"W
,
, LcLLL
M
MMr2   r   c                 v   t          j        |           }t          j                    }g }t          j                    |z
  |k     r[|                    d          }t	          |          }||k    rd S t          j        d           t          j                    |z
  |k     [t          d|  d| d| d| d	          )	NF)	recursiver   r   z children to start (/z
 started: z).)r   r   r   childrenlenr   r   )r   num_childrenr   pr   r   	num_alives          r1   wait_for_children_of_pidr     s    sAJE
)++

"W
,
,

U
++JJ	$$F
3 )++

"W
,
, 	:s 	: 	:	: 	:$	: 	:05	: 	: 	:  r2   c                 ,   t          j        |                                           }t          |          dk    rd S t          j        ||          \  }}t          |          dk    r,t          d                    d |D                                 d S )Nr   r   zOTimed out while waiting for process children to exit. Children still alive: {}.c                 6    g | ]}|                                 S rD   )name)r   r   s     r1   r   z4wait_for_children_of_pid_to_exit.<locals>.<listcomp>  s     0I0I0Ia0I0I0Ir2   )r   r   r   r   
wait_procsr   format)r   r   r   _r   s        r1    wait_for_children_of_pid_to_exitr    s    ~c""++--H
8}} 7;;;HAu
5zzA~~))/0I0I50I0I0I)J)J
 
 	
 ~r2   c                     t          j        dg          D ]U}|j        d         | t          j        j        j        z   k    r+|r|                                 A|                                 Vd S )Nr  )attrs)	r   process_iterr   rs   rt   ru   
EXE_SUFFIXkill	terminate)r  SIGKILLr   s      r1   kill_process_by_namer    sn     x000  6&>TCL$9$DDDD  r2   utf-8driver_scriptenvr   c                 h   t          j        t          j        dgt           j        t           j        t           j        |          }|5  |                    |                     |                    d         }|j        rxt          t          j        j                            ||                     t                              |j                   t          j        |j        |j        ||j                  t          j        j                            ||          }ddd           n# 1 swxY w Y   |S )zRun a driver as a separate process.

    Args:
        driver_script: A string to run as a Python script.
        env: The environment variables for the driver.

    Returns:
        The script's output.
    -stdinr   r   r  r   r   encode_typeN)r   r   r   
executabler   r   r   r   r   printrs   _commonutilsdecoder   errorr   r   r/   )r  r  r   r   r   outs         r1   run_string_as_driverr     s]    	o   D 
 C C!!-"6"6"6"G"GHHK? 	#+#**6v*FFGGGLL%%%/FDK   k&&v6&BBC C C C C C C C C C C C C C C Js   CD''D+.D+returnc                    t          j        t          j        dgt           j        t           j        t           j        |          }|5  |                    |                                         }fd|D             \  }}|j        r?t          |           t          |           t          j	        |j        |j
        ||          ||fcddd           S # 1 swxY w Y   dS )zRun a driver as a separate process.

    Args:
        driver_script: A string to run as a Python script.
        env: The environment variables for the driver.

    Returns:
        The script's stdout and stderr.
    r  r  r   c                 \    g | ](}t           j        j                            |           )S )r  )rs   r  r  r  )r   r   r   s     r1   r   z6run_string_as_driver_stdout_stderr.<locals>.<listcomp>  sA     
 
 
 K$$V$@@
 
 
r2   N)r   r   r   r  r   r   r   r   r  r   r/   )r  r  r   r   outputs_bytesout_strerr_strs     `    r1   "run_string_as_driver_stdout_stderrr'    sC    	o  D 
    (()=)=v)=)N)NOO
 
 
 
'
 
 
 ? 	'NNN'NNN/GW                                      s   BCCCc                 H   d                     g d          }t          j        t          j        d|gt          j        t          j        t          j        |          }|j                            |                     d                     |j        	                                 |S )zStart a driver as a separate process and return immediately.

    Args:
        driver_script: A string to run as a Python script.

    Returns:
        A handle to the driver process.
    z; )z
import syszscript = sys.stdin.read()zsys.stdin.close()zdel syszexec("del script\n" + script)z-cr  ascii)
rk   r   r   r   r  r   r  rq   r   close)r  r  scriptr   s       r1    run_string_as_driver_nonblockingr,    s     YY	
 	
 	
 F 	v&o  D 	J]))'22333JKr2   c                 V    | sd S t           j        j        j        j        |          j        S r   )r   ActorTableData
ActorState
DESCRIPTORvalues_by_numberr  )states    r1   convert_actor_stater3  *  s)     t!,7HOTTr2   
   c                 L   t          |          }t          j                    }t          j                    |z
  |k     rZt          t          |rdd|fgnd |                     | k    rd S t          j        d           t          j                    |z
  |k     Zt          d          )Nr2  =)filterslimitr   z)Timed out while waiting for global state.)r3  r   r   r'   r   r   )
num_actorsr2  r   r   s       r1   wait_for_num_actorsr:  0  s    &&EJ
)++

"W
,
,7<FgsE233$$      F
3 )++

"W
,
, B
C
CCr2   d   c                    | j                                         }t          |          j        }t	          j        |            t          j                    }t          j                    |z
  |k    rYt          |          }|j        dk    s|j        |k    rd S t          j        |dz             t          j                    |z
  |k    Yt          d
                    |                    )N)idDEAD     @@z*It took too much time to kill an actor: {})	_actor_idrn   r&   num_restartsrs   r  r   r2  r   RuntimeErrorr  )actorr   retry_interval_msactor_idcurrent_num_restartsstartactor_states          r1   kill_actor_and_wait_for_failurerI  B  s    ""$$H$111>HUOOOIKKE
)++

(
(8,,,'''*>>>F
$v-... )++

(
( CJJ8TT
U
UUr2   assertion_predictorrD  raise_exceptionsr0   c                 l      fd}	 t          |f|||d dS # t          $ r   di  Y dS w xY w)a*  Wait until an assertion is met or time out with an exception.

    Args:
        assertion_predictor: A function that predicts the assertion.
        timeout: Maximum timeout in seconds.
        retry_interval_ms: Retry interval in milliseconds.
        raise_exceptions: If true, exceptions that occur while executing
            assertion_predictor won't be caught and instead will be raised.
        **kwargs: Arguments to pass to the condition_predictor.

    Raises:
        RuntimeError: If the assertion is not met before the timeout expires.
    c                  <    	   di  dS # t           $ r Y dS w xY w)NTFrD   )AssertionError)rJ  r0   s   r1   _assertion_to_conditionz3wait_for_assertion.<locals>._assertion_to_conditiong  sF    	))&)))4 	 	 	55	s    
)r   rD  rK  NrD   )r   rB  )rJ  r   rD  rK  r0   rO  s   `   ` r1   wait_for_assertionrP  R  s    *     	&#	
/-		
 	

 	
 	
 	
 	
 	
  & & &%%f%%%%%%&s    33c                   z    e Zd ZU dZee         ed<   dZee         ed<   dZee	eef                  ed<   de
fdZdS )MetricSamplePatternNr  valuepartial_label_matchsamplec                     | j         | j         |j         k    rdS | j        | j        |j        k    rdS | j        @| j                                        D ]&\  }}|j                            |          |k    r dS 'dS )NFT)r  rS  rT  itemslabelsrC   )selfrU  labelrS  s       r1   matcheszMetricSamplePattern.matches  s    9 yFK''u:!zV\))u#/ $ 8 > > @ @ ! !u=$$U++u44 55 5 tr2   )__name__
__module____qualname__r  r   r^   __annotations__rS  rT  r   r+   r[  rD   r2   r1   rR  rR  z  sw         D(3-E8C=48$sCx.1888f      r2   rR  c                       e Zd ZU dZ ee          Zeee	e         f         e
d<    ee          Zeeef         e
d<    ee          Zeeef         e
d<   d ZdS )PrometheusTimeseriesac  A collection of timeseries from multiple addresses. Each timeseries is a
    collection of samples with the same metric name and labels. Concretely:
    - components_dict: a dictionary of addresses to the Component labels
    - metric_descriptors: a dictionary of metric names to the Metric object
    - metric_samples: the latest value of each label
    )default_factorycomponents_dictmetric_descriptorsmetric_samplesc                     | j                                          | j                                         | j                                         d S r   )rc  clearrd  re  rY  s    r1   r   zPrometheusTimeseries.flush  sG    ""$$$%%'''!!#####r2   N)r\  r]  r^  __doc__r   dictrc  r   r^   r   r_  rd  r*   re  	frozensetr+   r   rD   r2   r1   ra  ra    s           ,15+F+F+FOT#s3x-(FFF,1E$,G,G,GS&[)GGG.3eD.I.I.INDF*+III$ $ $ $ $r2   ra  metrics_to_check
timeseriesexport_addrc                      t          j                    d         }|d         }|d         }|pt          ||           fd}|S )a  A condition to check if a prometheus metrics reach a certain value.

    This is a blocking check that can be passed into a `wait_for_condition`
    style function.

    Args:
        metrics_to_check: A list of MetricSamplePattern. The fields that
            aren't `None` will be matched.
        timeseries: A PrometheusTimeseries object to store the metrics.
        export_addr: Optional address to export metrics to.

    Returns:
        A function that returns True if all the metrics are emitted.
    r   MetricsExportPortNodeManagerAddressc                      D ]]} t          g          j                                        }|D ]}|                     |          r nt	          d|  d|            dS ^dS )NzDidn't find zall samplesFT)fetch_prometheus_timeseriesre  valuesr[  r  )metric_patternre  metric_samplerl  	prom_addrrm  s      r1   r|   z%get_metric_check_condition.<locals>.f  s    . 	 	N8Z VVXX  "0 	 	!))-88 E 3>33!"  
 uu  tr2   )rs   nodesr   )rl  rm  rn  	node_infometrics_export_portaddrr|   rw  s   ``     @r1   get_metric_check_conditionr|    sl    & 	AI#$78)*DG}T3FGGI      " Hr2     )
timeout_msrD  raise_last_exc                   t          t          |          t                    rt          d          d}t	          j                    }d}||k    rR	  | |  dS # |$ r<}	|	}t	          j                    |z
  dz  }t	          j        |dz             Y d}	~	nd}	~	ww xY w||k    R|rS|r(t          j        t          |          ||j                  ng }
d	                    |
          }
t          d|
           d	S )
a  A helper function that waits until a given function
        completes without exceptions.

    Args:
        func: A function to run.
        exceptions: Exceptions that are supposed to occur.
        args: arguments to pass for a given func
        timeout_ms: Maximum timeout in milliseconds.
        retry_interval_ms: Retry interval in milliseconds.
        raise_last_ex: Raise the last exception when timeout.

    Return:
        Whether exception occurs within a timeout.
    z/exceptions arguments should be given as a tupler   NTr}  r?  r)   zTimed out while testing, F)

isinstancetypetuplerX   r   r   	tracebackformat_exception__traceback__rk   )funcr   r~  rD  r  r/   time_elapsedrG  last_exr   ex_stacks              r1   &wait_until_succeeded_without_exceptionr    s7   " $z""E** KIJJJLIKKEG
*
$
$	3D$KK4 	3 	3 	3G IKK%/47LJ(6122222222	3	 *
$
$  @ I&tG}}gw?TUUU 	
 778$$>H>>???5s   A B2BBc                     g }t          j        |           D ]Q\  }}}t          j        ||          D ]5}|                    t           j                            ||                     6R|S )zLooks at a file directory subtree for a filename pattern.

    Similar to glob.glob(..., recursive=True) but also supports 2.7
    )rA   walkfnmatchfilterro   rj   rk   )dirpathpatternr[  rootdirnames	filenamesfilenames          r1   recursive_fnmatchr    ss    
 G%'WW%5%5 9 9!h	y':: 	9 	9HNN27<<h778888	9Nr2   c                      d| i}|S )N_system_configrD   )r0   
ray_kwargss     r1   generate_system_config_mapr    s    &J r2   c                 r    t          |           }t          |          }|D ]	}||vr dS 
|D ]	}||vr dS 
dS )zChecks if two iterables (such as lists) contain the same elements. Elements
    do not have to be hashable (this allows us to compare sets of dicts for
    example). This comparison is not necessarily efficient.
    FT)list)elems_aelems_babxs        r1   same_elementsr    sk    
 	WAWA  A::55    A::55  4r2   c                     | S r   rD   )objs    r1   _putr     s    Jr2   c                 b    |rt          j        |           S t                              |           S r   )rs   putr  remote)r  use_ray_puts     r1   
put_objectr  %  s*      ws||{{3r2     c                     t          |           \  }}t          |          }d}t          j                    }||k    rt          j        t          j        t          j                  }|                    d           	 |                    ||f           nT# t          $ rG t          j                    |z
  dz  }t          j	        |dz             |
                                 Y w xY w|
                                 dS dS )Nr   r   r}  r?  TF)r   rI   r   socketAF_INETSOCK_STREAM
settimeoutr9   rX   r   r*  )	r   r~  rD  ipport_strr_   r  rG  r   s	            r1   wait_until_server_availabler  ,  s     ))LBx==DLIKKE
*
$
$M&.&*<==	Q	IIr4j!!!! 	 	 	 IKK%/47LJ(61222GGIIIH		
 	
			t5s   <B AC%$C%c                 D    fd|                                  D             S )z5Get all nodes except the one that we're connected to.c                 v    g | ]5}|j         t          j        j        j        j         k    rd u s	|j        d u 3|6S F)_raylet_socket_namers   rt   worker_global_nodehead)r   nodeexclude_heads     r1   r   z#get_other_nodes.<locals>.<listcomp>B  s\       #<+?@ @U""di5&8&8	 	 '9&8&8r2   )list_all_nodes)r   r  s    `r1   get_other_nodesr  @  s:       **,,   r2   c                 b    t          t          d |                                                     S )zGet all non-head nodes.c                     | j         du S NF)r  )r  s    r1   r   z$get_non_head_nodes.<locals>.<lambda>M  s    5 r2   )r  r  r  )r   s    r1   get_non_head_nodesr  K  s*    00'2H2H2J2JKKLLLr2   c                      t           j                            t           j        j        j        j        j                  } |                                  | S )zInitialize error info pub/subr   )	rs   _rayletGcsErrorSubscriberrt   r  global_workerr   r   	subscriber   s    r1   init_error_pubsubr  P  sC    &&#1<D 	' 	 	A KKMMMHr2       .Ac                    t          j                     |z   }g }t          j                     |k     rt          |          |k     r|                     |t          j                     z
            \  }}|snb|||d         k    r|                    |           nt          j        d           t          j                     |k     rt          |          |k     |S )zGets errors from GCS subscriber.

    Returns maximum `num` error strings within `timeout`.
    Only returns errors of `error_type` if specified.
    r  Nr  g{Gz?)r   r   r   ro   r   )
subscribernum
error_typer   deadlinemsgsr  
error_datas           r1   get_error_messager  Y  s     y{{W$HD
)++
 
 SYY__"49;;0FGG: 	z&/A!A!AKK
####Jt )++
 
 SYY__ Kr2   c                      t           j                            t           j        j        j        j        j                  } |                                  | S )zInitialize log pub/subr   )	rs   r  GcsLogSubscriberrt   r  r  r   r   r  r  s    r1   init_log_pubsubr  n  sC    $$#1<D 	% 	 	A KKMMMHr2   r  job_idc                    t          j                     |z   }g }t          j                     |k     rt          |          |k     r|                     |t          j                     z
            }|snr|r||d         k    rfr"t          fd|d         D                       r|                    |           t          j                     |k     rt          |          |k     |S )Nr  jobc              3   0   K   | ]} |           V  d S r   rD   )r   linematchers     r1   	<genexpr>zget_log_data.<locals>.<genexpr>  s.      LLwwt}},LLLLLLr2   lines)r   r   r   allro   )r  r  r   r  r  r  r  	logs_datas       `   r1   get_log_datar  w  s     y{{W$HD
)++
 
 SYY__OOHty{{,BOCC	 	 	f	% 000 	sLLLL79KLLLLL 	I )++
 
 SYY__ Kr2   c                 @    t          | ||||          }d |D             S )zGets log lines through GCS subscriber.

    Returns maximum `num` of log messages, within `timeout`.

    If `job_id` or `match` is specified, only returns log lines from `job_id`
    or when `matcher` is true.
    c                     g | ]
}|d          S )r  rD   r   msgs     r1   r   z#get_log_message.<locals>.<listcomp>  s    )))SCL)))r2   r  r  r  r   r  r  r  s         r1   get_log_messager    s-     
C&'BBD))D))))r2   c                 @    t          | ||||          }d |D             S )z"Get the source of all log messagesc                     h | ]
}|d          S )r   rD   r  s     r1   	<setcomp>z"get_log_sources.<locals>.<setcomp>  s    '''3CJ'''r2   r  r  s         r1   get_log_sourcesr    s-     
C&'BBD''$''''r2   c                    t          j                     |z   }g }t          j                     |k     rt          |          |k     r|                     |t          j                     z
            }|sn\|r||d         k    rf|r ||          st|                    |           t          j                     |k     rt          |          |k     |S )a/  Gets log batches through GCS subscriber.

    Returns maximum `num` batches of logs. Each batch is a dict that includes
    metadata such as `pid`, `job_id`, and `lines` of log messages.

    If `job_id` or `match` is specified, only returns log batches from `job_id`
    or when `matcher` is true.
    r  r  )r   r   r   ro   )r  r  r   r  r  r  batchesr  s           r1   get_log_batchr    s     y{{W$HG
)++
 
 S\\C%7%7OOHty{{,BOCC	 	 	f	% 000 	779-- 	y!!! )++
 
 S\\C%7%7 Nr2   c                 f    |                      dd          } |                     d          sd| z   S | S )zFormat web url.rV   zhttp://127.0.0.1http://)replace
startswith)urls    r1   format_web_urlr    s9    
++k#5
6
6C>>)$$ 3Jr2   c                  .    t           j        j        j        S r   )rs   rt   client_mode_hookis_client_mode_enabledrD   r2   r1   client_test_enabledr    s    <(??r2   c                      t          j                                        dd          } t          j                                        dd          }| |z
  S )z5Returns the number of bytes used in the object store.object_store_memoryr   )rs   cluster_resourcesrC   available_resources)totalavails     r1   object_memory_usager    sJ    !##''(=qAAE#%%))*?CCE5=r2   c              #      K   dd l }| D ]7}	  |j        d| d          }||j        fV  ## |j        j        $ r Y 4w xY wd S )Nr   r  z/metrics)requestsrC   textr   r   )prom_addressesr  r   responses       r1   fetch_raw_prometheusr    s      OOO!  	#x|$?g$?$?$?@@H8=((((("2 	 	 	H		 s   ,>>c                 J   i }i }g }| D ]}||vrt                      ||<   t          |           D ]o\  }}t          |          D ]Z}|j        D ]P}|||j        <   |                    |           d|j        v r&||                             |j        d                    Q[p|||fS )N	Component)setr  r,   samplesr  ro   rX  add)r  rc  rd  re  r   r  metricrU  s           r1   fetch_prometheusr    s    ON! - -/))'*uuOG$1.AA M M4X>> 	M 	MF . M M28"6;/%%f---&-//#G,00{1KLLL	M	M .>>r2   prom_addresesr   c           	         t          |           \  }}}|                                D ]D\  }}||j        vrt                      |j        |<   |j        |                             |           E|j                            |           |D ]H}||j        t          t          |j	                                                  d|j
        fgz             <   I|S )N_metric_name_)r  rW  rc  r  updaterd  re  rk  r  rX  r  )r  r   rc  rd  re  r   
componentsrU  s           r1   rs  rs    s     ;K; ;7O'  /4466 ; ;&000.1eeF"7+w'..z::::
$$%7888     	d6=..0011ov{5S4TTUU	
 	
 Mr2   r  c                     t          |           \  }}}t          t                    }|D ]"}||j                                     |           #|S )zReturn prometheus metrics from the given addresses.

    Args:
        prom_addresses: List of metrics_agent addresses to collect metrics from.

    Returns:
        Dict mapping from metric name to list of samples for the metric.
    )r  r   r  r  ro   )r  r  r	  samples_by_namerU  s        r1   fetch_prometheus_metricsr    sW     %^44MAq'!$''O 4 4$++F3333r2   c                     t          | |          j                                        }t          t                    }|D ]"}||j                                     |           #|S r   )rs  re  rt  r   r  r  ro   )r  r   r	  r  rU  s        r1   "fetch_prometheus_metric_timeseriesr     sh     * VVXX  "$''O 4 4$++F3333r2   r   c                     d                     | j        d                   }t          d|           t          |g|          S )z.Return prometheus timeseries from a RayContextzlocalhost:{}rz  zFetch metrics from)r  r8   r  r  )r   r   metrics_pages      r1   raw_metric_timeseriesr  ,  sC     "(():;P)QRRL	
----|nfEEEr2   system_metric	componentprometheus_server_addressc                    t           j                            t          j        j        j                                                  }d|  d| d| d}t          j	        | dt          |                     }|j        dk    rt          d|j                   |                                }d |d	         d
         D             S )aw  Get the system metric for a given component from a Prometheus server address.
    Please note:
    - This function requires the availability of the Prometheus server. Therefore, it
    requires the server address.
    - It assumes the system metric has a `Component` label and `pid` label. `pid` is the
    process id, so it can be used to uniquely identify the process.
    zsum(z{Component='z',SessionName='z'}) by (pid)z/api/v1/query?query=   zFailed to query Prometheus: c                 D    g | ]}t          |d          d                   S )rS  r   )float)r   items     r1   r   z3get_system_metric_for_component.<locals>.<listcomp>I  s)    IIIE$w-"##IIIr2   datar   )rA   rj   basenamers   rt   r  r  get_session_dir_pathr  rC   r   status_coderX   json)r  r  r  session_namequeryrespr   s          r1   get_system_metric_for_componentr*  5  s     7##(==?? L e=ddydddddE<$HH%,,HH D 3It7GIIJJJYY[[FIIvx0HIIIIr2   c                     t           j                            t                    }t	          j        |          }|j        j        }t           j                            |d|           S )z5Resolve the test config path from the config file dirztests/test_cli_patterns)rA   rj   realpath__file__pathlibPathparentrk   )config_file_namehererj   grandparents       r1   get_test_config_pathr4  L  sJ    7H%%D<D+$K7<<%>@PQQQr2   c                     t          |           }t          j        t          |                                                    }|S )z1Loads a config yaml from tests/test_cli_patterns.)r4  yaml	safe_loadrp   read)r1  config_pathconfigs      r1   load_test_configr;  T  s8    &'788K^D--224455FMr2   c                  &    dd l mc m}  d| _        d S )Nr   zhello world)ray._private.runtime_envrt   runtime_envVAR)r>  s    r1   set_setup_funcr@  [  s(    222222222#KOOOr2   c            
       t    e Zd Zddedee         ddfdZ	 	 	 ddedee         d	ee         dee	         fd
Z
dS )
BatchQueuer   Nmaxsizeactor_optionsr!  c                     |pi }|| _          t          j        t                    j        di |                    | j                   | _        d S )NrD   )rC  rs   r  _BatchQueueActoroptionsrC  )rY  rC  rD  s      r1   __init__zBatchQueue.__init__b  sO    %+0CJ'((0AA=AAHHVV 	


r2   
batch_sizetotal_timeoutfirst_timeoutc                 h    t          j        | j        j                            |||                    S )zGets batch of items from the queue and returns them in a
        list in order.

        Raises:
            Empty: if the queue does not contain the desired number of items
        )rs   rC   rC  	get_batchr  )rY  rI  rJ  rK  s       r1   rM  zBatchQueue.get_batchi  s2     wJ ''
M=QQ
 
 	
r2   )r   NNNN)r\  r]  r^  rI   r   r   rH  r   r   r
   rM  rD   r2   r1   rB  rB  a  s        
 
 
 
RV 
 
 
 
 )-)-	
 

  
  	

 
c
 
 
 
 
 
r2   rB  c                       e Zd ZddZdS )rF  Nc                   K   t          j                    }	 t          j        | j                                        |           d {V }|g}|r)t          j                    }t          |||z
  z
  d          }n# t          j        $ r t          w xY w||d}	 	 t          j                    }|	                    t          j        | j                                        |           d {V            |r)t          j                    }t          |||z
  z
  d          }n# t          j        $ r Y nw xY wnt          |dz
            D ]}	 t          j                    }|	                    t          j        | j                                        |           d {V            |r)t          j                    }t          |||z
  z
  d          }# t          j        $ r Y  nw xY w|S )Nr   Tr   )timeitdefault_timerasynciowait_forqueuerC   maxr   r"   ro   r   )	rY  rI  rJ  rK  rG  firstbatchendr  s	            r1   rM  z_BatchQueueActor.get_batch{  s=     $&&	!*4:>>+;+;]KKKKKKKKEGE F*,, #MS5[$A1 E E# 	 	 	K	$ !
	"022ELL%.tz~~/?/?OOOOOOOO   % N$244(+MS5[,I1(M(M+   E
 :>** 
 
	"022ELL%.tz~~/?/?OOOOOOOO   % N$244(+MS5[,I1(M(M+   EEs2   A A8 8BBD D0/D0
BGG! G!rN  )r\  r]  r^  rM  rD   r2   r1   rF  rF  z  s(        $ $ $ $ $ $r2   rF  c                 d    t           j                            |           }d|vrdS |d         dk    S )Nr2  FREMOVED)rs   utilplacement_group_table)pgtables     r1   is_placement_group_removedr`    s6    H**2..Eeu>Y&&r2   c                    | D ]!}t           j                            |           "d }t          |           t          j                                        d                               d           fd}t          |           d S )Nc                      t           j                                                                        D ]} | d         dk    r dS dS )Nr2  r[  FT)rs   r\  r]  rt  )pg_entrys    r1   wait_for_pg_removedz;placement_group_assert_no_leak.<locals>.wait_for_pg_removed  sL    6688??AA 	 	H I--uu .tr2   memoryr  c                      t          j                                                    D ]\  } }| v r|          |k    r dS d| v r dS  dS )NF_group_T)rs   r  rW  )resourcevalr  s     r1   wait_for_resource_recoveredzCplacement_group_assert_no_leak.<locals>.wait_for_resource_recovered  sk     466<<>> 	 	MHc,,,1B81LPS1S1SuuH$$uu %tr2   )rs   r\  remove_placement_groupr   r  pop)pgs_createdr^  rd  rj  r  s       @r1   placement_group_assert_no_leakrn    s     , ,''++++   *+++-//(###/000     233333r2         ?print_interval_srecord_interval_swarning_thresholdc                    t          j                    s
J d            t          j        d           G d d                      }t           j        j        j        j        }|                    d| di                              | ||	          }t          d
           t          j	        |j
                                                   t          d           |j                                         |S )ah  Run the memory monitor actor that prints the memory usage.

    The monitor will run on the same node as this function is called.

    Params:
        interval_s: The interval memory usage information is printed
        warning_threshold: The threshold where the
            memory usage warning is printed.

    Returns:
        The memory monitor actor.
    2The API is only available when Ray is initialized.r   num_cpusc            	       H    e Zd Z	 	 	 	 ddedededefd	Zd
 Zd Zd Zd Z	dS )0monitor_memory_usage.<locals>.MemoryMonitorActorr   rp  rq  r4  rr  rs  rt  nc                     || _         || _        d| _        || _        t	          j                    | _        || _        d| _        d| _	        d| _
        t          j        t          j                   dS )a  The actor that monitor the memory usage of the cluster.

            Params:
                print_interval_s: The interval where
                    memory usage is printed.
                record_interval_s: The interval where
                    memory usage is recorded.
                warning_threshold: The threshold where
                    memory warning is printed
                n: When memory usage is printed,
                    top n entries are printed.
            Fr   r)   levelN)rr  rs  
is_runningrt  memory_monitorMemoryMonitormonitorr{  peak_memory_usagepeak_top_n_memory_usage_last_print_timeloggingbasicConfigINFO)rY  rr  rs  rt  r{  s        r1   rH  z9monitor_memory_usage.<locals>.MemoryMonitorActor.__init__  sp    ( %5D!%6D"#DO%6D")799DLDF%&D" ,.D($%D!gl333333r2   c                     d S r   rD   rh  s    r1   readyz6monitor_memory_usage.<locals>.MemoryMonitorActor.ready  s    Dr2   c                 `  K   d| _         | j         rt          j                    }| j                                        \  }}t	          j        | j                  }|| j        k    r|| _        || _        ||| j	        z  k    rt          j        d||z  dz   d           || j        z
  | j        k    rRt          j        d| d|            t          j        d| j         d	           t          j        |           || _        t          j        | j                   d
{V  | j         d
S d
S )zRun the monitor.T)r{  zThe memory usage is high: r;  %zMemory usage: z / zTop z process memory usage:N)r  r   r  get_memory_usager  get_top_n_memory_usager{  r  r  rt  r  r   r  rr  r   rS  r   rs  )rY  nowused_gbtotal_gbtop_n_memory_usages        r1   runz4monitor_memory_usage.<locals>.MemoryMonitorActor.run  sb     "DO/ <ikk$(L$A$A$C$C!%3%JTV%T%T%T"T333-4D*3ED0X(>>>>OS(8JS8PSSS   ..1FFFL!H'!H!Hh!H!HIIIL!F!F!F!FGGGL!3444,/D)mD$:;;;;;;;;;# / < < < < <r2   c                 &   K   | j         }d| _         |S )z}Stop running the monitor.

            Returns:
                True if the monitor is stopped. False otherwise.
            F)r  rY  was_runnings     r1   stop_runz9monitor_memory_usage.<locals>.MemoryMonitorActor.stop_run  s       /K#DOr2   c                 "   K   | j         | j        fS )zReturn the tuple of the peak memory usage and the
            top n process information during the peak memory usage.
            )r  r  rh  s    r1   get_peak_memory_infozEmonitor_memory_usage.<locals>.MemoryMonitorActor.get_peak_memory_info%  s       )4+GGGr2   N)r   rp  rq  r4  )
r\  r]  r^  r   rI   rH  r  r  r  r  rD   r2   r1   MemoryMonitorActorrz    s         ')'('*'	4 '	4#'	4  %'	4  %	'	4
 '	4 '	4 '	4 '	4R	 	 		< 	< 	<,	 	 		H 	H 	H 	H 	Hr2   r  znode:gMbP?)	resources)rr  rs  rt  z/Waiting for memory monitor actor to be ready...z"Memory monitor actor is ready now.)rs   is_initializedr  rt   r  r  node_ip_addressrG  r  rC   r  r  )rr  rs  rt  r  current_node_ipmemory_monitor_actors         r1   monitor_memory_usager    s*   " UU!UUUZQH QH QH QH QH QH QH QHf l)7GO-55,?,,e4 6  f)++     

;<<<G &--//000	
.///##%%%r2   c                     ddl } t          j        dk    r|                     d           t	                      \  }}t          j        d          }t          j        	                    |d          }t          j        	                    |d          }t          |d          5 }|                    |           ddd           n# 1 swxY w Y   t          |d          5 }|                    |           ddd           n# 1 swxY w Y   d	t          j        d
<   |t          j        d<   |t          j        d<   |t          j        d<   |||fS )z.Sets up required environment variables for tlsr   Ndarwinz2Cryptography doesn't install in Mac build pipelinezray-test-certsz
server.crtz
server.keyrg   r@   RAY_USE_TLSRAY_TLS_SERVER_CERTRAY_TLS_SERVER_KEYRAY_TLS_CA_CERT)pytestr   r   skipr   tempfilemkdtemprA   rj   rk   rp   rq   rB   )r  certkeytemp_dircert_filepathkey_filepathfhs          r1   	setup_tlsr  ;  s   MMM
|xHIII.00ID# 011HGLL<88M7<<,77L	mS	!	! R
              	lC	 	  B
               !$BJ}(5BJ$%'3BJ#$$1BJ !00s$   CCCC>>DDc                     t          j        |            t          j        |           t          j        |           t           j        d= t           j        d= t           j        d= t           j        d= d S )Nr  r  r  r  )rA   remove
removedirsrB   )r  r  r  s      r1   teardown_tlsr  R  sc    IlImM(

=!

()

'(

$%%%r2   c                   |    e Zd ZdZ	 	 	 	 	 ddeded	ee         d
edee         f
dZd Z	d Z
d Zd Zd Zd Zd ZdS )ResourceKillerActoraJ  Abstract base class used to implement resource killers for chaos testing.

    Subclasses should implement _find_resource_to_kill, which should find a resource
    to kill. This method should return the args to _kill_resource, which is another
    abstract method that should kill the resource and add it to the `killed` set.
    <   r      r   Nkill_interval_skill_delay_smax_to_killbatch_size_to_killkill_filter_fnc                 $   || _         || _        d| _        || _        t	                      | _        t                                                      | _        || _	        || _
        || _        d| _        t          j        t          j                   d S )NFr}  )r  r  r  head_node_idr  killedr   create_futuredoner  r  r  kill_immediately_after_foundr  r  r  )rY  r  r  r  r  r  r  s          r1   rH  zResourceKillerActor.__init__d  s      /((ee,..<<>>	&"4,,1)',//////r2   c                     d S r   rD   rh  s    r1   r  zResourceKillerActor.readyz  s    r2   c                 0  K   d| _         t          j        | j                   | j         r|                                  d {V }| j         sn| j        rd}n/t          j                    | j        z  }t          j        |           |D ]} | j        |  | j	        t          | j                  | j	        k    rn)t          j        | j        |z
             d {V  | j         | j                            d           |                                  d {V  d S )NTr   )r  r   r   r  _find_resources_to_killr  randomr  _kill_resourcer  r   r  rS  r  
set_resultr  )rY  to_killssleep_intervalto_kills       r1   r  zResourceKillerActor.run}  sJ     
4$%%%o 	G!99;;;;;;;;H? 0 +!"!'43G!G
>***# . .##W---+DK0@0@DDT0T0T- 4~ EFFFFFFFFF! o 	G$ 		T"""mmoor2   c                    K   t           r   NotImplementedErrorrh  s    r1   r  z+ResourceKillerActor._find_resources_to_kill  s      !!r2   c                     t           r   r  )rY  r/   s     r1   r  z"ResourceKillerActor._kill_resource  s    !!r2   c                 R   K   | j         }|r|                                  d| _         |S r  )r  _cleanupr  s     r1   r  zResourceKillerActor.stop_run  s/      o 	MMOOOr2   c                 .   K   | j          d{V  | j        S )z(Get the total number of killed resourcesN)r  r  rh  s    r1   get_total_killedz$ResourceKillerActor.get_total_killed  s!      i{r2   c                     dS )zbCleanup any resources created by the killer.

        Overriding this method is optional.
        NrD   rh  s    r1   r  zResourceKillerActor._cleanup  s	    
 	r2   )r  r   r  r   N)r\  r]  r^  ri  r   r   rI   r   rH  r  r  r  r  r  r  r  rD   r2   r1   r  r  \  s          "$%&"#-10 0 0 	0
 c]0  0 !*0 0 0 0,    4" " "" " "    
    r2   r  c                       e Zd Zd ZdS )NodeKillerBasec                    K   g }|sډ j         rӈ fdt          j                    D             } j        r0t	          t                                           |                    }n|}t          |           j        dz   k     rt          j	        d           d {V  |d  j                 D ],}|
                    |d         |d         |d         f           -|s j         |S )Nc                 d    g | ],}|d          r"|d         j         k    r|d         j        v*|-S )AliveNodeID)r  r  )r   r  rY  s     r1   r   z:NodeKillerBase._find_resources_to_kill.<locals>.<listcomp>  sV       = (^t'888(^4;66	  766r2   r   r  rq  NodeManagerPort)r  rs   rx  r  r  r  r   r  rS  r   ro   )rY  nodes_to_killworker_nodes
candidates	candidates   `    r1   r  z&NodeKillerBase._find_resources_to_kill  s:      	DO 	   IKK  L " *!&)<)<)>)>"M"MNN

)
 <  4#:Q#>>>mA&&&&&&&&& ((A$*A(AB  	$$!(+!"67!"34   +   	DO 	: r2   N)r\  r]  r^  r  rD   r2   r1   r  r    s#            r2   r  rw  c                       e Zd Zd ZddZdS )RayletKillerc                     |c	 |                      ||d           n# t          $ r Y nw xY wt          j        d| d| d|            | j                            |           d S d S )NFgracefulzKilled node z at address: z, port: )_kill_rayletrX   r  r   r  r
  )rY  r   node_to_kill_ipnode_to_kill_ports       r1   r  zRayletKiller._kill_resource  s    (!!/3Du!UUUU   L@w @ @"@ @,=@ @   KOOG$$$$$ )(s    
**Fc                    dd l }ddlm} ddlm} t          ||          }|                    |          }|                    |          }		 |	                    t          j
        |                     d S # |$ r |rJ Y d S w xY w)Nr   _InactiveRpcErrornode_manager_pb2_grpcr  grpcgrpc._channelr  ray.core.generatedr  r   insecure_channelNodeManagerServiceStubShutdownRayletr!   ShutdownRayletRequest)
rY  r  r_   r  r  r  r  raylet_addresschannelstubs
             r1   r  zRayletKiller._kill_raylet  s    333333<<<<<<&r400''77$;;GDD	  6III     ! 	  	  	 <	 s   (A6 6	BBNr  )r\  r]  r^  r  r  rD   r2   r1   r  r    s7        
% 
% 
%           r2   r  c                       e Zd Zd ZdS )EC2InstanceTerminatorc                     |W	 t          |           n# t          $ r Y nw xY wt          j        d|d|            | j                            |           d S d S )NzTerminated instance, node_id=z
, address=)_terminate_ec2_instancerX   r  r   r  r
  )rY  r   r  r  s       r1   r  z$EC2InstanceTerminator._kill_resource  s    &'8888   LVVV_VVWWWKOOG$$$$$ '&s    
!!N)r\  r]  r^  r  rD   r2   r1   r  r    s#        % % % % %r2   r  c                   F     e Zd Zdddef fdZd Zdeddfd	Zd
 Z xZ	S )$EC2InstanceTerminatorWithGracePeriodro  )grace_period_sr   c                n     t                      j        |i | || _        t                      | _        d S r   )superrH  _grace_period_sr  _kill_threads)rY  r   r/   r0   	__class__s       r1   rH  z-EC2InstanceTerminatorWithGracePeriod.__init__  s9    $)&)))-47EEr2   c                     | j         vsJ  j                                        D ]D}|                                s.|                                  j                            |           E fd}t                              d|d|           t          j	        |||fd          }|
                                  j                            |            j                             |           d S )Nc                                          |            t          j        j                   t	          d|           d S )Nzray stop --force)_drain_noder   r   r  _execute_command_on_node)r   r  rY  s     r1   _kill_node_with_grace_periodzYEC2InstanceTerminatorWithGracePeriod._kill_resource.<locals>._kill_node_with_grace_period  sC    W%%%Jt+,,,
 %%7IIIIIr2   z Starting killing thread node_id=z, node_to_kill_ip=T)targetr/   daemon)r  r  copyis_aliverk   r  r   r   	threadingThreadrG  r
  )rY  r   r  r  threadr
  s   `     r1   r  z3EC2InstanceTerminatorWithGracePeriod._kill_resource  s   dk)))) (--// 	2 	2F??$$ 2"))&111	J 	J 	J 	J 	J 	MwMM?MMNNN!/?+
 
 

 	v&&&     r2   r   r!  Nc                 f   ddl m} t          j                            |          t          j                                        k    sJ t          j        d|           t          j	        d          }t          j
                            |          }t          j                    dz  | j        dz  z   }	 |                    ||j                            d	          d
|          \  }}n;# t          j        j        $ r$}t(                              d|           |d }~ww xY w|s
J d            d S )Nr   )autoscaler_pb2zDraining node node_id=auto)r{  r   r  r?  DRAIN_NODE_REASON_PREEMPTIONr)   zFailed to drain node node_id=zDrain node request was rejected)r  r  rs   r  from_hexnilr  r   ru   %canonicalize_bootstrap_address_or_dier  r   r   time_nsr  
drain_nodeDrainNodeReasonValuer   RayErrorr   r  )	rY  r   r  r   r   deadline_timestamp_msis_acceptedr  r   s	            r1   r  z0EC2InstanceTerminatorWithGracePeriod._drain_node$  sF    	655555z""7++sz~~/?/?????0g00111@fMMM[**7*;;
!%3!64;ORU;U V		'22.445STT%	 NK ~& 	 	 	LL;;;<<<G	 ====={==s   63C* *D">DD"c                     | j                                         D ]0}|                                 | j                             |           1| j         rJ d S r   )r  r  rk   r  )rY  r  s     r1   r  z-EC2InstanceTerminatorWithGracePeriod._cleanup=  s\    (--// 	. 	.FKKMMM%%f----%%%%%%r2   )
r\  r]  r^  rI   rH  r  r^   r  r  __classcell__r  s   @r1   r  r     s        46 : : :c : : : : : :! ! !8>3 >4 > > > >2& & & & & & &r2   r  c                   *     e Zd Z fdZd Zd Z xZS )WorkerKillerActorc                      t                      j        |i | d| _        ddlm} ddlm}  |            | _         |ddg          | _        d S )NTr   )StateApiClient)ListApiOptions)r2  r6  RUNNING)r  z!=zWorkerKillActor.run)r7  )	r  rH  r  ray.util.state.apir&  ray.util.state.commonr'  clienttask_options)rY  r/   r0   r&  r'  r  s        r1   rH  zWorkerKillerActor.__init__G  s    $)&))) -1)555555888888$n&&*N)5
 
 
r2   c                   K   ddl m} d }d }d }|| j        r| j                            |j        | j        d          }| j        /t	          t          |                                 |                    }|D ]'}|j	        |j
        |j        }|j        }|j
        } n(t          j        d           d {V  || j        |||fgS )Nr   )StateResourceF)rG  raise_on_missing_outputr   )r*  r.  r  r+  r  TASKSr,  r  r  	worker_idr   task_id
worker_pidrS  r   )rY  r.  process_to_kill_task_idprocess_to_kill_pidprocess_to_kill_node_idtaskstasks          r1   r  z)WorkerKillerActor._find_resources_to_killY  s     777777"&""&!)do)K$$#)(- %  E
 ".VD$7$7$9$95AABB  >-$,2J.2l+*./'.2l+E -$$$$$$$$$# ")do)& )*=?VWXXr2   c                 B   |t           j        d             }t           j        j                            |d          }|                    |                              |           t          j        d| d|            | j        	                    ||f           d S d S )Nc                 V    dd l } |j        |           }|                                 d S Nr   )r   r   r  )r   r   r   s      r1   kill_processz6WorkerKillerActor._kill_resource.<locals>.kill_processy  s-    %v~c**		r2   Fr   soft)scheduling_strategyzKilling pid z	 on node )
rs   r  r\  scheduling_strategiesr%   rG  r  r   r  r
  )rY  r4  r5  r6  r<  r?  s         r1   r  z WorkerKillerActor._kill_resourcet  s     *Z  Z .MM3 N       5H IIPP#   LV2VV=TVV   KOO46IJKKKKK- +*r2   )r\  r]  r^  rH  r  r  r!  r"  s   @r1   r$  r$  E  s`        
 
 
 
 
$Y Y Y6L L L L L L Lr2   r$  r  c	                    t          j                    s
J d            t          j                                                    }	|                     t          |	d          |d|                              |	|||||          }
t          d           t          j        |
j	                                                   t          d           |s|
j
                                         |
S )	Nrv  Fr=  ResourceKiller)r?  r   r  lifetime)r  r  r  r  r  z)Waiting for ResourceKiller to be ready...zResourceKiller is ready now.)rs   r  get_runtime_contextget_node_idrG  r%   r  r  rC   r  r  )resource_killer_clsr  r   rC  no_startr  r  r  r  r  resource_killers              r1   get_and_run_resource_killerrI    s    UU!UUU*,,88::L)11: u
 
 
  2   f'!-%     

5666GO!((**+++	
())) %""$$$r2   actor_handlezray.actor.ActorHandlec                 \    t          j        | j                            d                     S )Nc                 L    t          j                                                    S r   )rs   rD  rE  rh  s    r1   r   z#get_actor_node_id.<locals>.<lambda>  s    022>>@@ r2   )rs   rC   __ray_call__r  )rJ  s    r1   get_actor_node_idrN    s2    7!((@@	
 	
  r2   dc              #      K   t          j                    }t          j        |            	 d V  t          j        |           d S # t          j        |           w xY wr   )rA   getcwdchdir)rO  old_dirs     r1   rR  rR    sX      ikkGHQKKK
s   A Ac                      t          j                    5 } t          |           5  t          j        j                            |           dk    sJ t          dd          5 }|                    t          j
        d                     d d d            n# 1 swxY w Y   t          j        j                            |           dk    sJ t          dd          5 }|                    t          j
        d                     d d d            n# 1 swxY w Y   t          j        j                            |           dk    sJ t          j        d           t          d	d          5 }|                    t          j
        d
                     d d d            n# 1 swxY w Y   t          j        j                            |           dk    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   	test_filewbr;  test_file_22      subdirzsubdir/subdir_filer     )r  TemporaryDirectoryrR  rs   rt   r  get_directory_size_bytesrp   rq   rA   urandommkdir)tmp_dirr|   s     r1   test_get_directory_size_bytesra    s   		$	&	& K'5>> K K|!::7CCqHHHH+t$$ 	%GGBJsOO$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%|!::7CCsJJJJ-&& 	$!GGBJrNN###	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$|!::7CCsJJJJ
&-- 	#GGBJqMM"""	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	#|!::7CCsJJJJJK K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K Ks   G3;G(BGBGB=G(D GDGDAG%(FGFG F!-GG3G	G3"G	#G33G7:G7c                 Z   |                                  D ]}dD ]}t          j        t          j                            |                                |                    }t          t          d |                    }t          |          dk    rt          d| d|              dS dS )N)condapipworking_dir_filespy_modules_filesc                 .    |                      d           S )N)z.lock.txt)endswith)r|   s    r1   r   z(check_local_files_gced.<locals>.<lambda>  s    ajj9J.K.K*K r2   r   z(runtime_env files not GC'd from subdir 'z': FT)
r  rA   listdirrj   rk   get_runtime_env_dir_pathr  r  r   r  )r   r  rZ  	all_filesrW  s        r1   check_local_files_gcedrm    s    &&((  O 	 	F
T::<<fEE I  K KYWWXXE5zzA~~SSSESSTTTuuu 	 4r2   c                    |dg}| dk    ruddd|igi}|dk    rZ|dt          t          |                     dz  }|                    t          j        |                     t          |          }n|dk    r|}d|i}ns| dk    rm|dk    r[|d	t          t          |                     d
z  }|                    d                    |                     t          |          }	n|dk    r|}	d|	i}|S )Nzpip-install-test==0.5rc  dependenciesrd  filezenvironment-z.ymlpython_objectzrequirements-rh  rh   )hashr^   
write_textr6  dumprk   )
r   spec_formattmp_pathpip_list
conda_dict
conda_filerc  r>  pip_filerd  s
             r1   generate_runtime_env_dictr{    s)   +,$uuh.?&@A
&  !$L4H3F3F$L$L$LLJ!!$)J"7"7888
OOEEO++E&	%&  "K$s8}}2E2E"K"K"KKH		( 3 3444h--CCO++Cclr2   c                 B      fd}t          |dd           d S )Nc                  l   t          d         d          } t          |            rd                              | vrdS nd| v rdS r@t          t                    sg}n}d}|D ]}d                    |          | v rd}|sdS nd| v rdS rd	                              | vrdS nd
| v rdS dS )Nr   T)r   
stats_onlyzRestored {} MiBFRestoredzSpilled {} MiBSpilledz$Plasma filesystem mmap usage: {} MiBzPlasma filesystem mmap usage:)r   r  r  r  r  )r   spilled_lstfoundr{  r   fallbackrestoredspilleds       r1   okzcheck_spilled_mb.<locals>.ok   s    79#5$GGGa 	 ''11::u ; Qu 	gt,, &&i%E  ! !#**1--22 E u A~~u 	5<<XFFaOOu P /!33utr2      r}  )r   rD  r   )r   r  r  r  r  s   ```` r1   check_spilled_mbr    sK           > r1======r2   c                      t          j                    } t          j                    }t          j                    D ]}d|v r| |= ||= | |k    S )Nr  )rs   r  r  )r  r  rs      r1   *no_resource_leaks_excluding_node_resourcesr  "  s^    -//133"$$ ' 'Q;;!!$#A& 333r2   c                      d                     | d                   }t          d|            t          j        d           dS )z6Function called by reflection by test_cli_integration.r   
entrypointzhook intercepted: r   N)rk   r  r   exit)r0   cmds     r1   job_hookr  -  s@    
((6,'
(
(C	
$s
$
$%%%HQKKKKKr2   c                      dS )zg
    Example external hook to set up W&B API key in
    WandbIntegrationTest.testWandbLoggerConfig
    abcdrD   rD   r2   r1   wandb_setup_api_key_hookr  4  s	    
 6r2   c                 >   dd l }ddlm} ddlm} t          | d         | d                   } ||          }|                    |          }t          |          D ]<}		 |                    t          j
                    |          }
 n# |j        $ r Y 9w xY w|
J |
S )Nr   init_grpc_channelr  rq  r  r  )r  ray._private.grpc_utilsr  r  r  r   r  r   GetNodeStatsr!   GetNodeStatsRequestRpcError)raylet	num_retryr   r  r  r  r  r  r  r  replys              r1   get_node_statsr  =  s    KKK999999888888"#$f->&? N  //G 77@@D9  	%% 466 &  E E} 	 	 	H	Ls   (B		
BBc                 
   ddl m} ddlm} | st          j        j        j        }  || t          j	        d          }|
                    |          }t          j                    }|                    ||          }|j        }|S )Nr   r  )gcs_service_pb2_grpcF)asynchronousr  )r  r  r  r  rs   r  r  r4   r   GLOBAL_GRPC_OPTIONSNodeResourceInfoGcsServiceStubr    GetAllResourceUsageRequestGetAllResourceUsageresource_usage_data)	r4   r   r  r  gcs_channelgcs_node_resources_stubrequestr  resources_batch_datas	            r1   get_resource_usager  U  s    999999777777 :j-9##]6U  K 3QQ  8::G&::7G:TTH#7r2   c                     t          |           } t          j        |  d          }|                                 |                                d         d         d         S )Nz/api/cluster_statusr"  clusterStatusloadMetricsReport)r  r  rC   raise_for_statusr&  )	webui_urlr  s     r1   get_load_metrics_reportr  l  sW    y))I|y===>>H==??6"?34GHHr2   c                 $   dd l }ddlm} ddlm} t          | d         | d                   }|                    |          }|                    |          }	 |                    t          j
        |                     d S # |$ r |rJ Y d S w xY w)Nr   r  r  rq  r  r  r  )r  r  r  r  r  r  r  r  s           r1   kill_rayletr  t  s    KKK//////888888"#$f->&? N ##N33G 77@@D,BHUUUVVVVV   |s   (B 	BBc                      dd l fd j        g d          D             } d| v sJ t          |                                           S )Nr   c                     i | ]J}|j         d          j        j        fvr-|j         d         dv ,|j         d         |j         d         j        KS )r   r  )
gcs_serverzredis-servermemory_info)r   r   STATUS_DEADrss)r   r   r   s     r1   
<dictcomp>z'get_gcs_memory_used.<locals>.<dictcomp>  si     	 	 	Ih(<f>P'QQQ	&!%CCC	 		&49]37 DCCr2   )r   r  r  r  )r   r
  sumrt  )mr   s    @r1   get_gcs_memory_usedr    so    MMM	 	 	 	'F'(I(I(IJJ	 	 	A 1qxxzz??r2   /tmp/release_test_output.jsonTEST_OUTPUT_JSONdefault_file_nameenv_varc                    t           j                            ||          }| dt          t	          j                               }t          |d          5 }t          j        | |           |	                                 ddd           n# 1 swxY w Y   t          j
        ||           t                              d|            t                              t          j        |                      dS )z
    Safe (atomic) write to file to guard against malforming the json
    if the job gets interrupted in the middle of writing.
    z.tmp.wtNzWrote results to )rA   rB   rC   r^   rl   rm   rp   r&  rt  r   r  r   r   dumps)r   r  r  test_output_jsontest_output_json_tmpr|   s         r1   safe_write_to_results_jsonr    s    z~~g/@AA.HHS5F5FHH	"D	)	) Q	&!				               J#%5666
KK6$466777
KK
6""#####s   *BBBc                      t          j         t           j        t           j                  } |                     d           |                                 d         }|                                  |S )a/  
    Returns a port number that is not currently in use.

    This is useful for testing when we need to bind to a port but don't
    care which one.

    Returns:
        A port number that is not currently in use. (Note that this port
        might become used by the time you try to bind to it.)
    )rV   r   r   )r  r  r  bindgetsocknamer*  )sockr_   s     r1   get_current_unused_portr    sW     =);<<D 	IIa DJJLLLKr2   c                      t           dz  a ddlm} m}  G d d| |j                  }d |dd	t            t          j                                                    
          iS )
    Example external hook for test_component_activities_hook.
    Returns valid response and increments counter in `reason`
    field on each call.
    r   r   )	BaseModelExtrac                   B    e Zd ZU dZeed<   dZee         ed<   eed<   dS )Dexternal_ray_cluster_activity_hook1.<locals>.TestRayActivityResponsez
        Redefinition of dashboard.modules.api.api_head.RayActivityResponse
        used in test_component_activities_hook to mimic typical
        usage of redefining or extending response type.
        	is_activeNreason	timestamp)	r\  r]  r^  ri  r^   r_  r  r   r   rD   r2   r1   TestRayActivityResponser    sG         	 	  $$$$r2   r  )extratest_component1ACTIVE	Counter: r  r  r  )!ray_cluster_activity_hook_counterpydanticr  r  allowr	   r  r  )r  r  r  s      r1   #external_ray_cluster_activity_hook1r    s     &*%))))))))	 	 	 	 	)5; 	 	 	 	 	22B@BBlnn..00
 
 
 r2   c                  
    ddiS )z
    Example external hook for test_component_activities_hook.
    Returns invalid output because the value of `test_component2`
    should be of type RayActivityResponse.
    test_component2
bad_outputrD   rD   r2   r1   #external_ray_cluster_activity_hook2r    s     |,,r2   c                      dS )z
    Example external hook for test_component_activities_hook.
    Returns invalid output because return type is not
    Dict[str, RayActivityResponse]
    r  rD   rD   r2   r1   #external_ray_cluster_activity_hook3r    s	     <r2   c                       t          d          )z`
    Example external hook for test_component_activities_hook.
    Errors during execution.
    z'Error in external cluster activity hook)rX   rD   r2   r1   #external_ray_cluster_activity_hook4r    s    
 =
>
>>r2   c                  |    t           dz  a dddt            t          j                                                    diS )r  r   test_component5r  r  r  )#ray_cluster_activity_hook_5_counterr	   r  r  rD   r2   r1   #external_ray_cluster_activity_hook5r    sJ     (1,'!G"EGG!1133
 
 r2   c                  ,    dd l mc mc m}  d | _        d S r;  )ray.autoscaler.v2.utils
autoscalerv2r  cached_is_autoscaler_v2)us    r1   !reset_autoscaler_v2_enabled_cacher    s4    '''''''''''' $Ar2   node_ipc                 X    t          j        d|             d}t          ||            d S )NzTerminating instance a#  instanceId=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id/);region=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region);aws ec2 terminate-instances --region $region --instance-ids $instanceId)r  r   r	  )r  r}   s     r1   r  r    s>    L222333	R 
 Wg.....r2   r}   c                    t          j        d| d|             d|  }d| d| d}	 t          j        |dddd           d S # t          j        $ r5}t          d	|j                   t          d
|j                   Y d }~d S d }~ww xY w)NzExecuting command on node z: zpTOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600");zLssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2222 ray@z ''T)shellcapture_outputr  checkz
Exit code:zStderr:)r  debugr   r  r   r  r   r   )r}   r  multi_line_commandssh_commandr   s        r1   r	  r	     s    MCwCC'CCDDD		 	  Bah  B  Bl~  B  B  BK#tDt4	
 	
 	
 	
 	
 	
 ( # # #lAL)))i"""""""""#s   A B	*BB	z100:0:0z0:100:0z0:0:100)r  r  	in_flight)NFNr   )Nr   NNNNNFFNNNr   )FF)r   )r   r   r  )Nr  r   )Nr4  )r4  r;  )r4  r;  F)r  r;  )r  Nr   )r  r   NN)r   NN)ro  rp  rq  )NNFr  r   r   NrN  )rp  r  )r4  )r  r  )rS  r  r   r&  r  rA   r.  r  r  r   r   r  r  r   rQ  r  rl   collectionsr   
contextlibr   r   r   dataclassesr   r   r	   typingr
   r   r   r   r   r   r   urllib.parser   r  r6  rs   ray._private.memory_monitorrt   r  ray._private.servicesru   ray._private.utilsray.dashboard.consts	dashboardconstsr   ray._common.network_utilsr   r   ray._common.test_utilsr   ray._common.utilsr   ray._privater   ray._private.internal_apir   ray._private.tls_utilsr   ray._private.workerr   rY   r   r   r   r   r  r   r    r!   ray.util.queuer"   r#   r$   ray.util.scheduling_strategiesr%   ray.util.stater&   r'   r   	getLoggerr\  r   r   r  rj   abspathdirnamer-  RAY_PATHrk   rr   prometheus_client.corer*   prometheus_client.parserr+   r,   ImportErrorr.   r=   rE   rJ   rN   rb   r^   rI   boolr   r   r   r   r   r   r   r   r   r   r  r  r   r'  r,  r3  r:  rI  rP  rR  ra  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  rj  r  r  r  r  r  r  r  r  r  rs  r  r  r  r*  r4  r;  r@  rB  rF  r`  rn  r  r  r  r  r  r  r  r  r$  rI  rN  rR  ra  rm  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  RPC_FAILURE_MAPr  keysRPC_FAILURE_TYPESrD   r2   r1   <module>r     s	     				   				        



             # # # # # # G G G G G G G G G G ( ( ( ( ( ( ( (       B B B B B B B B B B B B B B B B B B         



 4 4 4 4 4 4 4 4 4     ( ( ( ( ( ( ( ( (     / / / / / / / / / B B B B B B B B 5 5 5 5 5 5 6 6 6 6 6 6      5 4 4 4 4 4 A A A A A A * * * * * * P P P P P P P P P P P P         
 5 4 4 4 4 4 4 4 4 4 I I I I I I 1 1 1 1 1 1 1 1 		8	$	$|w..VVB
7??27??27??8+D+DEEFF7<<>K  C------OOOOOOOOO() C C CFFC C C C C	C	! 	! 	!8 8 8D D DM M M9 9 98 #6 66
6 6 sm	6
 6 6 6 6x (,!%!%"!%)-%*M! M!M!
M!  }M! 	M!
 #M! #M! smM! M! DI&M! #M! M! M! M! M!`  &$ $ $ $:   4J J J J
J 
J 
JN Nc NE N N N N    

 

 

 

     $ s    < 9@    ! 25 
38_       D     <U U UD D D D$V V V V$  "	%& %&!%&%& %& 	%&
 %& %& %& %&P        , $ $ $ $ $ $ $ $* "&) )./)$) #) b$h	) ) ) )Z )-SX' ' ' ' 'T	 	 	    &           (   M M M
     *     	  SM	 
$Z   0  * *	* * SM	* 
$s)_* * * *(  	( 	(		( 	( SM		( 	( 	( 	(   	  SM	 
#Y   >  @T @ @ @ @T    	 	 	? ? ?&9     (T#Y 4T#Y;O     	I	';		#tCy.	 	 	 	F
F2F	#tCy.F F F FJJ#&JCFJ	%[J J J J.R R R  $ $ $
 
 
 
 
 
 
 
2% % % % %{ % % %P' ' '4 4 48 "t  t t t  t  t  t  t n1 1 1.& & &S S S S S S S Sl         (      F Q         >      < Q% % % % %N % % % QA& A& A& A& A&> A& A& A&H QGL GL GL GL GL+ GL GL GLZ # # # #L$;      S    K K K  "   . >  >  >  >F4 4 4       0       .I I I   "  " =/$ $$$ c]$ $ $ $&  . %& !&' #  >- - -  ? ? ?  &% % %	/S 	/T 	/ 	/ 	/ 	/#c #C # # # #,   D--//00   s   F F0/F0