&`iz#ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZddlmZmZmZmZmZddlmZddlZddlmcmZddlZddlmZm Z m!Z!m"Z"ddl#m$Z$m%Z%ddl&m'Z'ddl(m)Z)dd l*m+Z+dd lm,Z,m-Z-dd l.m/Z/m0Z0m1Z1m2Z2dd l3m4Z4m5Z5ddl6Z6ej7e8Z9Gd dZ:dS)N) defaultdict)IOAnyStrDictOptionalTuple)FileLock) build_addressget_localhost_ipis_ipv6 parse_address)LOGGING_ROTATE_BACKUP_COUNTLOGGING_ROTATE_BYTES)try_to_create_directory)ResourceAndLabelSpec)ResourceIsolationConfig) get_addressserialize_config) is_in_testopen_logtry_to_symlinkvalidate_socket_filepath) GcsClientget_session_key_from_storageceZdZdZ dmdededededed ef d Zd Zed Zd Z dZ dZ dZ e dZe dZe dZe dZe dZe dZe dZe dZe dZe dZe dZe dZe dZe dZe dZe d Ze d!Ze d"Ze d#Z e d$Z!d%Z"d&Z#d'Z$d(Z%d)Z&d*Z'd+Z(d,Z) dnd/e*d0e*d1e+e*fd2Z,d3Z- dod4e*d5ed6ed7ed8e.e+e*e+e*ff d9Z/ dod4e*d5ed6ed7ed8e.e+e0e1e+e0e1ff d:Z2 dpd4e*d/e*d5ed8e*fd;Z3dqd<Z4d=e*d>e*fd?Z5 dqd@e*dAe+e6d8e6fdBZ7drdDe6d8e*fdEZ8dFZ9dGZ:dHe+edIefdJZ;dKZ< dsdLe*dMe*dNe6dOedPef dQZ=dRZ>dSZ?dTZ@dUZAdVZBd8e*fdWZC dtdXedYedZefd[ZD dtd\ZEdudYefd]ZFdudYefd^ZGdudYefd_ZHdudYefd`ZIdudYefdaZJdudYefdbZKdudYefdcZLdvddZMdeZNdfZOdgZPdhZQdiZRdjZSdkZTdlZUd.S)wNodeaAn encapsulation of the Ray processes on a single node. This class is responsible for starting Ray processes and killing them, and it also controls the temp file policy. Attributes: all_processes: A mapping from process type (str) to a list of ProcessInfo objects. All lists have length one except for the Redis server list, which has multiple. FTheadshutdown_at_exit spawn_reaper connect_onlydefault_workerray_init_clusterc  |r%|rtd|||_||_t |o"t jj|_ |j |_ i|_ tj |_||_|r |s Jdt jd}|jc|a|d} t+| dkr| | d| |_t+| dz |_|jr+t+|jdkr|s|std|d it jt jt jt<d d t jt jt jt<d t>j  d|_!tE|_#||_$|jpi|_%|j&|_'tQt!j)d tT|_+tQt!j)dtX|_-|j+dksJ|j-dksJ|j.|_/|r|d|j0|_1d|_2|js.|3|j4|5|j6|_7|j7|r|8} | Ptrj9:;d} d| dt!j<|_7nt j=j>| |_7n|jrJt jj?|@dt>jAt>jB} t j=j>| |_7|rd|_CnX|jD2|jrJt jjEF|_Cnt|jH|jI|_C|J|jK} | 5|r|L} nt jMN} | J|| | |_O|s7t jjEP|Q| |R|_StUd|jSd|_V|jSrtjX|jS}|ddkrh|did}t|tr|d|_Vnt|tr||_V|r|\|r|j]|_^|j_|_`|ja|_b|j^|j`|j$jc|jbht jjEd|j0|jO}|d|_^|d|_`|d|j$_c|d|_bnI|e|j$j]d |_^|e|j$j_d! |_`|j$jfFd"|j$jfvr8|j$jfd"}tUd#|d$||_bnt jd"r8t jd"}tUd#|d%||_bnTt jghi}tUd#|||_b|jd&|j&'|_'|jd(|jk'|_k|jd)|jl'|_m|jd*|jn'|_o||jk|jm|j'|jo+|rot!j)t>jp}|r#|tQ|,|jq |jqdkr|jd-|_q|s|r|j s|r|s|j$s|r|t|s|utjwd.tjx}d/}d} t jjEy|j0|jb}n^#t$r'}t{d0|Yd}~nd}~wwxYwtjx|z |krtd1|j$jcdkr|d|j$_c|rJt jjEy|j0|jb}|d2i|_}|3|j4|3|j0|s|~dSdS)3a*Start a node. Args: ray_params: The RayParams to use to configure the node. head: True if this is the head node, which means it will start additional processes like the Redis servers, monitor processes, and web UI. shutdown_at_exit: If true, spawned processes will be cleaned up if this process exits normally. spawn_reaper: If true, spawns a process that will clean up other spawned processes if this process dies unexpectedly. connect_only: If true, connect to the node without starting new processes. default_worker: Whether it's running from a ray worker or not ray_init_cluster: Whether it's a cluster created by ray.init() z:'shutdown_at_exit' and 'connect_only' cannot both be true.z1ray.init() created cluster only has the head nodeRAY_REDIS_ADDRESSN,rz:System config parameters can only be set on the head node.Tworkerszdefault_worker.py)include_log_monitor resources worker_pathsetup_worker_pathRAY_ROTATION_MAX_BYTESRAY_ROTATION_BACKUP_COUNT)num_redis_shardsz%Y-%m-%d_%H-%M-%S_%fsession__ session_name num_retries)node_ip_addressz+Starting node with object spilling config: type filesystemparamsdirectory_pathobject_store_socket_nameraylet_socket_namenode_manager_portnode_id plasma_store)default_prefixraylet RAY_OVERRIDE_NODE_ID_FOR_TESTINGzSetting node ID to z& based on ray_params.env_vars overridez based on env overridedashboard_agent_listen_port) default_portmetrics_agent_portmetrics_export_portruntime_env_agent_port)rCrDrArE)gcs_server_portrF皙?zFailed to get node info zThe current node timed out during startup. This could happen because some of the raylet failed to startup or the GCS has become overloaded.labels) ValueError_register_shutdown_hooks_default_workerrboolray_privateutilsdetect_fate_sharing_supportkernel_fate_shareresource_isolation_config all_processes threadingLock removal_lockr"osenvirongetexternal_addressessplitlenappendr._system_configupdate_if_absentpathjoindirnameabspath__file__ ray_constantsSETUP_WORKER_FILENAME_resource_and_label_specr _localhost _ray_params_configrA_dashboard_agent_listen_portintgetenvr max_bytesr backup_count redis_address_redis_address gcs_address _gcs_address _gcs_clientvalidate_ip_portaddress_init_gcs_clientr1 _session_namecheck_persisted_session_namedatetimetodaystrftimegetpid_commondecodeinternal_kv_get_with_retryget_gcs_clientKV_NAMESPACE_SESSIONNUM_REDIS_GET_RETRIES _webui_urlwebuiservicesget_webui_url_from_internal_kvr dashboard_hostdashboard_port _init_tempr4_wait_and_get_for_node_addressutilget_node_ip_address_node_ip_addresswrite_node_ip_addressget_session_dir_path_get_object_spilling_config_object_spilling_configloggerdebug_fallback_directoryjsonloads isinstanceliststrvalidate_external_storageplasma_store_socket_name_plasma_store_socket_namer:_raylet_socket_namer<_node_idr;get_node_to_connect_for_driver_prepare_socket_fileenv_varsNodeID from_randomhex_get_cached_portrCrD_metrics_export_portrE_runtime_env_agent_portGCS_PORT_ENVIRONMENT_VARIABLErFstart_reaper_processupdate_pre_selected_portstart_head_processesstart_ray_processestimesleep monotonicget_node RuntimeErrorinfo Exception _node_labels _record_stats)self ray_paramsrrrr r!r"redis_address_envexternal_redis maybe_keydate_strr1r4configr8 node_infor<rF start_timeraylet_start_wait_time_ses e/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/node.py__init__z Node.__init__@s 4  ,  P  ) ) + + +- !%  MS\/KKMM" "   0 &$&%N,, 0  M L LL L L4JNN+>??  ( 05F5R.44S99N>""a''%%nQ&7888,:J )*-n*=*=*AJ '  % J-..223".3L  ## $  9 9::# !gll 9 9::3 $ )-%*,,%!06B ,6,R)RY'?AUVVWW I13N O O  ~"""" A%%%%(6  <  ' ' ' ; ; ;&2y $  ! !$, / / /  ! ! # # #(4   % L!==?? $'06688AA.  H*MH)L)Lry{{)L)LD&&),):)A)A))L)LD&&////"|1LL''))"!6 - C M   &)[%6%=%=l%K%K"  "DOO'////"%,"7"V"V"X"X"/-z/H## $4  " A"&"E"E"G"G"%(">">"@"@***##O#DDD /  L ! 7 7))++_   (,'G'G'I'I$ X$:V X X   $(  ' >Z <==Fzz&!!\11!'Hb!9!9!=!=>N!O!Ond33>/=a/@D,,44>/=D,  -  * * , , , 0 (-7-PD *'1'DD $&.DM.6+3#5==( L1PP$) 2;;U1V.+45I+J(5>?R5S 2 )) 4 .2-F-F 9..G..D *(,'@'@ 3H(A((D $ )56$:J:SSS*34VW <'<<<!(  BCC (*%GH R7RRRSSS ' *00226688 <7<<=== ' -1,A,A )#?-B- - ) #'"7"7 z/L#8# # %)$9$9 ! 0N%:% % !(,'<'< $#:(=( ( $ ###6 $ 9(,(I#'#? $     V i (STTO R++C >( !!I#@@@KK >1 > >????????@>##j03KKK#D "1Q665>?R5S 2  < -66  I !* h ; ;D  dl+++ d./// !     ! !sQ>Q>S>STT  #,AA$BUBUBWBWXX   F,/CCEE -- 7o77     rcfd}tj|fd}tjj|dS)Nc6dddS)NFT check_aliveallow_graceful)kill_all_processes)argsrs ratexit_handlerz5Node._register_shutdown_hooks..atexit_handlers"  # #d # K K K K Krc^ddtjddS)NFTrr&)rsysexit)signumframers rsigterm_handlerz6Node._register_shutdown_hooks..sigterm_handlers-  # #d # K K K HQKKKKKr)atexitregisterrNrOrPset_sigterm_handler)rrrs` rrKzNode._register_shutdown_hookssm L L L L L '''       ..?????rctjd|_|jrN|jt jj |jj |_ n|jj |j rJt j j|dt jt j}t jj||_ n|jj |_ t)|j |jr0t*j|j |j|_n|j |j|j rJt j j|dt jt j}t jj||_n/t*j|j |j|_t*j|j t j}t)|jt7||jt*j|jd|_t)|jt*j|jd|_t)|jt*j|jd}t)|t*j|j|jj|_t)|jt jj }|d }t*j!|r7t*j|jd }t7||dSdS) NcdS)Nrrrrz!Node._init_temp..sr)temp_dirrr2 session_dirsocketslogsoldz /tpu_logstpu_logs)" collectionsr_incremental_dictrrjr`rNrrPget_ray_temp_dirr _temp_dirrLrOrrrfrrrrrXrarbry _session_dirSESSION_LATESTr _sockets_dir _logs_dirruntime_env_dir_name_runtime_env_dirget_user_temp_dirisdir)rrrsession_symlink old_logs_dir user_temp_dir tpu_log_dirtpu_logs_symlinks rrzNode._init_temps&!,!8!C!C 9 ;   - -*;;== .   "-6DNN(0////<-HH''))!6 - C I "%!2!9!9(!C!C!%!1!:/// 9 U " T^T=O P PD  ~%);)C////!l0KK''))!!6 - C L %(K$5$<$<[$I$I!!$&GLLAS$T$T!',,t~}7STT  1222(9:::GLL):IFF 1222d&7@@///w||DNE::  --- "  t/D! !   5666 );;== &111 7== % % :!w||DNJGG  +[ 9 9 9 9 9 : :rc|jstt|jj|jj|jj|jj|jj|jj |j |j |_|jS)zAResolve and return the current ResourceAndLabelSpec for the node.)is_headr4) rhrrjnum_cpusnum_gpusmemoryobject_store_memoryr)rIresolverr4rs rget_resource_and_label_specz Node.get_resource_and_label_specs|, O,@ ) ) ' 4 * ' --gdi9MgNN  ),,rc|jS)zGet the node ID.)rr s rr<z Node.node_ids }rc|jS)z!Get the current Ray session name.)ryr s rr1zNode.session_name"s !!rc|jS)z Get the IP address of this node.)rr s rr4zNode.node_ip_address's $$rc|jS)zGet the address for bootstrapping, e.g. the address to pass to `ray start` or `ray.init()` to start worker nodes, that has been converted to ip:port format. rtr s rrwz Node.address,s   rc2|j Jd|jS)zGet the gcs address.NzGcs address is not setrr s rrszNode.gcs_address4s% ,,.F,,,  rc|jS)zGet the cluster Redis address.)rrr s rrqzNode.redis_address:s ""rc|jjS)zGet the cluster Redis username.)rjrr s rrzNode.redis_username?..rc|jjS)zGet the cluster Redis password.)rjrr s rrzNode.redis_passwordDrrc|jS)z(Get the node's plasma store socket name.)rr s rrzNode.plasma_store_socket_nameIs --rc$|jd|jS)z&Get a unique identifier for this node.:)r4rr s r unique_idzNode.unique_idNs&II)GIIIrc|jS)zGet the cluster's web UI url.)rr s r webui_urlzNode.webui_urlSs rc|jS)z"Get the node's raylet socket name.)rr s rr:zNode.raylet_socket_nameXs ''rc|jjS)zGet the node manager's port.)rjr;r s rr;zNode.node_manager_port]s11rc|jS)z!Get the port that exposes metrics)rr s rrDzNode.metrics_export_portbs ((rc|jS)z3Get the port that exposes runtime env agent as http)rr s rrEzNode.runtime_env_agent_portgs ++rc<dt|j|jS)z6Get the address that exposes runtime env agent as httpzhttp://)r rrr s rruntime_env_agent_addresszNode.runtime_env_agent_addressls#^t'Z[[]]]rc|jS)z%Get the dashboard agent's listen port)rlr s rrAz Node.dashboard_agent_listen_portqs 00rc |j|jdS)z+Get the logging config of the current node.)log_rotation_max_byteslog_rotation_backup_count)rorpr s rlogging_configzNode.logging_configvs'+n)-):   rc |j|j|j|j|j|j|j|j|j|j d S)zGet a dictionary of addresses.) r4rqobject_store_addressr:rrrDrsrwrA) rrqrrrrrrsrwrAr s r address_infozNode.address_info~sM $4!/$($B"&":,#'#<+|+/+K   rc|jS)zGet the node labels.)rr s r node_labelszNode.node_labelss   rc|jSN)rr s rrz Node.is_heads yrcF|j||jSr/)rurxr s rrzNode.get_gcs_clients&   #  ! ! # # #rc |jr#|jtjdj}nd}t tjD]}d}d} |j}t||j j }|j |_ |jr| dd||_ nm#t$r`||YnJtj}t"d|t'jdYwxYw|j t+|drt-t.j|jd5}d|Dd d}dddn #1swxYwYd d |zd z}t9d |jrd nddt;|d|dt.j|jdd| t9d |jrd ndd|t<jj !|j dS)Nr)rw cluster_idsdummyzConnecting to GCS: r&rzgcs_server.errc"g|] }d|vsd|v | S)z C z E r).0rs r z)Node._init_gcs_client..s&UUUAEQJJ%ST**a***ri z Failed to startz connect toz GCS. Last z lines of error files:z.Please check zgcs_server.outz% for details. Last connection error: z GCS. Last connection error: )"rrTrfPROCESS_TYPE_GCS_SERVERprocessrangerrsrrjr2internal_kv_getrurpoll traceback format_excrrrrhasattropenrXrarbr readlinesrr]rN experimental internal_kv_initialize_internal_kv) r gcs_processr0rslast_exclienterrerrors error_msgs rrxzNode._init_gcs_clients 9 ,]-RS KK}:;;  AKG ". "'#/:#)"39;**8T:::#)    *{/?/?/A/A/MEE#.00 <7<<=== 1     #t[)) "',,t~7GHHIISVUUUUF !2776??2T9 "FDI!G<FF [[FF FF%'GLLAQ$R$RFF=D FF#3DI!G<33)033 $<>(++ IN&vv~'EFh&&&el,PQQQrc||jj}|-tjt jdk}|S)N1)rjredirect_outputrXrYrZrf,LOGGING_REDIRECT_STDERR_ENVIRONMENT_VARIABLE)rrhs rshould_redirect_logszNode.should_redirect_logssE*:  " !N  rnameunique create_out create_errreturnc|sdSd}d}|r||d|}|r||d|}||fS)aGet filename to dump logs for stdout and stderr, with no files opened. If output redirection has been disabled, no files will be opened and `(None, None)` will be returned. Args: name: descriptive string for this log file. unique: if true, a counter will be attached to `name` to ensure the returned filename is not already used. create_out: if True, create a .out file. create_err: if True, create a .err file. Returns: A tuple of two file handles for redirecting optional (stdout, stderr), or `(None, None)` if output redirection is disabled. )NNNoutrlrI)rj_get_log_file_name)rrkrlrmrn log_stdout log_stderrs rget_log_file_nameszNode.get_log_file_namessx,((** :   M00uV0LLJ  M00uV0LLJ:%%rc|||||\}}|dnt|}|dnt|}||fS)aOpen log files with partially randomized filenames, returning the file handles. If output redirection has been disabled, no files will be opened and `(None, None)` will be returned. Args: name: descriptive string for this log file. unique: if true, a counter will be attached to `name` to ensure the returned filename is not already used. create_out: if True, create a .out file. create_err: if True, create a .err file. Returns: A tuple of two file handles for redirecting optional (stdout, stderr), or `(None, None)` if output redirection is disabled. rlrmrnN)rvr) rrkrlrmrnlog_stdout_fnamelog_stderr_fnamertrus rget_log_file_handleszNode.get_log_file_handles=sj,.2-D-D J:.E. . **.5TT8DT;U;U -5TT8DT;U;U :%%rc|d}|r!|d|||j}n*tj|j|d|}|S)aGenerate partially randomized filenames for log files. Args: name: descriptive string for this log file. suffix: suffix of the file. Usually it is .out of .err. unique: if true, a counter will be attached to `name` to ensure the returned filename is not already used. Returns: A tuple of two file names for redirecting (stdout, stderr). r[)rWrXrY)striprerrXrarb)rrkrWrlrds rrszNode._get_log_file_nameZsu$c""  H**#6||D+HHw||DNt4F4Ff4F4FGGHrc@|t}tjt|jr tjn tjtj}|d|d}ttj D]}tj |d}||vrtjt|jr tjn tjtj} |d|fn$#t$r|YwxYw|||cSt d||S)N)r7rr&ir7z-Unable to succeed in selecting a random port.)setsocketr rAF_INET6AF_INET SOCK_STREAMbind getsocknamer;rfNUM_PORT_RETRIESrandomrandintOSErrorclosererror)rallocated_portssportr0new_portnew_ss r_get_unused_portzNode._get_unused_portvsj  "!eeO M&t'<== QFOO6>     w}}q! }566  A~dE22H?**M#*4+@#A#AUv~"E  B>****      GGIII KKMMMOOO DEEE   sD  EE socket_pathr>cl|}tjdkr-|*dt|j|}nt||||j}n,ttj |t| ddd|S)aePrepare the socket file for raylet and plasma. This method helps to prepare a socket file. 1. Make the directory if the directory does not exist. 2. If the socket file exists, do nothing (this just means we aren't the first worker on the node). Args: socket_path: the socket file to prepare. win32Nztcp://)rXrYz://r&) rplatformr rirrerrrXrarcrr\)rrr>results rrzNode._prepare_socket_files <7 " ""V]4?DRSS MBBBBB4?t3D3D i') * * 0 07>>),, %)S))%QIb!$$$%%%%%%%%%%%%%%%i%% 3$$TYq\\222 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3-//t~!>>>=8CDD#&mDN&C&J&J&L&L"M"M+ 0O0O#'L#Mt'<'<_'M'M;? dn-i8)S))0QImQ///0000000000000003 0 0 0 0 0 0 0 0 0 0 0 0 0 0 08 s)0G2B;/ G2;B? ?G2B? G2(D > G2 D G2D B'G29G G2G G2"G #G22G69G6< timeout_sc t|D]}tjj|}||cSt jd|dzdkr=t dtj d|dtdtj d|d|d ) aWait until the RAY_NODE_IP_FILENAME file is avialable. RAY_NODE_IP_FILENAME is created when a ray instance is started. Args: timeout_s: If the ip address is not found within this timeout, it will raise ValueError. Returns: The node_ip_address of the current session if it finds it within timeout_s. Nr& rzCan't find a `z ` file from z@. Have you started Ray instance using `ray start` or `ray.init`?z. for z[ seconds. A ray instance hasn't started. Did you do `ray start` or `ray.init` on this host?) r;rNrOrget_cached_node_ip_addressrrrrrrfRAY_NODE_IP_FILENAMErJ)rrir4s rrz#Node._wait_and_get_for_node_addresssy!!  A!l3NN))++O*&&&& JqMMM2v{{ 1]%G11!%!:!:!>  % % ' ' N  -n*,,?   5T=OOOOO F =ABBBrinclude_dashboardraise_on_failurecL|dddd\}}tjj|||jj|j|j |j |j |j |j |jj|j|j|j||\|_}t&j|jvsJ|b|g|jt&j<|d|jdt&jdSdS)aStart the dashboard. Args: include_dashboard: If true, this will load all dashboard-related modules when starting the API server. Otherwise, it will only start the modules that are not relevant to the dashboard. raise_on_failure: If true, this will raise an exception if we fail to start the API server. Otherwise it will print a warning if we fail to start the API server. dashboardTrx)rrrorprrNs webui:url)rvrNrOrstart_api_serverrjrrsr2rrrrrrrRrorprrfPROCESS_TYPE_DASHBOARDrTrinternal_kv_putencodeKV_NAMESPACE_DASHBOARD)rrrrrrs rrzNode.start_api_server9s>.2-D-D $.E. . **), (=(N(N     +   O   ! !  ! N N  !0-n*,,)O) ) %"34;MMMMM  #HD }C D    ! ! 1 1&&((4       $ #rc |jj}|dksJ|j Jd|j Jd|dddd\}}t jj|j |j |||j |jj |jj |j|j||jj|j }t$j|jvsJ|g|jt$j<t+|j||_dS) zStart the gcs server.rNzGCS server is already running.z GCS client is already connected. gcs_serverTrx) log_dirrrr1rrrrrFrCr4)rjrFrtrurvrNrOrstart_gcs_serverrqrr1rrrkrRrCrrfr9rTr )rrFrrrs rrzNode.start_gcs_serverfs,*:"""" ((*J((('')K'''-1-D-D $4.E. . **|,==  N,,*+:+:<-+#/B 1>   4D9P9P  - :Q: : 6 6  # #  6 $   + +  # #  8 $   - - &66  = = ? ?   |,9     M  !   .  $  * O   ! !   (   . N    ! N  , , . .     '1 1 1 ()1 *!,<<+1 ,!,<<-1 ."->>/1 0!% 0 D D11 2 +::31 4 +::51 6 $/BB71 8$(#3#J#J91 :!% 9 9;1 <)-(8(T(T=1 >&?1 @&A1 B$:#9C1 D$:#9E1 F-L,KG1 H-L,KI1 J/P.OK1 L/P.OM1 N'22O1 P--Q1 R$S1 TnnU1 V**W1 X#'"2"H"HY1 Z(11[1 \&00]1 ^//_1 `'+&D&Da1 d08JJJJJAM=<===rc^ddlm}|dddd\}}tjj|j|j|||j j |j |j |j |j|d }tj|jvsJ|g|jtj<dS) zStart the monitor. Autoscaling output goes to these monitor.err/out files, and any modification to these files may break existing cluster launching commands. r)is_autoscaler_v2monitorTrx)fetch_from_server)rrautoscaling_configrrorp monitor_ip autoscaler_v2N)ray.autoscaler.v2.utilsrrvrNrOr start_monitorrsrrjrrRrorprrfPROCESS_TYPE_MONITORrT)rrrrrs rrzNode.start_monitors =<<<<<-1-D-D dt.E. . **|,::   N,,#/B-n*,**TBBB;   19KKKKKBN==>>>rc L|dd\}}tjj|j|j|jj|||jj |jj |j |j  }tj|jvsJ|g|jtj<dS)z$Start the ray client server process.ray_client_serverTrr) stdout_file stderr_filerrrr#N)r{rNrOrstart_ray_client_serverrwrrjray_client_server_portrrrRr#rfPROCESS_TYPE_RAY_CLIENT_SERVERrT)rrrrs rrzNode.start_ray_client_servers#'#<#< $=$ $  [|,DD L  !   3##+:+:-&*&DE   ;4CUUUUU L =GHHHrcddlmcmcm}|||j|d|j dtj }|sf| dtj }||j dksJd|jd|d |d |j d tj |d |j d tj |jjrR|d |jj d tjdSdS)zWrite the cluster metadata to GCS. Cluster metadata is always recorded, but they are not reported unless usage report is enabled. Check `usage_stats_head.py` for more details. rN)r"rFzutf-8z Session name z does not match persisted value z1. Perhaps there was an error connecting to Redis.s session_dirTstemp_dirstracing_startup_hook)rrrrput_cluster_metadatarr"rryrrfrr<rrrjtracing_startup_hookKV_NAMESPACE_TRACING)rraddedcurr_vals r_write_cluster_info_to_kvzNode._write_cluster_info_to_kvs <;;;;;;;;;;;**    ! !D4I +   ##%%55    % % ' '   .     **,,<<!CHt188AAAAA. 2..#+...BAA --    $ $ & &   .    --  N ! ! # #   .       0     ! ! 1 1' 5<<>>2        rctd|jd|jJ|jJ||J||jj s| |jj r| |jj d}n |jj }||jj |dS)z!Start head processes on the node.1Process STDOUT and STDERR is being redirected to r[NF)rr)rrrrtrurrrrj no_monitorrrrrr)rraise_on_api_server_failures rrzNode.start_head_processesMs UDN U U U    (((''' ""$$000 &&(((* !      2 +  ( ( * * *   - 5*/ ' '*.*:*L ' ".@8      rctd|jd|jstjj|j|j dd}tj j }|||}|j|ksJd|jd|||_|}|jtjjrBddlm}||jtjjtj j|j|j|jj|j|jj \}}}|j!"r|j!#||jj$r|%|&|||d S) z'Start all of the processes on the node.rr[F)allow_cluster_id_nilfetch_cluster_id_if_nilzThe system config from GCS is not a superset of the local system config. There might be a configuration inconsistency issue between the head node and non-head nodes. Local system config: z, GCS system config: r)r)rrrN)'rrrrrN_rayletGcsClientOptionscreatersr2rrOstate GlobalState_initialize_global_stateget_system_configrkitemsr rIrZRAY_NODE_ACCELERATOR_TYPE_KEYray._common.usagerrecord_hardware_usagerdetermine_plasma_store_configr rrjrrrrS is_enabledadd_object_store_memoryr(rr) r gcs_options global_state new_configresource_and_label_specrrrr s rrzNode.start_ray_processesjsS UDN U U U   y &+6== ##%%%*(- >K <-99;;L  1 1+ > > >%7799J<%%'':+;+;+=+====4*.44(2 44>==&DL#'"B"B"D"D " ) - - K 5   4 3 3 3 3 3  + +'.22K=    L ! ? ? # 7 N!->#7'2 @       ) 4 4 6 6 X  * B BCV W W W   / %  " " $ $ $ *,>@STTTTTrcPd|jD}tj|jvr\|jtjdjj}t j|}|d|Dz }d |S)aReturns a list of system processes that will be isolated by raylet. NOTE: If a new system process is started before the raylet starts up, it needs to be added to self.all_processes so it can be moved into the raylet's managed cgroup hierarchy. cLg|]!}t|djj"S)r)rr:pidr4ps rr5zENode._get_system_processes_for_resource_isolation..s6   &'C!  ! !   rrc6g|]}t|jSr)rrrs rr5zENode._get_system_processes_for_resource_isolation..s #U#U#U1CJJ#U#U#Urr%) rTrrfrr:rpsutilProcesschildrenrb)rsystem_process_pids dashboard_piddashboard_processs rrz1Node._get_system_processes_for_resource_isolations  +/+=+D+D+F+F     /43E E E .}/STc !'} = =  #U#U8I8R8R8T8T#U#U#U U xx+,,,rrrwaitcx|j5|||||ddddS#1swxYwYdS)aNKill a process of a given type. If the process type is PROCESS_TYPE_REDIS_SERVER, then we will kill all of the Redis servers. If the process was started in valgrind, then we will raise an exception if the process has a non-zero exit code. Args: process_type: The type of the process to kill. allow_graceful: Send a SIGTERM first and give the process time to exit gracefully. If that doesn't work, then use SIGKILL. We usually want to do this outside of tests. check_alive: If true, then we expect the process to be alive and will raise an exception if the process is already dead. wait: If true, then this method will not return until the process in question has exited. Raises: This process raises an exception in the following cases: 1. The process had already died and check_alive is true. 2. The process had been started in valgrind and had a non-zero exit code. )rrrN)rW_kill_process_impl)r process_typerrrs r_kill_process_typezNode._kill_process_typesB     # #-' $                     s /33cl||jvrdS|j|}|tjkrt|dksJ|D]}|j}||rt d|d4|jr|| |j dkrd|d|j d}|j Gt|j d 5} |d | zz }dddn #1swxYwY|jGt|jd 5} |d | zz }dddn #1swxYwYt |)|jr8t!j|jt&jt+jd |rB|d} | | n#t.j$rYnwxYw|*||r| |j|=dS) zSee `_kill_process_type`.Nr&z&Attempting to kill a process of type 'z$', but this process is already dead.rz1Valgrind detected some errors in process of type z . Error code r[rz PROCESS STDOUT: z PROCESS STDERR: rG)rTrfPROCESS_TYPE_REDIS_SERVERr]r:r=rr terminater returncoderrAreadruse_valgrind_profilerrXkillrsignalSIGINTrr subprocessTimeoutExpired) rr!rrr process_infosrr:messagertimeout_secondss rr zNode._kill_process_impls# t1 1 1 F*<8 =B B B}%%****)2 #2 #L"*G||~~)&O(OOO ( !!### %**Q ,QQ;B;MQQQ$/;!,":C@@HA#'>*=>>JPX  # "6#57G H H H+KKMM  7 " !.""5 6 6 6 6 6  7 7r)FTTFFF)r7r7N)FTT)Fr/)r)FF)FTF)T)TFF)V__name__ __module__ __qualname____doc__rMrrz staticmethodrvrrKrr propertyr<r1r4rwrsrqrrrrrr:r;rDrEr#rAr(r+r-rrrxrMrPrrTrVrrrerjrrvrrr{rsrrrmrrrrrrrrrrrrrr"r r4r6r9r;r=r?rArrGrIrLrNrUrrrrrrrr4s  !%!"$!&C!C!C! C!  C!  C!C!C!C!C!C!J    0  \    ,@@@"=:=:=:~ - - -X""X"%%X%!!X!!!X! ##X#//X///X/..X.JJXJX((X(22X2))X),,X,^^X^11X1  X    X  !!X!   9O9O9Ov%%%!!!!!! SW"R"R"R(+"RBJ3-"R"R"R"RH     & & & & &  & x}hsm+ , & & & &J &&&& &  & x6 #Xbj%99 : &&&&B   8""""HS:=A777,4SM7 7777r# # # S# # # # J"   (+$,TN+FJ++++Z R R RN#" `O`O`O `O! `O  `O  `O`O`O`ODPPP6   *111f   :;U;U;Uz-c----2 %  ''' '  ''''TJO=-=-=-=-~   d        t        D        $                4        t     4444l   "*** . . . / / /(2(2(2T0&0&0&d77777rr);rrr{rarloggingrXrr*rr,rr]rUrr>rtypingrrrrrfilelockr rNray._private.ray_constantsrOrfray._private.servicesray._common.network_utilsr r r r ray._common.ray_constantsrrray._common.utilsr$ray._private.resource_and_label_specr&ray._private.resource_isolation_configrrrray._private.utilsrrrr ray._rayletrrr getLoggerrnrrrrrrst     ######44444444444444 222222222 XWWWWWWW555555EEEEEEJJJJJJ???????? @???????  8 $ $f7f7f7f7f7f7f7f7f7f7r