
    &`in             `          d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlmZ d dlmZmZmZmZ d dlmZ d dlZd dlmc mZ d dlmZmZmZm Z m!Z! d dlm"Z" d dl#m$Z$ d dl%m&Z&m'Z' d d	l(m)Z) d dl*Z*dZ+ej,        d
k    rdZ-ndZ-ej,        d
k    rdndZ.dZ/ej0        1                    ej0        2                    ej0        2                    e3                    dd          Z4ej0        5                    ej0        2                    ej0        2                    e3                              Z6dZ7ej0        1                    dd          Z8ej0        1                    dd          Z9ej0        1                    e6ddddde.z             Z:ej0        1                    e6ddddde.z             Z;ej0        1                    e6dd          Z<ej0        =                    e<          re<ndZ<ej0        1                    e6dde.z             Z>ej0        1                    e6dd          Z?dZ@dZAd ZBd!ZCd"ZDd#ZE ejF        eG          ZH ejI        d$g d%          ZJd&eeK         fd'ZLd( ZMd) ZN	 dd+eKd,eKd-eOfd.ZPd/ ZQd0 ZRd1eKd2eKd3eeK         d4eKfd5ZS G d6 d7ejT                  ZUd8eKfd9ZVd: ZWd;eeK         fd<ZXd=eKd;eeK         fd>ZYe-fd?eKd@eKdAeZfdBZ[dC Z\dD Z]dE Z^dF Z_	 dd=eKd;eeK         d&eeK         fdGZ`	 dd=eKd;eeK         d&eKfdHZadIeKfdJZbdKeKfdLZcddMZdd,eKd&eKfdNZed,eKdOeeK         d&dfdPZfdQ ZgddRZh	 	 	 	 	 	 	 	 	 	 ddSeeK         d4eKdTeOdUeei         dVeeK         dWeOdXeOdYeOdZeOd[eOd\eee                  d]eee                  d^eOfd_Zjdd`Zk	 	 	 	 	 dd,eKdaeKd?eKdTeeO         dbeZdceZddeeK         deeeK         fdfZl	 	 	 	 	 	 ddgeeO         dheOdKeKd?eKdieKdOeKd;eKdjeKd,eKdkeeZ         dTeeO         dbeZdceZddeeK         deeeK         fdlZmdm Zn	 	 	 	 	 	 	 ddneKdoeKddeeK         deeeK         dpeKdqeeK         dreeK         dseei         dTeeO         dteeZ         dueeZ         dOeeK         fdvZo	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddneKd?eKdweKdOeKdxeZdyeKdzeKd{eKd|eKd}eKd;eKd,eKd~eKdoeKdeKdeKdeZdpeKdeOde$deeZ         deeZ         deeeZ                  deeZ         dqeeK         dreeK         dueeZ         deeZ         deeZ         deeZ         dWeOdeOdeeK         deeK         deeK         deeK         deeK         deeK         deOdTeeO         deeZ         dbeZdceZdeOdUeei         deeK         deeK         f^dZpd ZqdIeKdzeKdyeKdqeKdreKd,eKdOeKd}eKfdZrdIeKdzeKdyeKdqeKdreKd,eKdoeKdOeKd}eKfdZs	 	 	 ddeZd;eKdeeK         deeK         deOf
dZt	 	 	 	 	 	 	 	 dd?eKdaeKddeeK         deeeK         deeK         dTeeO         dbeZdceZdeeK         deOfdZu	 	 	 	 	 	 	 	 ddeKdeKdeZd\eeZ         d]eeZ         dqeeK         dreeK         dTeeO         deeK         deKdeeK         fdZvdeeeK                  d&eOfdZwdS )    N)Path)IOAnyStrListOptional)FileLock)build_addressget_localhost_ipis_ipv6 node_ip_address_from_perspectiveparse_address)RAY_NODE_IP_FILENAME)ResourceIsolationConfig)	GcsClientGcsClientOptions)Languagewin32   <   z.exe Fz.._private
autoscalerv2coresrcrayrayletgcs
gcs_serverzlibjemalloc.socppdefault_workerlibzxNot all Ray Dashboard dependencies were found. To use the dashboard please install Ray using `pip install ray[default]`.RAY_JEMALLOC_LIB_PATHRAY_JEMALLOC_CONFRAY_JEMALLOC_PROFILE#RAY_INTERNAL_MEM_PROFILE_COMPONENTS RAY_INTERNAL_MEM_PROFILE_OPTIONSProcessInfoprocessstdout_filestderr_fileuse_valgrinduse_gdbuse_valgrind_profileruse_perftools_profileruse_tmuxreturnc                      g } t                      r|                     d           t                      r|                     d           | S )zDetect whether flags related to site packages are enabled for the current
    interpreter. To run Ray in hermetic build environments, it helps to pass these flags
    down to Python workers.
    z-Sz-s)_no_siteappend_no_user_site)flagss    i/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/services.py_site_flagsr9   u   sJ    
 Ezz T TL    c                  $    t           j        j        S N)sysr7   no_site r:   r8   r4   r4      s    9r:   c                  $    t           j        j        S r<   )r=   r7   no_user_siter?   r:   r8   r6   r6      s    9!!r:   T	componentsession_dir
unbufferedc                   	 t           j        g}|r|                    d           t          j        t
          d          }|s|S t          |                    d                    }	 ddl}n!# t          $ r t          d| d          w xY w| |v rt          |          }|j        }|dz  }|                    d	
           || d|  dz  }t          j        t          d          		r	                    d          ng 	t          	fddD                       sdt          |          g	dd<   |                    dddg	           |S )a  Build the Python executable command.

    It runs a memory profiler if env var is configured.

    Args:
        component: Name of the component. It must be one of
            ray_constants.PROCESS_TYPE*.
        session_dir: The directory name of the Ray session.
        unbuffered: If true, Python executable is started with unbuffered option.
            e.g., `-u`.
            It means the logs are flushed immediately (good when there's a failure),
            but writing to a log file can be slower.
    -ur   ,r   Nz4Memray is required to memory profiler on components z. Run `pip install memray`.profileT)exist_ok_memory_z.binc              3       K   | ]}|v V  	d S r<   r?   ).0optoptionss     r8   	<genexpr>zF_build_python_executable_command_memory_profileable.<locals>.<genexpr>   s'      JJc3'>JJJJJJr:   )z--live-oz--outputrP   -mmemrayrun)r=   
executabler5   osgetenv RAY_MEMRAY_PROFILE_COMPONENT_ENVsetsplitrR   ImportErrorr   namemkdirRAY_MEMRAY_PROFILE_OPTIONS_ENVanystrextend)
rB   rC   rD   commandcomponents_to_memory_profilerR   session_nameprofile_diroutput_file_pathrN   s
            @r8   3_build_python_executable_command_memory_profileablerf      s   " 	G  t#%9-Mr#R#R ' #&'C'I'I#'N'N#O#O 
 
 
 
I+I I I
 
 	


 000;''"'!I-4(((&L)Q)Q))Q)Q)QQ):DAA(/7'--$$$RJJJJ-IJJJJJ 	9 #&6"7"78GAaCLh88999Ns   'A, ,B
c                 2    t          j        | d dd          S )NTFallow_cluster_id_nilfetch_cluster_id_if_nil)r   create)gcs_server_addresss    r8   _get_gcs_client_optionsrm      s(    "! %	   r:   c                     t          j        t          j        |                               d                                        d          S )Nutf-8)base64	b64encodejsondumpsencodedecode)configs    r8   serialize_configrw      s9    DJv..55g>>??FFwOOOr:   jemalloc_pathjemalloc_confjemalloc_compsprocess_typec                     t          |t                    sJ |J |                                }| si S | t          j                            dd          d}||v r|r|                    d|i           |S )a  Read the jemalloc memory profiling related
    env var and return the dictionary that translates
    them to proper jemalloc related env vars.

    For example, if users specify `RAY_JEMALLOC_LIB_PATH`,
    it is translated into `LD_PRELOAD` which is needed to
    run Jemalloc as a shared library.

    Params:
        jemalloc_path: The path to the jemalloc shared library.
        jemalloc_conf: `,` separated string of jemalloc config.
        jemalloc_comps: The list of Ray components
            that we will profile.
        process_type: The process type that needs jemalloc
            env var for memory profiling. If it doesn't match one of
            jemalloc_comps, the function will return an empty dict.

    Returns:
        dictionary of {env_var: value}
            that are needed to jemalloc profiling. The caller can
            call `dict.update(return_value_of_this_func)` to
            update the dict of env vars. If the process_type doesn't
            match jemalloc_comps, it will return an empty dict.
    NRAY_LD_PRELOAD_ON_WORKERS0)
LD_PRELOADr}   MALLOC_CONF)
isinstancelistlowerrU   environgetupdate)rx   ry   rz   r{   env_varss        r8   propagate_jemalloc_env_varr      s    > nd+++++###%%''L 	 $%'Z^^4OQT%U%U H ~%%-%6777Or:   c                   F     e Zd Zej        dk    r fdZ fdZ xZS  xZS )ConsolePopenr   c                    t          | j        t          j                  r| j                                         | j        r!|                     t          j                   d S t          t          |                                            d S r<   )r   stdinioIOBaseclose_use_signalssend_signalsignalCTRL_BREAK_EVENTsuperr   	terminate)self	__class__s    r8   r   zConsolePopen.terminate   sw    $*bi00 #
  """  6  !899999lD))3355555r:   c                    t           j        }d}t          j        j                                        r|}d}|r|                    |          pd|z  ||<   ||         |z  | _         t          t          |           j
        |i | d S )Nr   creationflags)
subprocessCREATE_NEW_PROCESS_GROUPr   r   utilsdetect_fate_sharing_supportr   r   r   r   __init__)r   argskwargs
new_pgroupflags_to_add	flags_keyr   s         r8   r   zConsolePopen.__init__  s     $<JL|!==?? *  *'I P%+ZZ	%:%:%?a<$Oy! &y 1J >D.E,%%.??????r:   )__name__
__module____qualname__r=   platformr   r   __classcell__)r   s   @r8   r   r      su        
|w	6 	6 	6 	6 	6	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ r:   r   flagc                    t                      }t          j        dg          D ]}	 |j        d         }t	          |          rn|D ]k}|                    d          D ]S}|                    |           r<|                    d          d         }|dk    r|dk    r|                    |           Tl# t          j        $ r Y t          j	        $ r Y w xY w|S )z
    Attempts to find all valid Ray addresses on this node, specified by the
    flag.

    Params:
        flag: `--redis-address` or `--gcs-address`
    Returns:
        Set of detected addresses.
    cmdline =   r   None)
rX   psutilprocess_iterinfo_is_raylet_processrY   
startswithaddAccessDeniedNoSuchProcess)r   	addressesprocr   arglistarg	proc_addrs          r8   _find_address_from_flagr     s   t I#YK00  	 i	*G "'** 9& 
9 
9G  '}}S11 9 9>>$// 9(+		#q(9I  )B93F3F )i 8 8 89 " 	 	 	D# 	 	 	D	s   B
B22CCCc                       t          d          S )z3Finds any local GCS processes based on grepping ps.z--gcs-address)r   r?   r:   r8   find_gcs_addressesr   p  s    "?333r:   temp_dirc                 J    t           j        j                            |           S )zFinds the latest Ray cluster address to connect to, if any. This is the
    GCS address connected to by the last successful `ray start`.)r   r   r   read_ray_addressr   s    r8   find_bootstrap_addressr   u  s     <..x888r:   addrc                    t           j                            t          j                  }||dk    r|} | | dk    r| S t                      }t          |          }t          |          dk    r$|"t          	                    d| d| d           n:t          |          dk    r'| dk    r!t          |                                          }|| dS t          d	          |S )
a.  Attempts to find the address of Ray cluster to use, in this order:

    1. Use RAY_ADDRESS if defined and nonempty.
    2. If no address is provided or the provided address is "auto", use the
    address in /tmp/ray/ray_current_cluster if available. This will error if
    the specified address is None and there is no address found. For "auto",
    we will fallback to connecting to any detected Ray cluster (legacy).
    3. Otherwise, use the provided address.

    Returns:
        A string to pass into `ray.init(address=...)`, e.g. ip:port, `auto`.
    Nr   autor   %Found multiple active Ray instances: z". Connecting to latest cluster at z^. You can override this by setting the `--address` flag or `RAY_ADDRESS` environment variable.r   zCould not find any running Ray instance. Please specify the one to connect to by setting `--address` flag or `RAY_ADDRESS` environment variable.)rU   r   r   ray_constants RAY_ADDRESS_ENVIRONMENT_VARIABLEr   r   lenloggerwarningr   popConnectionError)r   r   env_addr	gcs_addrsbootstrap_addrs        r8    get_ray_address_from_environmentr   {  s    z~~mLMMHBDFNN"$$I+H55N
9~~n85I 5 5/=5 5 5	
 	
 	
 	
 
Y!		 i,,..<4!9   r:   gcs_addressnode_plasma_store_socket_nametimeoutc                    t          j        | ddd          }t          j        j                                        }|                    |           t          j                    }t          j                    |z
  |k     rT|                                }d |D             }||v rdS t          j	        d           t          j                    |z
  |k     Tt          d| d| d	          )
a5  Wait until this node has appeared in the client table.
    NOTE: Makes an RPC to the GCS up to every 0.1 seconds to
    get all node info. Use only for testing.

    Args:
        gcs_address: The gcs address
        node_plasma_store_socket_name: The
            plasma_store_socket_name for the given node which we wait for.
        timeout: The amount of time in seconds to wait before raising an
            exception.

    Raises:
        TimeoutError: An exception is raised if the timeout expires before
            the node appears in the client table.
    NTFrh   c                     g | ]
}|d          S )ObjectStoreSocketNamer?   )rL   clients     r8   
<listcomp>z!wait_for_node.<locals>.<listcomp>  s+     %
 %
 %
06F*+%
 %
 %
r:   皙?zTimed out after zE seconds while waiting for node to startup. Did not find socket name z* in the list of object store socket names.)r   rk   r   r   stateGlobalState_initialize_global_statetime
node_tablesleepTimeoutError)r   r   r   gcs_optionsglobal_state
start_timeclientsobject_store_socket_namess           r8   wait_for_noder     s   ( #)Te  K <%1133L))+666J
)++

"W
,
,))++%
 %
:A%
 %
 %
! ),EEEFJsOOO )++

"W
,
, 	(7 	( 	($A	( 	( 	(  r:   c                     t           j        j                                        }t	          |           }|                    |           |                    |          S r<   )r   r   r   r   rm   r   get_node_to_connect_for_driver)r   node_ip_addressr   r   s       r8   r   r     sM    <%1133L)+66K))+66666GGGr:   c                     t           j        j                                        }t	          |           }|                    |           |                    |          S )zB
    Get the node information from the global state accessor.
    )r   r   r   r   rm   r   get_node)r   node_idr   r   s       r8   r   r     sO     <%1133L)+66K))+666  )))r:   c                      t           j        j                                        sJ t           j        j                            dt
          j                  } | $t           j        j        	                    |           nd S )Nz	webui:url	namespace)
r   experimentalinternal_kv_internal_kv_initialized_internal_kv_getr   KV_NAMESPACE_DASHBOARD_commonr   ru   )	webui_urls    r8   get_webui_url_from_internal_kvr     sm    '@@BBBBB ,==}C >  I 3<2G3;##I...TQr:   c                      t           j        j        j        t	          d          t           j        j        j                                        S )a  See if the remaining processes are alive or not.

    Note that this ignores processes that have been explicitly killed,
    e.g., via a command like node.kill_raylet().

    Returns:
        True if the remaining processes started by ray.init() are alive and
            False otherwise.

    Raises:
        Exception: An exception is raised if the processes were not started by
            ray.init().
    NzVThis process is not in a position to determine whether all processes are alive or not.)r   r   worker_global_nodeRuntimeErrorremaining_processes_aliver?   r:   r8   r   r     sE     |'/6
 
 	
 <+EEGGGr:   c                 2   | | dk    rt          | |          } | | dk    rdS t          |           }|t          d|            |\  }}	 t          |          }n-# t          $ r  t
                              d|  d            w xY wt          ||          S )a  Canonicalizes Ray cluster bootstrap address to host:port.
    Reads address from the environment if needed.

    This function should be used to process user supplied Ray cluster address,
    via ray.init() or `--address` flags, before using the address to connect.

    Returns:
        Ray cluster address string in <host:port> format or None if the caller
        should start a local Ray instance.
    Nr   localzInvalid address format: zFailed to convert z to host:port)r   r   
ValueErrorresolve_ip_for_localhost	Exceptionr   	exceptionr	   )r   r   parsedhostportbootstrap_hosts         r8   canonicalize_bootstrap_addressr
    s     |tv~~/h??|twt4  F~:D::;;;JD$1$77   AdAAABBB ...s   A *Bc                     t          | |          }||S t                      }t          |          dk    rt          d          t          |          dk    rt          d| d          |                                S )a  Canonicalizes Ray cluster bootstrap address to host:port.

    This function should be used when the caller expects there to be an active
    and local Ray instance. If no address is provided or address="auto", this
    will autodetect the latest Ray instance created with `ray start`.

    For convenience, if no address can be autodetected, this function will also
    look for any running local GCS processes, based on pgrep output. This is to
    allow easier use of Ray CLIs when debugging a local Ray instance (whose GCS
    addresses are not recorded).

    Returns:
        Ray cluster address string in <host:port> format. Throws a
        ConnectionError if zero or multiple active Ray instances are
        autodetected.
    r   Nr   zCould not find any running Ray instance. Please specify the one to connect to by setting the `--address` flag or `RAY_ADDRESS` environment variable.r   r   zm. Please specify the one to connect to by setting the `--address` flag or `RAY_ADDRESS` environment variable.)r
  r   r   r   r   )r   r   r   running_gcs_addressess       r8   %canonicalize_bootstrap_address_or_dier  %  s    & 4D8LLLN!.00
 !!Q&&:
 
 	

  !!A%%:4I : : :
 
 	

 !$$&&&r:   bootstrap_addressc                     t          |           }|t          d|  d          |\  }}	 t          |          }n!# t          $ r t          d| d          w xY w|dk     s|dk    rt          d| d          ||fS )	NzMalformed address z. Expected '<host>:<port>'.zMalformed address port z. Must be an integer.i   i  zInvalid address port z-. Must be between 1024 and 65535 (inclusive).)r   r  int)r  ip_portipr  s       r8   extract_ip_portr  L  s    -..GS!2SSS
 
 	
 HBP4yy P P PN4NNNOOOPd{{dUll%D % % %
 
 	
 t8Os	   ; Ar  c                 n    | st          d|            | dk    s| dk    s| dk    rt                      S | S )a  Convert to a remotely reachable IP if the host is "localhost",
            "127.0.0.1", or "::1". Otherwise do nothing.

    Args:
        host: The hostname or IP address.

    Returns:
        The same host but with the local host replaced by remotely
            reachable IP.
    zMalformed host: z	127.0.0.1z::1	localhost)r  get_node_ip_address)r  s    r8   r  r  _  sQ      42D22333{demmt{/B/B"$$$r:   c                     t           j        j        j        t           j        j        j        j        S t
          j        st                      S t          |           S r<   )	r   r   r   r   r   r   ENABLE_RAY_CLUSTERr
   r   )addresss    r8   r  r  x  sH    
|'3|"/??+ "  !!!+G444r:   c                 V   t          t          j                            | t                              }i }t          t          |                                          dz             5  |                                s	 ddd           dS |	                                5 }|
                    t          j        |                     ddd           n# 1 swxY w Y   d|v r|d         cddd           S t          j                                        cddd           S # 1 swxY w Y   dS )a  Get a node address cached on this session.

    If a ray instance is started by `ray start --node-ip-address`,
    the node ip address is cached to a file RAY_NODE_IP_FILENAME.
    Otherwise, the file exists, but it is emptyl.

    This API is process-safe, meaning the file access is protected by
    a file lock.

    Args:
        session_dir: Path to the Ray session directory.

    Returns:
        node_ip_address cached on the current node. None if the node
        the file doesn't exist, meaning ray instance hasn't been
        started on a current node. If node_ip_address is not written
        to a file, it means --node-ip-address is not given, and in this
        case, we find the IP address ourselves.
    .lockNr   )r   rU   pathjoinr   r   r_   absoluteexistsopenr   rr   loadr   utilr  )rC   	file_pathcached_node_ip_addressfs       r8   get_cached_node_ip_addressr&    s   ( RW\\+/CDDEEI	#i((**++g5	6	6 
2 
2!! 	
2 
2 
2 
2 
2 
2 
2 
2 ^^ 	8"))$)A,,777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8  666)*;<
2 
2 
2 
2 
2 
2 
2 
2 8//11
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2 
2sB   &D	D(CDC	DC	D4DD"%D"r   c                    t          t          j                            | t                              }i }t          t          |                                          dz             5  |                                sC|	                    d          5 }t          j        i |           ddd           n# 1 swxY w Y   |	                                5 }|                    t          j        |                     ddd           n# 1 swxY w Y   |                    d          }||r@||k    r	 ddd           dS t                              dt           d| d| d	| d
	           ||d<   |	                    d          5 }t          j        ||           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )a  Write a node ip address of the current session to
    RAY_NODE_IP_FILENAME.

    If a ray instance is started by `ray start --node-ip-address`,
    the node ip address is cached to a file RAY_NODE_IP_FILENAME.

    This API is process-safe, meaning the file access is protected by
    a file lock.

    The file contains a single string node_ip_address. If nothing
    is written, it means --node-ip-address was not given, and Ray
    resolves the IP address on its own. It assumes in a single node,
    you can have only 1 IP address (which is the assumption ray
    has in general).

    node_ip_address is the ip address of the current node.

    Args:
        session_dir: The path to Ray session directory.
        node_ip_address: The node IP address of the current node.
            If None, it means the node ip address is not given
            by --node-ip-address. In this case, we don't write
            anything to a file.
    r  w)modeNr   z4The node IP address of the current host recorded in z (z,) is different from the current IP address: z. Ray will use z as the current node's IP address. Creating 2 instances in the same host with different IP address is not supported. Please create an enhnacement request tohttps://github.com/ray-project/ray/issues.)r   rU   r  r  r   r   r_   r  r  r   rr   dumpr   r!  r   r   r   )rC   r   r#  r$  r%  cached_node_ips         r8   write_node_ip_addressr,    s   2 RW\\+/CDDEEI	#i((**++g5	6	6 5 5!! 	!S)) !Q	"a   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ^^ 	8"))$)A,,777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 0334EFF& !_445 5 5 5 5 5 5 5  NNE2E E6DE E +E E <KE E E
 
 
 9H"#45S)) 5Q	0!4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5;5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s   &+F?B3'F?3B7	7F?:B7	;F?(D:F?D
	
F?D
	#F?>AF?F'F?'F+	+F?.F+	/F??GGc                  ,    t          j        dd          S )z|Get the specified node instance id of the current node.

    Returns:
        The node instance id of the current node.
    RAY_CLOUD_INSTANCE_IDr   )rU   rV   r?   r:   r8   get_node_instance_idr/    s     9,b111r:   c                    ddl }t          t          d          si t          _        t          j        }d}t          |          D ]}t          j                            |           }|Tt          t          |                     \  }}	|
                    |t          |	          ||          }|t          j        | <   	 |                                 |c S # t          $ ro}
t          j                            |            ||dz
  k    rt          d|  d|
           t!          j        |           t%          d|d	z            }Y d}
~
d}
~
ww xY wdS )
a  Create a Redis client.

    Args:
        redis_address: The IP address and port of the Redis server.
        password: The password for Redis authentication.
        username: The username for Redis authentication.

    Returns:
        A Redis client.
    r   N	instancesgMbP?)r  r  usernamepasswordr   zUnable to connect to Redis at z:    )redishasattrcreate_redis_clientr1  r   START_REDIS_WAIT_RETRIESranger   r  r  StrictRedisr  pingr  r   r   r   r   min)redis_addressr3  r2  r5  num_retriesdelayicliredis_ip_address
redis_portes              r8   r7  r7    s    LLL&44 +(*%8KE; & &!+//>>;+:5mDD, ,(j ##%__!!	 $  C <?)-8	&HHJJJJJJ 		& 		& 		&)--m<<<K!O##"I]IIaII   Ju519%%EEEEEE		&!& &s   <C
EA$EEra   
fate_shareenv_updatescwdr-   r.   r/   r0   r1   r+   r,   
pipe_stdinc           
         d|                                  d}t          j                            |          dk    rt                              d|           d}d|                                  d}t          j                            |          dk    rt                              d|           d}d|                                  d}t          j                            |          dk    rt                              d|           d}d|                                  d}t          j                            |          dk    rt                              d|           d}	d|                                  d	}t          j                            |          dk    rt                              d|           d}t          j                            d
          t          j                            t          t                    }t          j                            t                    }t          j                            t                    }|sg n|
                    d          }t          ||||          }ni }d|v }t          |||||g          dk    rt          d          |i }t          |t                    st          d          t          j                                        }|                    |           |r|	st          d          t          j                            t(          j        j                                        d| dt1          j                               }| d         }| dd         }d                    d |D                       }t3          |d          5 }|                    d|            ddd           n# 1 swxY w Y   d|d|g} |rg d| z   } |rddg| z   } |r*t          j        d         |d
<   t          j        d          |d!<   |                    |           |	rd"d#d$d                    |            g} r-t(          j        j                                        s
J d%            fd&}ot:          j        d'k    }d(}t:          j        d'k    r1t          d) | D                       } | d*k    rt          d+|            t?          | |||
||rt@          j!        ndt:          j        d'k    r|nd|r|nd,          }!|r	 t(          j        j        "                    |!           tG          j$        |!j%                  &                                 n/# tF          j'        tP          f$ r |!)                                  w xY wd- }"tU          |! |"|
           |"|          |||||	.          S )/a  Start one of the Ray processes.

    TODO(rkn): We need to figure out how these commands interact. For example,
    it may only make sense to start a process in gdb if we also start it in
    tmux. Similarly, certain combinations probably don't make sense, like
    simultaneously running the process in valgrind and the profiler.

    Args:
        command: The command to use to start the Ray process.
        process_type: The type of the process that is being started
            (e.g., "raylet").
        fate_share: If true, the child will be killed if its parent (us) dies.
            True must only be passed after detection of this functionality.
        env_updates: A dictionary of additional environment variables to
            run the command with (in addition to the caller's environment
            variables).
        cwd: The directory to run the process in.
        use_valgrind: True if we should start the process in valgrind.
        use_gdb: True if we should start the process in gdb.
        use_valgrind_profiler: True if we should start the process in
            the valgrind profiler.
        use_perftools_profiler: True if we should profile the process
            using perftools.
        use_tmux: True if we should start the process in tmux.
        stdout_file: A file handle opened for writing to redirect stdout to. If
            no redirection should happen, then this should be None.
        stderr_file: A file handle opened for writing to redirect stderr to. If
            no redirection should happen, then this should be None.
        pipe_stdin: If true, subprocess.PIPE will be passed to the process as
            stdin.

    Returns:
        Information about the process that was started including a handle to
            the process that was started.
    RAY_	_VALGRIND1z#Detected environment variable '%s'.T_VALGRIND_PROFILER_PERFTOOLS_PROFILER_TMUX_GDBr   NrG   )rx   ry   rz   r{   r   r   zAt most one of the 'use_gdb', 'use_valgrind', 'use_valgrind_profiler', 'use_perftools_profiler', and 'use_jemalloc_mem_profiler' flags can be used at a time.z0The 'env_updates' argument must be a dictionary.z;If 'use_gdb' is true, then 'use_tmux' must be true as well.	gdb_init__r   r   c                 8    g | ]}d                      |          S )z'{}'format)rL   r   s     r8   r   z%start_ray_process.<locals>.<listcomp>  s$    LLLCV]]3//LLLr:   r(  zrun gdbz-x)valgrindz--track-origins=yesz--leak-check=fullz--show-leak-kinds=allz!--leak-check-heuristics=stdstringz--error-exitcode=1rW  z--tool=callgrindPERFTOOLS_PATHPERFTOOLS_LOGFILE
CPUPROFILEtmuxznew-sessionz-dzckernel-level fate-sharing must only be specified if detect_fate_sharing_support() has returned Truec                      dd l }  | j        | j        | j        h           rDt          j                            d          r't          j        j	        
                                 d S d S d S )Nr   linux)r   pthread_sigmask	SIG_BLOCKSIGINTr=   r   r   r   r   r   set_kill_on_parent_death_linux)r   rE  s    r8   
preexec_fnz%start_ray_process.<locals>.preexec_fn  s~    v/&-AAA 	@#,11':: 	@L==?????	@ 	@ 	@ 	@r:   r      c                 ,    g | ]}t          |          S r?   )r   )rL   xs     r8   r   z%start_ray_process.<locals>.<listcomp>  s    222Q#a&&222r:   i|  z7command is limited to a total of 31767 characters, got )envrG  stdoutstderrr   rb  r   c                 X    | '	 | j         S # t          $ r t          |           cY S w xY wd S r<   )r[   AttributeErrorr_   )streams    r8   _get_stream_namez+start_ray_process.<locals>._get_stream_name  sG    #{"! # # #6{{"""#ts    ''r)   )+upperrU   r   r   r   r   r#   JEMALLOC_SOr$   r%   rY   r   sumr  r   dictcopyr   r  r  r   r   r   get_ray_temp_dirr   r   writer   r   r=   r   r   r   PIPEset_kill_child_on_death_win32r   ProcesspidresumeErrorOSErrorkillr(   )#ra   r{   rE  rF  rG  r-   r.   r/   r0   r1   r+   r,   rH  valgrind_env_varvalgrind_profiler_env_varperftools_profiler_env_vartmux_env_vargdb_env_varjemalloc_lib_pathry   rz   jemalloc_env_varsuse_jemalloc_mem_profilermodified_envgdb_init_pathray_process_pathray_process_argsrun_argsgdb_init_filerb  win32_fate_sharingCREATE_SUSPENDED
total_chrsr*   rl  s#     `                                r8   start_ray_processr    s   f >l0022===	z~~&''3..9;KLLL O|'9'9';'; O O O	z~~/00C779;TUUU $!Q(:(:(<(<!Q!Q!Q	z~~011S889;UVVV!%5,,,..555L	z~~l##s**9<HHH3++--333K	z~~k""c))9;GGG	z~~l##+JNN+@+NN
'899(<==#1P~7K7KC7P7P6+')%	
 
 
  -1B B 	%&)	
 	
 		 		 !
 
 	
 k4(( MKLLL:??$$L$$$ A 	M  
 K..00444ty{{44
 
 #1:"122;88LL;KLLLMM-%% 	3 1x 1 1222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3*D-@ 
 
 
   =12W< E%'Z0@%A\"%'Z0C%D\")*** H =$388G3D3D0FG 
|!==?? 	
 	
>	
 	
?
@ @ @ @ @ $?(? "
|w 22'22233
$!$ $  
 !+5joo!$!8!8::d*<C&&!	 	 	G  	L<<WEEEN7;''..0000g& 	 	 	LLNNN	   $$[11$$[11!35	 	 	 	s   ,OOOAU( (,Vc                    	 t           j        dk    rt          j                     n{# t          $ rn}|j        }|t
          j        k    r)t          j                    t          j                    k    rn$t          
                    d| d           Y d}~dS Y d}~nd}~ww xY wt          j                            t          t          d          }t           j        d|g}t!          |t"          j        d|           }|S )	ag  Start the reaper process.

    This is a lightweight process that simply
    waits for its parent process to die and then terminates its own
    process group. This allows us to ensure that ray processes are always
    terminated properly so long as that process itself isn't SIGKILLed.

    Returns:
        ProcessInfo for the process that was started.
    r   z:setpgrp failed, processes may not be cleaned up properly: .Nzray_process_reaper.pyrF   T)rH  rE  )r=   r   rU   setpgrprz  errnoEPERMgetpgrpgetpidr   r   r  r  RAY_PATHRAY_PRIVATE_DIRrT   r  r   PROCESS_TYPE_REAPER)rE  rD  errcodereaper_filepathra   process_infos         r8   start_reaperr     s    <7""JLLL   'ek!!bjllbikk&A&ANNQQQQQ  
 44444 DDDD	 gll8_>UVVO~t_5G$)	  L s   #& 
BABBlogs_dir	max_bytesbackup_countstdout_filepathstderr_filepathc           	      t   t           j                            t          t          d          }t
          j        d|d|  d| d| d| d| g}	|r|	                    d|            |r|	                    d	|            |Y|W|	                    d           t          j	        
                    t          j                  }
|	                    d|
            d
}|rt          t           j        d          }d
}|rt          t           j        d          }t          |	t          j        |||          }|S )ay  Start a log monitor process.

    Args:
        session_dir: The session directory.
        logs_dir: The directory of logging files.
        gcs_address: GCS address for pubsub.
        fate_share: Whether to share fate between log_monitor
            and this process.
        max_bytes: Log rotation parameter. Corresponding to
            RotatingFileHandler's maxBytes.
        backup_count: Log rotation parameter. Corresponding to
            RotatingFileHandler's backupCount.
        redirect_logging: Whether we should redirect logging to
            the provided log directory.
        stdout_filepath: The file path to dump log monitor stdout.
            If None, stdout is not redirected.
        stderr_filepath: The file path to dump log monitor stderr.
            If None, stderr is not redirected.

    Returns:
        ProcessInfo for the process that was started.
    zlog_monitor.pyrF   --session-dir=--logs-dir=--gcs-address=--logging-rotate-bytes=--logging-rotate-backup-count=--stdout-filepath=--stderr-filepath=N--logging-filename=rB   --logging-format=r(  r+   r,   rE  )rU   r  r  r  r  r=   rT   r5   r   LOGGER_FORMAT_STDERRrU  PROCESS_TYPE_LOG_MONITORr   devnullr  )rC   r  r   rE  r  r  r  r  log_monitor_filepathra   logging_formatr+   r,   r  s                 r8   start_log_monitorr  )  s|   @ 7<</CSTT 	&&& h  &&&-)--777	G  ?=O==>>> ?=O==>>>?#: 	,---&;BB#< C 
 
 	;>;;<<<K ,2:s++K ,2:s++$.  L r:   include_dashboardraise_on_failurecluster_id_hexlogdirr  c                 
   	 |	d}t           j        }	nd}t          j        t          |          rt          j        nt          j        t          j                  }|                    t          j        t          j	        d           	 |
                    ||	f           |                                 n8# t          j        $ r&}|j        dv rt          d| d|	 d          |d}~ww xY wt          j        j                                        }| r/|r-t&                              d	| d
           t)          d          | dn| }d}t*          j                            t0          |d          }g t3          t           j        |d          |d| d|	 d| d| d d| d| d| d| d| d| }|r|                    d|            |r|                    d|            |Y|W|                    d           t           j                            t           j                  }|                    d |            ||                    d!           |s*|                    d"           |                    d#           d}|rt=          t*          j        d$          }d}|rt=          t*          j        d$          }tA          |t           j        |||
%          }tC          ||&          }t          j"        j#        $                    |           d}d}tK          j%                    }tK          j%                    |z
  t           j&        k     rt          j"        j#        '                    t           j(        t           j)        '          }||*                    d(          }nT|j+        ,                                }|n8tK          j-        d)           tK          j%                    |z
  t           j&        k     ||d*| nd+}t&                              d,|            fd-} rd.}!t&                              d/|! d0           	  | d1|!2          }"ni# t(          $ r\}t&                              d3| d4           	  | d5|!2          }"n%# t(          $ r}t)          d6| d7          d}~ww xY wY d}~nd}~ww xY wd8d8                    t]          |"|! d                             z   }#t)          |#          t)          d9          ||sd+}||fS # t(          $ r*}|r||t&                              |           Y d}~d:S d}~ww xY w);a  Start a API server process.

    Args:
        include_dashboard: If true, this will load all dashboard-related modules
            when starting the API server, or fail. If None, it will load all
            dashboard-related modules conditioned on dependencies being present.
            Otherwise, it will only start the modules that are not relevant to
            the dashboard.
        raise_on_failure: If true, this will raise an exception
            if we fail to start the API server. Otherwise it will print
            a warning if we fail to start the API server.
        host: The host to bind the dashboard web server to.
        gcs_address: The gcs address the dashboard should connect to
        cluster_id_hex: Cluster ID in hex.
        node_ip_address: The IP address where this is running.
        temp_dir: The temporary directory used for log files and
            information for this Ray session.
        session_dir: The session directory under temp_dir.
            It is used as a identifier of individual cluster.
        logdir: The log directory used to generate dashboard log.
        port: The port to bind the dashboard web server to.
            Defaults to 8265.
        max_bytes: Log rotation parameter. Corresponding to
            RotatingFileHandler's maxBytes.
        backup_count: Log rotation parameter. Corresponding to
            RotatingFileHandler's backupCount.
        stdout_filepath: The file path to dump dashboard stdout.
            If None, stdout is not redirected.
        stderr_filepath: The file path to dump dashboard stderr.
            If None, stderr is not redirected.

    Returns:
        A tuple of :
            - Dashboard URL if dashboard enabled and started.
            - ProcessInfo for the process that was started.
    N2   r   r   >   '  0   b   zFailed to bind to :z because it's already occupied. You can use `ray start --dashboard-port ...` or `ray.init(dashboard_port=...)` to select a different port.z7Ray dashboard dependencies failed to install properly: a  .
Potential causes include:
1. --include-dashboard is not supported when minimal ray is used. Download ray[default] to use the dashboard.
2. Dashboard dependencies are conflicting with your python environment. Investigate your python environment and try reinstalling ray[default].
z/Cannot include dashboard with missing packages.T	dashboardzdashboard.pyF)rD   --host=--port=z--port-retries=--temp-dir=
--log-dir=r  r  r  r  --cluster-id-hex=--node-ip-address=r  r  r  r  r  	--minimalz --modules-to-load=UsageStatsHeadz--disable-frontendr(  r  )r  
cluster_idr   ro   r   z, return code r   zFailed to start the dashboard c           	      h   t           j                            	|           }d}g }t          |d          5 }t	          j        |                                dt          j                  5 }|                                }t          |          D ]Y}|	                    dd|dz
            }|dk    r n6|
                    ||dz   |                             d                     |}Zd	d	d	           n# 1 swxY w Y   d	d	d	           n# 1 swxY w Y   |
                    d
| d| d           |S )z-Read a log file and return the last 20 lines.   rbr   )access   
r   ro   Nz	The last z
 lines of z5 (it contains the error message from the dashboard): )rU   r  r  r   mmapfilenoACCESS_READsizer9  rfindr5   ru   )
filenamelines_to_readdashboard_loglinesr%  mmendrR  sepr  s
            r8   read_logz"start_api_server.<locals>.read_log@  s    "VX > > "-.. &!188::q9IJJJ &b ggii!&}!5!5 & &A"$((5!S1W"="=C"byy %!LLC!GcM):)A)A')J)JKKK"%CC& & & & & & & & & & & & & & && & & & & & & & & & & & & & & K K K K K K   s6   4D*A>C4(D4C8	8D;C8	<DDDr  zXError should be written to 'dashboard.log' or 'dashboard.err'. We are printing the last z lines for you. See 'https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#logging-directory-structure' to find where the log file is.zdashboard.log)r  z)Couldn't read dashboard.log file. Error: z. It means the dashboard is broken even before it initializes the logger (mostly dependency issues). Reading the dashboard.err file which contains stdout/stderr.zdashboard.errz#Failed to read dashboard.err file: zc. It is unexpected. Please report an issue to Ray github. https://github.com/ray-project/ray/issues
zFailed to start a dashboard.NN)/r   DEFAULT_DASHBOARD_PORTsocketr   AF_INET6AF_INETSOCK_STREAM
setsockopt
SOL_SOCKETSO_REUSEADDRbindr   errorr  r  r   r   r   get_dashboard_dependency_errorr   r  rU   r  r  r  rf   PROCESS_TYPE_DASHBOARDr5   r  rU  r   r  r  r   r   r   _initialize_internal_kvr   RAY_DASHBOARD_STARTUP_TIMEOUT_Sr   DASHBOARD_ADDRESSr   ru   r*   pollr   reversed)$r  r  r  r   r  r   r   r  rC   r  rE  r  r  r  r  port_retriesport_test_socketrD  dashboard_dependency_errorinclude_dashdashboard_dirdashboard_filepathra   r  r+   r,   r  
gcs_clientdashboard_urldashboard_returncodestart_time_sreturncode_strr  r  r  last_log_strs$          `                            r8   start_api_serverr  w  s   jX<L 7DDL%}#*4==Dfn"    ''!#  
 %%tTl333 &&((((<    7o--$9T 9 9D 9 9 9   G &)\%7%V%V%X%X"  		O!; 		OLL[Jd [ [ [   MNNN%6%>TTDU $W\\(M>RR
@4   
 
 d
 d
 -l,,
 %($$
 "!!
 +[**
 2i11
 <\;;
 +[**
  100!
" 322#
(  	CNNAAABBB 	CNNAAABBB"'> NN0111*?FF'> G  N NN?~??@@@%1NN;''' 	1
 NN=>>>NN/000 	0rz3//K 	0rz3//K(0##!
 
 
 {~NNN
$<<ZHHH#y{{IKK,&)VVV,8II/'> J  M ( - 4 4W = =#/#7#<#<#>#> #/ JsOOO IKK,&)VVV$   (3 8!5777 
 LLJ.JJKKK    *  #@ "5$5 5 5  $H_MRRREE    LLWA W W W   ( V V V$   'H! H H H   (  $dii~9O0P0P&Q&QQ---   >???%11 Ml**    	NLLOOO:::::s   BT# +B2 1T# 2C'!C""C''M.T# Q$ #T# $
S
.SRS
R=%R88R==S T# S

AT# #
U-UUc                 @   |                      dd          }d}t          |          dk    rt          |d                   \  }}nVt          |          dk    s
|d         dvrt          d|  d          t          |d                   \  }}|d         d	k    rd
}|||fS )Nz://r   Fr   r4  )r5  redisszInvalid redis address zL.Expected format is ip:port or redis://ip:port, or rediss://ip:port for SSL.r  T)rY   r   r   r  )r=  partsenable_redis_sslrB  rC  s        r8   get_addressr    s    q))E
5zzQ'4U1X'>'>$** u::??eAh.AAA/ / / /  
 (5U1X'>'>$*8x#Z)999r:   r=  log_dirrc   redis_usernameredis_passwordrv   gcs_server_portmetrics_agent_portc           	         |	dk    sJ t           d| dt          |           d|	 d|
 d| d| dt          j         g}|r	|d	| gz  }|r	|d
| gz  }| r(t	          |           \  }}}|d| d| d|rdnd gz  }|r	|d| gz  }|r	|d| gz  }d}|rt          t          j        d          }d}|rt          t          j        d          }t          |t          j
        |||          }|S )a  Start a gcs server.

    Args:
        redis_address: The address that the Redis server is listening on.
        log_dir: The path of the dir where gcs log files are created.
        stdout_filepath: The file path to dump gcs server stdout.
            If None, stdout is not redirected.
        stderr_filepath: The file path to dump gcs server stderr.
            If None, stderr is not redirected.
        session_name: The current Ray session name.
        redis_username: The username of the Redis server.
        redis_password: The password of the Redis server.
        config: Optional configuration that will
            override defaults in RayConfig.
        gcs_server_port: Port number of the gcs server.
        metrics_agent_port: The port where metrics agent is bound to.
        node_ip_address: IP Address of a node where gcs server starts.

    Returns:
        ProcessInfo for the process that was started.
    r   
--log_dir=z--config_list=z--gcs_server_port=--metrics-agent-port=r  --session-name=z--ray-commit=--stdout_filepath=--stderr_filepath=z--redis_address=z--redis_port=z--redis_enable_ssl=truefalsez--redis_username=z--redis_password=Nr(  r  )GCS_SERVER_EXECUTABLErw   r   
__commit__r  r   rU   r  r  r   PROCESS_TYPE_GCS_SERVER)r=  r  r  r  rc   r  r   rv   rE  r  r  r   ra   rB  rC  r  r+   r,   r  s                      r8   start_gcs_serverr    s   F Q 	W3)&1133._..4 244._..(,(((((	G  <:::;; <:::;; 
9D]9S9S6*&61/11(J((K,<"I&&'KK
 	

  :88899 :88899K ,2:s++K ,2:s++$-  L r:   r   node_manager_portraylet_nameplasma_store_namer  worker_pathsetup_worker_pathresource_dirplasma_directoryfallback_directoryobject_store_memoryis_head_noderesource_isolation_configmin_worker_portmax_worker_portworker_port_listobject_manager_portmetrics_export_portdashboard_agent_listen_portruntime_env_agent_portuse_profilerraylet_stdout_filepathraylet_stderr_filepathdashboard_agent_stdout_filepathdashboard_agent_stderr_filepath!runtime_env_agent_stdout_filepath!runtime_env_agent_stderr_filepath
huge_pagessocket_to_useray_debugger_external	node_namewebuic0                 v   |t          |          t          u sJ |r| rt          d          |                                }0|j        }1|0                    dd          }2t          dt          t          j	                    |2                    }3d
                    d |0                                D                       }4d}5t          j        d	          d
}5d}6	 t                      }7t          j                            |7          rd
}6n# t$          $ r Y nw xY w|5o|6}8|8d
u rt'          ||||||||	          }9ng }9t          j                            t(                    rt+          |||||||||		  	        }:ng }:t,          j        |	gt1                      z   |d| dd| d| d|  d| d|* d|+ d| d| d| d|
 d|/ d| gz   };|;                    d           |r	|;d| gz  };|r	|;d| gz  };|d}|d}|d}d}<|1rt5          j        |1          }<g t9          t:          j        |          t          j        
                    t>          dd          d| d| d | d!| dd| d| d|
 d"| d#| d|* d|+ d| d| d$| }=|#r|=                    d%|#            |$r|=                    d&|$            |#Y|$W|=                    d'           t:          j         !                    t:          j        (          }>|=                    d)|>            tD          j#        j$        %                                |=                    d*           g t9          t:          j&        |          t          j        
                    t>          d+d,d-d.          d| d| d| d$| d/| d|* d|+ d#| d|
 }?|%r|?                    d%|%            |&r|?                    d&|&            |%Y|&W|?                    d'           t:          j         !                    t:          j&        (          }>|?                    d)|>            tN          d0| d1| d2| d3| d4| d5| d6| d7| d8|3 d9|4 d:tQ          j)        |;           d;tQ          j)        |9           d<tQ          j)        |:           d=tT           d>|
 d?| d@| dA| d| dB| dC| dD| dE| dF| dG|,rdnd d| d| dH|< d| g}@|+                                rtY          j-        dI|j.         dJ|j/         dK|j0         dL           |@                    dM           |@                    dN|j.                    |@                    dO|j/                    |@                    dP|j0                    |@                    dQ|j1                    |!r|@                    dR|!            |"r|@                    dS|"            |r|@                    dT           ||@                    dU|            |@                    dV!                    t          |j2                                       |@                    dW!                    tQ          j)        |=                               |@                    dX!                    tQ          j)        |?                               |'r|@                    dY           |)r|)3                                 |.|@                    dZ|.            d}A|!rti          t          j5        d[          }Ad}B|"rti          t          j5        d[          }Btm          |@t:          j7        |d| d\t          j8        v |A|B|(|-]
  
        }C|CS )^a  Start a raylet, which is a combined local scheduler and object manager.

    Args:
        redis_address: The address of the primary Redis server.
        gcs_address: The address of GCS server.
        node_id: The hex ID of this node.
        node_ip_address: The IP address of this node.
        node_manager_port: The port to use for the node manager. If it's
            0, a random port will be used.
        raylet_name: The name of the raylet socket to create.
        plasma_store_name: The name of the plasma store socket to connect
             to.
        worker_path: The path of the Python file that new worker
            processes will execute.
        setup_worker_path: The path of the Python file that will set up
            the environment for the worker process.
        temp_dir: The path of the temporary directory Ray will use.
        session_dir: The path of this session.
        resource_dir: The path of resource of this session .
        log_dir: The path of the dir where log files are created.
        resource_and_label_spec: Resources and key-value labels for this raylet.
        plasma_directory: A directory where the Plasma memory mapped files will
            be created.
        fallback_directory: A directory where the Object store fallback files will be created.
        object_store_memory: The amount of memory (in bytes) to start the
            object store with.
        session_name: The current Ray session name.
        resource_isolation_config: Resource isolation configuration for reserving
            memory and cpu resources for ray system processes through cgroupv2
        is_head_node: whether this node is the head node.
        min_worker_port: The lowest port number that workers will bind
            on. If not set, random ports will be chosen.
        max_worker_port: The highest port number that workers will bind
            on. If set, min_worker_port must also be set.
        worker_port_list: An explicit list of ports to be used for
            workers (comma-separated). Overrides min_worker_port and
            max_worker_port.
        object_manager_port: The port to use for the object manager. If this is
            None, then the object manager will choose its own port.
        redis_username: The username to use when connecting to Redis.
        redis_password: The password to use when connecting to Redis.
        metrics_agent_port: The port where metrics agent is bound to.
        metrics_export_port: The port at which metrics are exposed to.
        dashboard_agent_listen_port: The port at which the dashboard agent
            listens to for HTTP.
        runtime_env_agent_port: The port at which the runtime env agent
            listens to for HTTP.
        use_valgrind: True if the raylet should be started inside
            of valgrind. If this is True, use_profiler must be False.
        use_profiler: True if the raylet should be started inside
            a profiler. If this is True, use_valgrind must be False.
        raylet_stdout_filepath: The file path to dump raylet stdout.
            If None, stdout is not redirected.
        raylet_stderr_filepath: The file path to dump raylet stderr.
            If None, stderr is not redirected.
        dashboard_agent_stdout_filepath: The file path to dump
            dashboard agent stdout. If None, stdout is not redirected.
        dashboard_agent_stderr_filepath: The file path to dump
            dashboard agent stderr. If None, stderr is not redirected.
        runtime_env_agent_stdout_filepath: The file path to dump
            runtime env agent stdout. If None, stdout is not redirected.
        runtime_env_agent_stderr_filepath: The file path to dump
            runtime env agent stderr. If None, stderr is not redirected.
        huge_pages: Boolean flag indicating whether to start the Object
            Store with hugetlbfs support. Requires plasma_directory.
        fate_share: Whether to share fate between raylet and this process.
        max_bytes: Log rotation parameter. Corresponding to
            RotatingFileHandler's maxBytes.
        backup_count: Log rotation parameter. Corresponding to
            RotatingFileHandler's backupCount.
        ray_debugger_external: True if the Ray debugger should be made
            available externally to this node.
        env_updates: Environment variable overrides.
        node_name: The name of the node.
        webui: The url of the UI.
    Returns:
        ProcessInfo for the process that was started.
    Nz2Cannot use valgrind and profiler at the same time.CPUr   r   rG   c                 "    g | ]} d j         | S )z{},{}rT  )rL   kvs     r8   r   z start_raylet.<locals>.<listcomp>  s!    @@@	@@@r:   FjavaTr  z5--node-manager-port=RAY_NODE_MANAGER_PORT_PLACEHOLDERz--object-store-name=z--raylet-name=z--redis-address=r  r  r  z--runtime-env-agent-port=r  r  r  z--webui=z--cluster-id=%RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER--redis-username=--redis-password=r   r  zagent.pyz--metrics-export-port=z--grpc-port=z--listen-port=r  r  r  r  r  r  r  r  r  r   runtime_envagentzmain.pyz--runtime-env-dir=z--raylet_socket_name=z--store_socket_name=z--object_manager_port=z--min_worker_port=z--max_worker_port=z--node_manager_port=z
--node_id=z--node_ip_address=z--maximum_startup_concurrency=z--static_resource_list=z--python_worker_command=z--java_worker_command=z--cpp_worker_command=z--native_library_path=z--temp_dir=z--session_dir=r  z--resource_dir=z--metrics_export_port=z--runtime_env_agent_port=z--object_store_memory=z--plasma_directory=z--fallback_directory=z--ray-debugger-external=z	--labels=z,Resource isolation enabled with cgroup_path=z, system_reserved_cpu=z system_reserved_memory=r  z--enable-resource-isolationz--cgroup-path=z--system-reserved-cpu-weight=z--system-reserved-memory-bytes=z--system-pids=r  r  z--headz--worker_port_list=z --num_prestart_python_workers={}z--dashboard_agent_command={}z--runtime_env_agent_command={}z--huge_pagesz--node-name=r(  RAYLET_PERFTOOLS_PATH)r-   r.   r/   r0   r+   r,   rE  rF  )9typer  r  to_resource_dictlabelsr   maxr<  multiprocessing	cpu_countr  itemsshutilwhichget_ray_jars_dirrU   r  r  r  build_java_worker_commandDEFAULT_WORKER_EXECUTABLEbuild_cpp_worker_commandr=   rT   r9   r5   rr   rs   rf   r   PROCESS_TYPE_DASHBOARD_AGENTr  r  rU  r   r   r   r  PROCESS_TYPE_RUNTIME_ENV_AGENTRAYLET_EXECUTABLEr   list2cmdlineDEFAULT_NATIVE_LIBRARY_PATH
is_enabledloggingr   cgroup_pathsystem_reserved_cpu_weightsystem_reserved_memorysystem_pidsnum_cpusr   r   r  r  PROCESS_TYPE_RAYLETr   )Dr=  r   r   r   r  r  r  r  r  r  r   rC   r  r  resource_and_label_specr  r  r  rc   r  r  r  r  r  r  r  r   r  r  r  r   r-   r!  r"  r#  r$  r%  r&  r'  r(  rE  r)  r  r  r*  rF  r+  r,  static_resourcesr:  num_cpus_staticmaximum_startup_concurrencyresource_argumenthas_java_commandray_java_installedjars_dirinclude_javajava_worker_commandcpp_worker_commandstart_worker_commandlabels_json_strdashboard_agent_commandr  runtime_env_agent_commandra   r+   r,   r  sD                                                                       r8   start_rayletra    s   @ (T2C-D-D-K-K-KK O OMNNN /??AA$+F '**5!44O"%	3(**O<<# #
 @@'7'='='?'?@@@  |F'#%%7>>(## 	&!%    $:(:Lt7	
 	
 !	w~~/00  5

 

   N	
 --		 222C6#466*[**.}..8$6881i11;\;;@(>@@*[**,l,,$($$u(J((
	
 2  GHHH G!E^!E!E FF G!E^!E!E FF "O -*V,,	<6

 

 	X{J77	
 	/_.. 	7!466 	,)++ 	7466 	@ 	3022 	'&& 	!h   	'&& 	W 	.)--  	877!" 	),((#$ 	'&&%& 	)J(('* ' 
&&B!@BB	
 	
 	
 ' 
&&B!@BB	
 	
 	
 	(/+3 	 &&'<===&;BB#@ C 
 
 	 &&'K>'K'KLLL
|88::F 	 &&{333!	<8+

 

! 	Xz='9MM	!
 	/_..! 	=$:<<! 	'&&! 	)J((! 	,\++! 	.)--! 	877! 	W! 	!h  ! ) 
!((D!BDD	
 	
 	
 ) 
!((D!BDD	
 	
 	
 	*1-5 	"(()>???&;BB#B C 
 
 	"(()M^)M)MNNN 	---20226!466._..._..2022W._..F)DFF5"355R:#:;O#P#PRRO!89L!M!MOOM
 78J K KMM>!<>> h  &&&W(,((4 2446!466<$:<<6!4660.004 244F(=#D111FF&&&(,((%O%%$
$$=GB !++-- QZ;T;` Z Z#<#WZ Z&?&VZ Z Z	
 	
 	

 	4555O(A(MOOPPPb,E,`bb	
 	
 	
 	`.G.^``	
 	
 	
 	O(A(MOOPPP FD,BDDEEE FD,BDDEEE !x   #?-=??@@@NN*11#6M6V2W2WXX   NN&--#$;<<	
 	
  
 NN(//#$=>>	
 	
  
  '~&&& &9&&	
 	
 	
 K ,2:s++K ,2:s++$)!* 72: E  L s   /D
 

DDc                  `   t           } t          j                            t          j                            | d                    }t          j                            |          st          d          t          j                            t          j                            | d                    S )zPReturn a directory where all ray-related jars and
    their dependencies locate.jarszfRay jars is not packaged into ray. Please build ray with java enabled (set env var RAY_INSTALL_JAVA=1))r  rU   r  abspathr  r  r   )current_dirrY  s     r8   rA  rA    s     Kwrw||K@@AAH7>>(## 
/
 
 	

 7??27<<V<<===r:   c                    g }| |                     d| f           |                     d           ||                     d|f           ||                     d|f           ||                     d|f           ||                     d|f           ||                     d|f           |                     d	t          f           |                     d
t          j                            |d          f           |                     d|f           t
          j        g|gz   d |D             z   }	|	dgz  }	|	dgz  }	|	S )a  This method assembles the command used to start a Java worker.

    Args:
        bootstrap_address: Bootstrap address of ray cluster.
        plasma_store_name: The name of the plasma store socket to connect
           to.
        raylet_name: The name of the raylet socket to create.
        redis_username: The username to connect to Redis.
        redis_password: The password to connect to Redis.
        session_dir: The path of this session.
        node_ip_address: The IP address for this node.
        setup_worker_path: The path of the Python file that will set up
            the environment for the worker process.
    Returns:
        The command string for starting Java worker.
    Nzray.address)zray.raylet.node-manager-port!RAY_NODE_MANAGER_PORT_PLACEHOLDERzray.object-store.socket-namezray.raylet.socket-namezray.redis.usernamezray.redis.passwordzray.node-ipzray.homezray.logging.dirlogszray.session-dirc                 "    g | ]} d j         | S )z-D{}={}rT  )rL   pairs     r8   r   z-build_java_worker_command.<locals>.<listcomp>  s#    
5
5
5t9T"
5
5
5r:   r2  z*io.ray.runtime.runner.worker.DefaultWorker)r5   RAY_HOMErU   r  r  r=   rT   )
r  r  r  r  r   rC   r   r  pairsra   s
             r8   rB  rB    sx   4 E$m%67888	LLVWWW$46GHIII.<===!*N;<<<!*N;<<<"m_5666	LL*h'(((	LL#RW\\+v%F%FGHHH	LL#[1222	
	
5
5u
5
5
5	6  788G<==GNr:   c	                 r    t           j        |t          d| d| dd|  d| d| d| d| d	| d
g}	|	S )a  This method assembles the command used to start a CPP worker.

    Args:
        bootstrap_address: The bootstrap address of the cluster.
        plasma_store_name: The name of the plasma store socket to connect
           to.
        raylet_name: The name of the raylet socket to create.
        redis_username: The username to connect to Redis.
        redis_password: The password to connect to Redis.
        session_dir: The path of this session.
        log_dir: The path of logs.
        node_ip_address: The ip address for this node.
        setup_worker_path: The path of the Python file that will set up
            the environment for the worker process.
    Returns:
        The command string for starting CPP worker.
    z--ray_plasma_store_socket_name=z--ray_raylet_socket_name=z9--ray_node_manager_port=RAY_NODE_MANAGER_PORT_PLACEHOLDERz--ray_address=z--ray_redis_username=z--ray_redis_password=z--ray_session_dir=z--ray_logs_dir=z--ray_node_ip_address=r2  )r=   rT   rC  )
r  r  r  r  r   rC   r  r   r  ra   s
             r8   rD  rD    s}    < 	!=*;==1K11C,*,,000000*[**#'##222/G  Nr:   c                 Z   t          | t                    st          |           } |r/t          j        dk    st          j        dk    st	          d          t
          j        j                                        }|Wt          j        dk    st          j        dk    rt
          j	        j        
                                }|| k    rd}nt          j                            d          s9| t          j        k    r)t	          d                    | dz  |dz                      t
          j        j                                        }t$                              d	                    t
          j        j                                        || d
z  dz                       n#t
          j        j                                        }| |k    rt	          d          n9t          j                            |          }t$                              d           t          j                            |          st	          d| d          |r|t	          d          | t          j        k     r-t	          d                    | t          j                            t          j        dk    rt| t          j        k    rdt          j                            d          dk    rAt	          d                    | dz  t          j        dz  t          j        dz                      ||}nt          j                            |          }t          j                            |          st	          d| d          t$                              d                    t7          | dz  d          ||                     ||| fS )a  Figure out how to configure the plasma object store.

    This will determine:
    1. which directory to use for the plasma store. On Linux,
    we will try to use /dev/shm unless the shared memory file system is too
    small, in which case we will fall back to /tmp. If any of the object store
    memory or plasma directory parameters are specified by the user, then those
    values will be preserved.
    2. which directory to use for the fallback files. It will default to the temp_dir
    if it is not extracted from the object_spilling_config.

    Args:
        object_store_memory: The object store memory to use.
        plasma_directory: The user-specified plasma directory parameter.
        fallback_directory: The path extracted from the object_spilling_config when the
                            object spilling config is set and the spilling type is to
                            filesystem.
        huge_pages: The user-specified huge pages parameter.

    Returns:
        A tuple of plasma directory to use, the fallback directory to use, and the
        object store memory to use. If it is specified by the user, then that value will
        be preserved.
    r]  linux2z3The huge_pages argument is only supported on Linux.Nz/dev/shm#RAY_OBJECT_STORE_ALLOW_SLOW_STORAGEzThe configured object store size ({} GB) exceeds /dev/shm size ({} GB). This will harm performance. Consider deleting files in /dev/shm or increasing its size with --shm-size in Docker. To ignore this warning, set RAY_OBJECT_STORE_ALLOW_SLOW_STORAGE=1.g    eAa  WARNING: The object store is using {} instead of /dev/shm because /dev/shm has only {} bytes available. This will harm performance! You may be able to free up space by deleting files in /dev/shm. If you are inside a Docker container, you can increase /dev/shm size by passing '--shm-size={:.2f}gb' to 'docker run' (or add it to the run_options list in a Ray cluster config). Make sure to set this to more than 30% of available RAM.g?i   @zRThe requested object store memory size is greater than the total available memory.zAobject_store_memory is not verified when plasma_directory is set.zThe plasma directory file z& does not exist or is not a directory.zKIf huge_pages is True, then the plasma_directory argument must be provided.z]Attempting to cap object store memory usage at {} bytes, but the minimum allowed is {} bytes.darwin!RAY_ENABLE_MAC_LARGE_OBJECT_STORErL  a  The configured object store size ({:.4}GiB) exceeds the optimal size on Mac ({:.4}GiB). This will harm performance! There is a known issue where Ray's performance degrades with object store size greater than {:.4}GB on a Mac.To reduce the object store capacity, specify`object_store_memory` when calling ray.init() or ray start.To ignore this warning, set RAY_ENABLE_MAC_LARGE_OBJECT_STORE=1.zThe fallback directory file zXDetermine to start the Plasma object store with {} GB memory using {} and fallback to {}i ʚ;r4  )r   r  r=   r   r  r   r   r   get_system_memoryr   get_shared_memory_bytesrU   r   r   r   REQUIRE_SHM_SIZE_THRESHOLDrU  get_user_temp_dirr   r   r  rd  r   isdir!OBJECT_STORE_MINIMUM_MEMORY_BYTES!MAC_DEGRADED_PERF_MMAP_SIZE_LIMITdebuground)r  r   r  r  r(  system_memory	shm_avails          r8   determine_plasma_store_configr~    s   > )3// 7!"566 P3<722clh6N6NNOOOK%7799M
 <7""clh&>&>*BBDDI ///#-  JNN#HII'-*RRR A
 BH+c19s?B B	 	 	 $';#4#F#F#H#H J KQ&);;==!+s3u=K K     #{0BBDD ..3   / 7??+;<<WXXX7==)** 
a)9aaa
 
 	
  
&.:
 
 	

 ]LLL::@&#]%T; ;
 
 	
 	  -"QQQJNN>??3FF7 8>v#e+?%G?%G8 8
 
 	
  !%W__-?@@7==+,, 
e+=eee
 
 	

 LL	&&,f%-q11'
 '
   /1DDDr:   autoscaling_config
monitor_ipautoscaler_v2c
                 z   |	r,t           j                            t          t          d          }
n+t           j                            t          t
          d          }
t          j        d|
d| d| d| g}| J |                    d|             |r|                    d|            |r|                    d	|            |Y|W|                    d
           t          j
                            t          j                  }|                    d|            |r%|                    dt          |          z              |r|                    d|z              d}|rt          t           j        d          }d}|rt          t           j        d          }t!          |t          j        |||          }|S )aa  Run a process to monitor the other processes.

    Args:
        gcs_address: The address of GCS server.
        logs_dir: The path to the log directory.
        stdout_filepath: The file path to dump monitor stdout.
            If None, stdout is not redirected.
        stderr_filepath: The file path to dump monitor stderr.
            If None, stderr is not redirected.
        autoscaling_config: path to autoscaling config file.
        max_bytes: Log rotation parameter. Corresponding to
            RotatingFileHandler's maxBytes.
        backup_count: Log rotation parameter. Corresponding to
            RotatingFileHandler's backupCount.
        monitor_ip: IP address of the machine that the monitor will be
            run on. Can be excluded, but required for autoscaler metrics.
    Returns:
        ProcessInfo for the process that was started.
    z
monitor.pyrF   r  r  r  Nr  r  r  r  r  r  z--autoscaling-config=z--monitor-ip=r(  r  )rU   r  r  r  AUTOSCALER_V2_DIRAUTOSCALER_PRIVATE_DIRr=   rT   r5   r   r  rU  PROCESS_TYPE_MONITORr_   r   r  r  )r   r  r  r  r  rE  r  r  r  r  
entrypointra   r  r+   r,   r  s                   r8   start_monitorr    s   >  RW\\(,=|LL

W\\(,BLQQ
 	 h  -)--777G """NN1K11222 ?=O==>>> ?=O==>>>?#: 	,---&;BB#8 C 
 
 	;>;;<<< J.5G1H1HHIII 53444K ,2:s++K ,2:s++$*  L r:   proxyr  ray_client_server_ipray_client_server_portruntime_env_agent_addressserver_typeserialized_runtime_env_contextc                    t          t                                                    j        d         }t          j                            |ddt          j                  }|}t          j
        |ddd|  d| d| d	|	 d
t          j        t          j                   g	}|r|                    d|            |r|                    d|            |
r|                    d|
            |	dk    rt          |          dk    sJ |r|                    d|            t!          |t          j        |||          }|S )a9  Run the server process of the Ray client.

    Args:
        address: The address of the cluster.
        ray_client_server_ip: Host IP the Ray client server listens on.
        ray_client_server_port: Port the Ray client server listens on.
        stdout_file: A file handle opened for writing to redirect stdout to. If
            no redirection should happen, then this should be None.
        stderr_file: A file handle opened for writing to redirect stderr to. If
            no redirection should happen, then this should be None.
        redis_username: The username of the Redis server.
        redis_password: The password of the Redis server.
        runtime_env_agent_address: Address to the Runtime Env Agent listens on via HTTP.
            Only needed when server_type == "proxy".
        server_type: Whether to start the proxy version of Ray Client.
        serialized_runtime_env_context (str|None): If specified, the serialized
            runtime_env_context to start the client server in.

    Returns:
        ProcessInfo for the process that was started.
    r   r   workersrQ   zray.util.client.serverz
--address=r  r  z--mode=z--language=r3  r4  z!--serialized-runtime-env-context=r  r   z--runtime-env-agent-address=r  )r   __file__resolveparentsrU   r  r  r   SETUP_WORKER_FILENAMEr=   rT   r   NamePYTHONr5   r   r  PROCESS_TYPE_RAY_CLIENT_SERVER)r  r  r  r+   r,   r  r   rE  r  r  r  root_ray_dirr  ray_client_server_hostra   r  s                   r8   start_ray_client_serverr  	  s   D >>))++3A6Lj)]-P  2 W*(***(**+6hmHO4466
G  =;>;;<<< =;>;;<<<% 
P0NPP	
 	
 	
 g,--1111  SQ6OQQRRR$4  L r:   r   c                     | t          |           dk    rdS t          j                            | d                   }d|v S )zCheck if the command line belongs to a raylet process.

    Args:
        cmdline: List of command line arguments or None

    Returns:
        bool: True if this is a raylet process, False otherwise
    Nr   Fr   )r   rU   r  basename)r   rT   s     r8   r   r   Q	  sB     #g,,!++u!!'!*--Jz!!r:   )Tr<   r  )
NNFFFFFNNF)Nr   r   NN)NNr   r   NN)NNNNNNN)NNNNNNNNNNFFNNNNNNFNNr   r   FNNN)NNF)NNNNr   r   NF)NNNNNNr  N)xrp   collectionsr  r   rr   rK  r  r<  rU   r?  r   r  r   r=   r   pathlibr   typingr   r   r   r   filelockr   r   ray._private.ray_constantsr   r   ray._common.network_utilsr	   r
   r   r   r   r   &ray._private.resource_isolation_configr   ray._rayletr   r   ray.core.generated.common_pb2r   r   resourcer   _timeout
EXE_SUFFIXRUN_RAYLET_PROFILERr  r  dirnamer  rk  rd  r  r  r  r  rG  r  rn  r  rC  rI  "DASHBOARD_DEPENDENCY_ERROR_MESSAGEr#   r$   r%   rW   r]   	getLoggerr   r   
namedtupler(   r_   r9   r4   r6   boolrf   rm   rw   r   Popenr   r   r   r   r   r  r   r   r   r   r   r
  r  r  r  r  r&  r,  r/  r7  rp  r  r  r  r  r  r  ra  rA  rB  rD  r~  r  r  r   r?   r:   r8   <module>r     s%         				        				        



        - - - - - - - - - - - -       


 2 2 2 2 2 2 2 2 2              < ; ; ; ; ; J J J J J J 3 3 3 3 3 3 3 3 2 2 2 2 2 2 <7HHH|w..VVB
   7<<(A(ABBD$OO7??27??27??8+D+DEEFFlJ?? GLLt44  GLLfeUHh.C   feUE<*+D   gll8V-=>>W^^K88Bkkd GLL5:JZ:WXX  !gll8UEBB  # 0 ' - 
 $I   "D 
 
	8	$	$$k$	 	 	 T#Y      
" " "
 :>. ..!$.26. . . .b  P P P++ + I	+
 + + + +\@ @ @ @ @:# @ @ @8X# X X X Xv4 4 4
9Xc] 9 9 9 9.3 .(3- . . . .h ' ''#&' ' ' ' 'TH H H* * *R R RH H H. *./ /
/!#/c]/ / / /@ *.$' $'
$'!#$'$' $' $' $'Ns    &3    2
5 
5 
5 
5!2C !2C !2 !2 !2 !2H:5s :5Xc] :5t :5 :5 :5 :5z2 2 2+& +& +& +&d #'"'#((,(,` `#Y`` ` $	`
 
#` ` `  ` !` ` "V*%` "V*%` ` ` ` `F& & & &Z "&%)%)K KKK K 	K
 K K c]K c]K K K Kp !%%)%)M M~MM M 	M
 M M M M M 3-M M M M c]M c]M M M M`: : :2 %)$(!!%%)(,%)Q QQQ c]Q c]	Q
 Q SMQ SMQ TNQ Q c]Q !Q c]Q Q Q QT &*%),0)-$($((,)-15,0,0,059597;7;!%#'"'"&#al lll l 	l
 l l l l l l l l l l  !l" #l$ %l& 'l( )l*  7+l, c]-l. c]/l0 tCy)1l2 "#3l4 SM5l6 SM7l8 !9l: "#;l< "*#=l> %SM?l@ AlB ClD %SMElF %SMGlH &.c]IlJ &.c]KlL (0}MlN (0}OlP QlR SlT C=UlV WlX YlZ  [l\ $]l^ }_l` C=al l l l^> > >::: : 	:
 : : : : : : :z--- - 	-
 - - - - - - - -f '+(,XE XEXEXE smXE !	XE
 XE XE XE XE| &*%)(,!% $Q QQQ c]Q c]	Q
 !Q Q Q Q Q Q Q Q Qp "&!%$($(!%/348G GGG  G #	G
 #G SMG SMG G  (}G G %-SMG G G GT"c 3 " " " " " " "r:   