
    &`i0\             4       9   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z	d dlZd dlmZ d dlmZmZmZmZ d dlZd dlZd dlZd dlZd dlmc mc mZ d dlmc mZ d dlmc mZ d dl m!Z!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d d	l+m,Z,m-Z-m.Z. d d
l/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8m9Z9m:Z: d dl;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZH d dlImJZJ d dlKmLZL d dlMmNZN d dlOmPZP d dlQmRZR d dlSmTZT d dlUmVZV d dlWZW ejX        eY          ZZd Z[ ej\                     ej]        ddej^        e_ej`                   ej]        ddeja        e_ejb                   ejc                    d                                                 Zd eje                     ejf        dde_           ej]        dd de_d!"           ej]        d#d$degejh        d%&           ej]        d'degejh        d(&           ej]        d)ddd*+          eTd,                                                                                     Zid-ee_         fd.Zjd/ Zkd0 Zlede                                 ej]        d1de_d2"           ej]        d3d4ddd56          d7e_d8emfd9                                    Znede                                 ej]        d:de_d;"           ej]        d1de_d<"           ej]        d#egdd=ejo         d>?           ej]        d@dde_dAB           ej]        dCdde_ejp        dDE           ej]        dFdde_ejq        dGE           ej]        dHdde_dIB           ej]        dJdegdK"           ej]        dLdegd dM&           ej]        dNdegdOdP&           ej]        dQdegdRdS&           ej]        dTddUV           ej]        dWdegddX&           ej]        dYddegdZB           ej]        d[degd\"           ej]        d]degd^"           ej]        d_degd`"           ej]        daddbe_dc ejr                    ddk    rdendfz              ej]        dgdddh+           ej]        didemdjk           ej]        dldejs        dmn           ej]        dodegejh        dpt                    ejh                  &           ej]        dqegeju        drs           ej]        dtegddus           ej]        dvegddws           ej]        dxdddy+           ej]        dzde_d{"           ej]        d|de_d}"           ej]        d~de_d"           ej]        dddd+           ej]        ddd           ej]        ddde jv        d           ej]        ddddd           ej]        degdds           ej]        ddddd           ej]        de_ddd           ej]        dddd+           ej]        dddd+           ej]        ddddfe_d           ej]        ddddfe_d           ej]        ddemdk           ej]        ddddd           ej]        ddewd"           ej]        ddegd"           ej]        ddde_dB          e8eTd                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 Zxede                                 ej]        dddd           ej]        dddd          e8eTdemdegfd                                                            Zyede                                 ejf        dde_           ej]        ddegd"           ej]        ddegd"           ej]        dddd+           ej]        dddd+           ej]        ddddd+           ej]        dd de_d!"           ej]        d)ddd*+           ej]        dddd+           ej]        dddd+           ej]        dddd+          e8eTd                                                                                                                                                                         Zzede                                 ejf        dde_           ej]        ddddd+           ej]        ddddĬ+           ej]        dd de_d!"           ej]        ddddƬ+          e8eTdǄ                                                                                                 Z{ede                    dȦ           ejf        dde_           ej]        ddddd+           ej]        ddddʬ+           ej]        dd de_d!"          d˄                                                             Z|ede                                 ejf        dde_           ej]        dddegdά           ej]        dd de_d!"          e8dτ                                                             Z}ede                                 ejf        dde_           ej]        ddddѬ+           ej]        ddddӬ+           ej]        ddddլ+           ej]        dd de_d!"           ej]        d)ddd*+           ej]        ddddج           ej]        dd$ddegdڬۦ          e8eTd܄                                                                                                                                     Z~ede                                 ejf        dde_           ejf        dde_           ejf        dde_           ej]        dd de_d!"          e8d߄                                                                         Zede                                 ejf        dde_           ejf        dde_           ejf        dde_           ej]        dd de_d!"           ej]        ddddd          e8d                                                                                     Zede                    ddi           ejf        dde_           ej]        dddd+           ej]        ddddѬ+           ej]        dddd+           ej]        ddddլ+           ej]        dd de_d!"           ej]        d)ddd*+           ej]        dd$ddegdڬۦ           ejf        dde_           ej]        dde_d"           ejf        dd           ej]        dddd+           ej]        ddd񬄦          e8	 ddee_         fd                                                                                                                                                                                    Zede                                 ejf        dde_           ejf        dde_           ej]        dd ej        e<          dd&           ej]        dddd+           ej]        ddddѬ+           ej]        dddd+           ej]        ddddլ+           ej]        dd de_d!"           ej]        d)ddd*+           ej]        dd$ddegdڬۦ           ej]        dddd+          e8d                                                                                                                                                             Zede                                 ejf        dde_           ej]        dd de_d!"          d                                     Zede                                 ejf        dde_           ej]        dd de_d!"          d                                     Zede                                d             Zede                                d             Zede                                d             Zede                                d             Zede                                 ej]        d1de_d"          d                          Zede                                 ej]        d1de_d2"           ej]        d ej        ddg          dds           ej]        d ej        g d          dds           ej]        d	 ej        g d
          dds           ej]        ddemdd           ej]        dddd+           ej]        ddegdds          d                                                                                                 Zede                                 ej]        d1de_d2"           ej]        d3d4dddd          eTd7e_d8emfd                                                Zede                    dȦ           ej]        dddemddd           ej]        ddde_dd&           ej]        d ddd!+           ej]        d"ddd#+           ej]        d$ddd%+           ej]        d&ddd'+           ej]        d(ddd)+           ej]        d*d+de_dd,&          	 	 	 	 	 	 	 	 dd-emd.ee_         d/emd0emd1emd2emd3emd4ee_         fd5                                                                                                            Zede                                 ejf        dde_           ej]        d6d7de_d8"           ej]        d9d:de_dd;&           ej]        d<d=de_dd>&           ej]        d?d@de_ddA&           ej]        dBdCdemdddD           ej]        ddde_dd&           ej]        d ddd!+           ej]        d"dddE+           ej]        d$ddd%+           ej]        d&ddd'+           ej]        d(ddd)+           ej]        d*d+de_dd,&          	 	 	 	 	 	 	 	 	 	 	 	 	 ddee_         dFee_         dGee_         dHee_         dIee_         dJeem         d.ee_         d/emd0emd1emd2emd3emd4ee_         fdK                                                                                                                                                                        Zede                    dȦ           ej]        d1de_d2"          dL                         Zede                    dȦ           ej]        d1de_d2"           ej]        dMde_dN"           ej]        dOd ej        dP eNj                                        D                       dQ"           ej]        dRde_dS"           ej]        dTdegddU&          d7e_dVe_dWe_dXe_dYegf
dZ                                                                        Zede                    d[d\           ej]        dde_d]"           ej]        d^de_d_"          d`e_dae_dbdfdc                                    Zede                    ddd\           ej]        d1de_d2"           ej]        dede_df"           ej]        dgdddh+          di                                                 Zede                                 ej]        djdkdddl6           ej]        dmdnde_do"          e8dp                                                 Zede                    dȦ          dq             Z ej\        drs          dt             Zee                    dus          dv             Zee                    dws          dx             Zede                    dys           ej]        dzddd{+          d|                         Zed                    ei           ed                    en           ed                    ex           ed                    ey           ed                    ez           ed                    e~           ed                    e           ed                    e           ed                    e{           ed                    e|           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    e           ed                    eV           ed                    e           ed                    eRd}s           ed                    e           	 d d~lmZmZmZmZ ed                    eds           ed                    eds           ed                    eds           ed                    ed/s           n+# e$ r#ZeZn                    de            Y dZ[ndZ[ww xY w	 d dlmZ ed                    eds           n+# e$ r#ZeZn                    de            Y dZ[ndZ[ww xY w	 d dlmZ ed                    e           n+# e$ r#ZeZn                    de            Y dZ[ndZ[ww xY wd ZeYdk    r e             dS dS (      N)datetime)ListOptionalSetTuple)build_addressparse_address)	usage_lib)
load_class)ensure_token_if_auth_enabled)memory_summary) parse_node_labels_from_yaml_fileparse_node_labels_jsonparse_node_labels_string)format_returncodesetup_process_exit_logger)ResourceIsolationConfig)get_ray_client_dependency_errorparse_resources_json)add_click_logging_optionscf
cli_logger)RUN_ENV_TYPESattach_clustercreate_or_update_clusterdebug_statusexec_clusterget_cluster_dump_archiveget_head_node_ipget_local_dump_archiveget_worker_node_ips	kill_nodemonitor_clusterrsyncteardown_cluster)RAY_PROCESSES)FAKE_HEAD_NODE_ID)autoscaler_pb2)install_and_start_prometheus)symmetric_run)	PublicAPI)check_open_portsc                     dd l mc mc m} |                    |           }|r;|d         t
          j        k    r't          d|d          dt
          j                   d S d S )Nr   ray_versionz.Ray version mismatch: cluster has Ray version z but local Ray version is )ray._common.usage.usage_lib_commonusager
   get_cluster_metadataray__version__RuntimeError)
gcs_clientray_usage_libcluster_metadatas      g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/scripts/scripts.py_check_ray_versionr:   G   s    777777777777$99*EE 
,];sNN:.: :(+: :
 
 	

 
NN    z--logging-levelF)requireddefaulttypehelpz--logging-formatc                     t          j        |                                           }t          j        j                            ||           t          j        |           d S )N)format_tmpl)	logginggetLevelNameupperr3   _privateray_loggingsetup_loggerr   
set_format)logging_levellogging_formatlevels      r9   clirL   S   sU    "  !4!4!6!677EL))%@@@n555555r;   cluster_config_fileT)r<   r>   z--cluster-namez-nz%Override the configured cluster name.)r<   r>   r?   --portz-pz*The local port to forward to the dashboard)r<   r>   r=   r?   z--remote-portz&The remote port your dashboard runs onz--no-config-cachez'Disable the local cluster config cache.)is_flagr=   r?   c                 (   	 ||fg}t          j        d                    ||                     t          | |||           t          j        d           dS # t          $ r/}t          j        d                    |||                    dd}~ww xY w)z<Port-forward a Ray cluster's dashboard to the local machine.z]Attempting to establish dashboard locally at http://localhost:{}/ connected to remote port {})override_cluster_nameport_forwardno_config_cachez$Successfully established connection.zFailed to forward dashboard from remote port {1} to local port {0}. There are a couple possibilities: 
 1. The remote port is incorrectly specified 
 2. The local port {0} is already in use.
 The exception is: {2}N)clickechoformatr   	ExceptionClickException)rM   cluster_nameportremote_portrS   rR   es          r9   	dashboardr]   i   s    H;
 	
$fT;77	
 	
 	
 	".%+		
 	
 	
 	
 	
9:::::   "+ ,26$Q+G+G	
 

 	s   AA 
B"*BB	live_jobsc                 @   t           j        j                            dt          j                  }|D ]}|                    d          rt           j        j                            |t          j                  }t          j	        |          d         | vr1t           j        j        
                    |t          j                   t          d           d|t          d          d         z   }	 t           j        j                            |t          j                  }|rt          j	        |          }d
|v s
|d         | vr3t           j        j        
                    |t          j                    dS t          |d                   \  }}t           j        j                            |t#          |                     t           j        j        
                    |t          j                   t%          |             dS t'          j        d           )dS )zContinue active debugging session.

    This function will connect 'ray debug' to the right debugger
    when a user is stepping between Ray tasks.
    RAY_PDB_	namespaces   RAY_PDB_CONTINUEjob_idz.Continuing pdb session in different process...s   RAY_PDB_RAY_PDB_CONTINUE_NTexit_debuggerpdb_addressg      ?)r3   experimentalinternal_kv_internal_kv_listray_constantsKV_NAMESPACE_PDB
startswith_internal_kv_getjsonloads_internal_kv_delprintlenr	   utilrpdb_connect_pdb_clientintcontinue_debug_sessiontimesleep)r^   active_sessionsactive_sessiondatakeysessionhostrZ   s           r9   rw   rw      s'    &2DDm< E  O *      $$%899 	 #/@@-*H A  D z$):: ,=="m.L >    BCCCs3F/G/G/I/I JJC '3DD=#A E    "j..G&'11WX5Fi5W5W(4EE=+I F    !.w}/E!F!FJD$HM55dCIIFFF$0AA}'E B    +9555FF
3% 	      r;   c                     | dS | S )N  )ss    r9   none_to_emptyr      s    yrHr;   c           	          g }d t          |  D             | D ]C}|                    d                    fdt          |          D                                  D|S )z7Format a table as a list of lines with aligned columns.c                 @    g | ]}t          d  |D                       S )c              3   4   K   | ]}t          |          V  d S N)rr   ).0xs     r9   	<genexpr>z*format_table.<locals>.<listcomp>.<genexpr>   s(      ))SVV))))))r;   )max)r   cols     r9   
<listcomp>z format_table.<locals>.<listcomp>   s/    AAAc))S)))))AAAr;   z | c              3   V   K   | ]#\  }}d                      ||                   V  $dS )z{0:{1}}N)rV   )r   ir   	col_widths      r9   r   zformat_table.<locals>.<genexpr>   s;      UUTQy''9Q<88UUUUUUr;   )zipappendjoin	enumerate)tableresultliner   s      @r9   format_tabler      sz    FAAS%[AAAI 
 
JJUUUUYt__UUUUU	
 	
 	
 	
 Mr;   	--addressz#Override the address to connect to.-vz	--verbosez5Shows additional fields in breakpoint selection page.)r<   rO   r?   addressverbosec                    t          j        |           } t                              d|  d           t	          j        | d           t          j                            dd          dk    r2t          t          j        j         dt          j        j                    	 d
 t          j        j                                        D             }t#          |           t          j        j                            dt*          j                  }t          d           g }|D ]}t/          j        t          j        j                            |t*          j                            }|d         |v r|                    |           dt          j        j                            |t*          j                   t9          |d d	          }|rg dg}t;          |          D ]\  }}t=          j        |d                                        d          }	|                    tC          |          |	|d         |d         dz   tC          |d                   z   |d         |d         tE          |d                   |d         g           ng dg}t;          |          D ]}\  }}t=          j        |d                                        d          }	|                    tC          |          |	|d         |d         dz   tC          |d                   z   g           ~t;          tG          |                    D ]\\  }}
t          |
           |dk    rB||dz
           d         $                    d          st          ||dz
           d                    ]tK          d           }|d!k    rt                       tM          |          }t/          j        t          j        j                            ||         t*          j                            }tO          |d"                   \  }}t          j(        j)        *                    |tM          |                     )#z?Show all active breakpoints and exceptions in the Ray debugger.Connecting to Ray instance at .F)r   log_to_driver	RAY_DEBUG1legacya  NOTE: The distributed debugger https://docs.ray.io/en/latest/ray-observability/ray-distributed-debugger.html is now the default due to better interactive debugging support. If you want to keep using 'ray debug' please set RAY_DEBUG=legacy in your cluster (e.g. via runtime environment).Tc                 .    h | ]}|d          
|d         S )IsDeadJobIDr   )r   jobs     r9   	<setcomp>zdebug.<locals>.<setcomp>  s5     
 
 
 S]
L
 
 
r;   r`   ra   zActive breakpoints:rc   c                     | d         S )N	timestampr   )r|   s    r9   <lambda>zdebug.<locals>.<lambda>  s    D,= r;   )r}   reverse)indexr   Ray taskfilename:linenozTask IDz	Worker IDzActor IDzNode IDr   z%Y-%m-%d %H:%M:%S	proctitlefilename:linenotask_id	worker_idactor_idnode_id)r   r   r   r      	tracebackzNoneType: Nonez2Enter breakpoint index or press enter to refresh: r   rf   )+services%canonicalize_bootstrap_address_or_dieloggerinfor3   initosenvirongetrq   coloramaForeYELLOWRESETrE   statejobsrw   rg   rh   ri   rj   rk   rn   ro   rm   r   rp   sortedr   r   utcfromtimestampstrftimestrr   r   rl   inputrv   r	   rs   rt   ru   )r   r   r^   rz   sessions_datar{   r|   r   r   dater   inpr   r~   r   rZ   s                   r9   debugr      s    <WEEG
KK;;;;<<<HWE2222	z~~k3''833}# T T
 ?Gm>QT T	
 	
 	
Z?
 
$'L$6$;$;$=$=
 
 
	 	y)))*6HH-"@ I 
 
 	#$$$- 	 	N: ,=="m.L >   D H~**$$T**** ,=="m.L >     ==t
 
 
  +		 	 	E %]33  40k1BCCLL'  A[)Z(3.T(^1D1DDY[)%d:&677Y	   	$ KJJKE$]33  40k1BCCLL'  A[)Z(3.T(^1D1DD	    !e!4!455 	9 	9GAt$KKKAvvmAE2;?JJ  v mAE*;7888HII"99GGGHHEj ,==#E*m6T >   G
 'w}'=>>JD$HM--dCII>>>uZ?r;   z--node-ip-addresszthe IP address of this nodezthe address to use for Rayz?the port of the head ray process. If not provided, defaults to z:; if port is set to 0, we will allocate an available port.)r>   r<   r?   z--node-namezUthe user-provided identifier or name for this node. Defaults to the node's ip_address)r<   hiddenr>   r?   --redis-usernamez2If provided, secure Redis ports with this username)r<   r   r>   r=   r?   --redis-passwordz2If provided, secure Redis ports with this password--redis-shard-portszGthe port to use for the Redis shards other than the primary Redis shardz--object-manager-portz/the port to use for starting the object managerz--node-manager-portz-the port to use for starting the node managerz--min-worker-porti'  zZthe lowest port number that workers will bind on. If not set, random ports will be chosen.z--max-worker-portiN  z`the highest port number that workers will bind on. If set, '--min-worker-port' must also be set.z--worker-port-listzsa comma-separated list of open ports for workers to bind on. Overrides '--min-worker-port' and '--max-worker-port'.)r<   r?   z--ray-client-server-portzjthe port number the ray client server binds on, default to 10001, or None if ray[client] is not installed.z--memoryzzThe amount of memory (in bytes) to make available to workers. By default, this is set to the available memory on the node.z--object-store-memoryzThe amount of memory (in bytes) to start the object store with. By default, this is 30% of available system memory capped by the shm size and 200G but can be set higher.z
--num-cpuszthe number of CPUs on this nodez
--num-gpuszthe number of GPUs on this nodez--resources{}zHA JSON serialized dictionary mapping resource name to resource quantity.Windowsa  

Windows command prompt users must ensure to double quote command line arguments. Because
JSON requires the use of double quotes you must escape these arguments as well, for
example:

    ray start --head --resources="{\"special_hardware\":1, \"custom_label\":1}"

Windows powershell users need additional escaping:

    ray start --head --resources="{\""special_hardware\"":1, \""custom_label\"":1}"
r   --headz'provide this argument for the head node--include-dashboardz4provide this argument to start the Ray dashboard GUI)r=   r>   r?   z--dashboard-hostzthe host to bind the dashboard server to, either localhost (127.0.0.1) or 0.0.0.0 (available from all interfaces). By default, this is 127.0.0.1)r<   r=   r?   z--dashboard-portz8the port to bind the dashboard server to--defaults to {}z--dashboard-agent-listen-portz4the port for dashboard agents to listen for http on.)r>   r=   r?   z--dashboard-agent-grpc-portz4the port for dashboard agents to listen for grpc on.z--runtime-env-agent-portzBThe port for the runtime environment agents to listen for http on.--blockzprovide this argument to block forever in this command.Process exit logs will be saved to ray_process_exit.log in the logs directory.z--plasma-directoryz.object store directory for memory mapped filesz--object-spilling-directoryzThe path to spill objects to. This path will also be used as the fallback directory when the object store is full of in-use objects and cannot spill.z--autoscaling-configz-the file that contains the autoscaling configz--no-redirect-outputz5do not redirect non-worker stdout and stderr to filesz
--temp-dirz_manually specify the root temporary dir of the Ray process, only works when --head is specified)r=   r?   z--system-configz'Override system configuration defaults.)r=   r   r>   r?   z--enable-object-reconstructionzDSpecify whether object reconstruction will be used for this cluster.)rO   r=   r   r?   z--metrics-export-portzDthe port to use to expose Ray metrics through a Prometheus endpoint.z--no-monitorzIIf True, the ray autoscaler monitor for this cluster will not be started.)rO   r   r=   r?   z--tracing-startup-hookzThe function that sets up tracing with a tracing provider, remote span processor, and additional instruments. See docs.ray.io/tracing.html for more info.)r>   r   r=   r?   z--ray-debugger-externalzwMake the Ray debugger available externally to the node. This is only safe to activate if the node is behind a firewall.z--disable-usage-statsz5If True, the usage stats collection will be disabled.z--labelsza string list of key-value pairs mapping label name to label value.These values take precedence over conflicting keys passed in from --labels-file.Ex: --labels "key1=val1,key2=val2")r<   r   r=   r>   r?   z--labels-filezNa path to a YAML file containing a dictionary mapping of label keys to values.z--include-log-monitorzIf set to True or left unset, a log monitor will start monitoring the log files of all processes on this node and push their contents to GCS. Only one log monitor should be started per physical host to avoid log duplication on the driver process.z--enable-resource-isolationa  Enable resource isolation through cgroupv2 by reserving memory and cpu resources for ray system processes. To use, only cgroupv2 (not cgroupv1) must be enabled with read and write permissions for the raylet. Cgroup memory and cpu controllers must also be enabled.)r<   rO   r=   r?   z--system-reserved-cpuah   The number of cpu cores to reserve for ray system processes. Cores can be fractional i.e. 1.5 means one and a half a cpu core. By default, the value will be atleast 1 core, and at maximum 3 cores. The default value is calculated using the formula min(3.0, max(1.0, 0.05 * num_cores_on_the_system)) This option only works if --enable_resource_isolation is set.z--system-reserved-memorya1  The amount of memory (in bytes) to reserve for ray system processes. By default, the value will be atleast 500MB, and at most 10GB. The default value is  calculated using the formula min(10GB, max(500MB, 0.10 * memory_available_on_the_system)) This option only works if --enable_resource_isolation is set.z--cgroup-patha[  The path for the cgroup the raylet should use to enforce resource isolation. By default, the cgroup used for resource isolation will be /sys/fs/cgroup. The process starting ray must have read/write permissions to this path.  Cgroup memory and cpu controllers be enabled for this cgroup. This option only works if enable_resource_isolation is True.c-                   K d}-| d}-t          j        |           } t          |t          t                    }	 t          |'          }.nb# t          $ rU}/t          j        dt	          j        d|'           t          |/          t	          j        d                     Y d}/~/nd}/~/ww xY w	 t          |&          }0n# t          $ r}/	 t          |&          }0t          j        dt          d	           n[# t          $ rN t          j        d
t	          j        d|&           t          |/          t	          j        d                     Y nw xY wY d}/~/nd}/~/ww xY wi |.|0}1|r|st          j        d| d           d}t!          |)|,|*|+          }2|sdnd}3t#                      du }4|4r|d}t%          j        j        j        di d| d|r|n| d|	d|
d|d|d|d|d|d|d|d|d|3d|d|d |d!|1d"|d#|d$|d%dd&|d'|d(|d)|d*|d+|d,|d-|d.| d/|!d0|"d1|#d2|$d3|(d4|2}5t,          j        t0          j        v r3 t5          t0          j        t,          j                           |5|           |r"|%rt7          j        d           t7          j        d5           t          j                     |t,          j        }|5j         J |5j!        J ||5_!        t0          j        "                    d6          rd7tF          i|5_$        tJ          j&        t0          j        v r<d8t0          j        v r.d9t0          j        v r t0          j        '                    d:d;           d}6|&|$|(                    d<          }tS          |          }6|:t          j        d=t	          j        d>          t	          j        d?          |           |(                    d<          }7tS          |7          d@k    r|7*                    |7dA                    |5+                    |7B           tS          |7          d@z
  }6|t,          j,        k    r:t          j        dCt	          j        dD          t	          j        d>                     |t,          j-        k    r:t          j        dEt	          j        dF          t	          j        d>                     |5+                    t          j.                    G           t          j/        dH|5j0                   |5+                    ||6dI           |Ztc          |5j0        |          }8t          j2        |          }9|8|9k    r+|9t          j3                    v rti          dJ|8 dK|9 dL          tk          |dM           t$          j        j6        7                    |5d||N          }:|:j8        }9t          j                     dO};t          j9        dPtS          |;          z             t          j9        |;           t          j9        dPtS          |;          z             t          j                     t          j:        dQ          5  |:j;        dR         }<t,          j<        rXt          j=        dS           t          j=        t	          j        dT          t,          j>        rdUt,          j?         dVndW|9           t          j                     t          j=        dX           t          j@                    5  t          j=        dYt	          jA        dZ                     t          j=        d[t	          jA        d\          |-r@d]B                    t	          jA        d^          t	          jC        d_| z   d_z                       ndW           ddd           n# 1 swxY w Y   |<rt          j                     t          j=        d`           t          j=        t	          j        da          |<           t          j                     t          j=        db           t          j=        dc           t          j                     t          j=        dd           t          j=        t	          j        de                     t          j                     t          j=        df           t          j=        dgB                    t	          j        dh                               |<rt          j                     t          j=        di           t          j=        dgB                    t	          j        |<                               t          j                     t          j=        t	          jD        dj                     ddd           n# 1 swxY w Y   |9|5_E        nt,          j<        sNt          j        dkt	          j        t,          j?                             t          dlt,          j?         dm          |It          j        dnt	          j        d>          t	          j        do                     t          dp          |||dq}=|=F                                D ]S\  }>}?|?t          j        drt	          j        |>          t	          j        do                     t          |> ds          t          jH        ||t          }9|9Mt          j        dut	          j        d>          t	          j        |                     t          dv| dw          |9|5_E        |5+                    t          j.        |9          G           t          j/        dH|5j0                   tk          |dM           t$          j        j6        7                    |5d||N          }:|:I                                }|:J                                 t          j                     dO};t          j9        dPtS          |;          z             t          j9        |;           t          j9        dPtS          |;          z             t          j                     t          j=        dd           t          j=        t	          j        de                     t          jK                     |5jE        J t$          j        jL        M                    |5jE        |           |r.|:N                                }@t0          jO        P                    |@dx          }At          j                     t          j:        t	          j        dy                    5  t          j=        dz           t          j=        d{           t          j=        d|t	          j        |A                     t          jK                     	 t          |A          }Bn.# t          $ r!}/t          j        d}|/           d}BY d}/~/nd}/~/ww xY wddd           n# 1 swxY w Y   	 t          jS        d@           |:T                                }CdAt          jV        d~t          jV        z  dt          jV        z   gKKfd|CD             }DtS          |D          dAk    rt          j                     t          jW        d           g }Et          j@                    5  |DD ]\  }F}Gt          jW        dt	          j        t          |F                    dt          |GjX                  i           t          |Gdd          }Ht          |H          }I|E*                    d|F d|I d           	 ddd           n# 1 swxY w Y   	 ddP                    |E          z   }J|BW                    d|J           n,# t          $ r}/t          j        d|/           Y d}/~/nd}/~/ww xY wt          j                     t          jW        d           |:[                    dd           t1          j\        d@           dS )z2Start Ray processes manually on the local machine.FNTz_The file at `{}` is not a valid YAML file, detailed error: {} Valid values look like this: `{}`z--labels-file=z)--labels-file='gpu_type: A100
region: us'zppassing node labels with `--labels` in JSON format is deprecated and will be removed in a future version of Ray.   )
stacklevelzc`{}` is not a valid string of key-value pairs, detailed error: {} Valid values look like this: `{}`z	--labels=z--labels="key1=val1,key2=val2"z`--temp-dir=z` option will be ignored. `--head` is a required flag to use `--temp-dir`. temp_dir is only configurable from a head node. All the worker nodes will use the same temp_dir as a head node. )enable_resource_isolationcgroup_pathsystem_reserved_cpusystem_reserved_memoryi'  node_ip_address	node_namemin_worker_portmax_worker_portworker_port_listray_client_server_portobject_manager_portnode_manager_portmemoryobject_store_memoryredis_usernameredis_passwordredirect_outputnum_cpusnum_gpus	resourceslabelsautoscaling_configplasma_directoryobject_spilling_directory
huge_pagestemp_dirinclude_dashboarddashboard_hostdashboard_portdashboard_agent_listen_portmetrics_agent_portruntime_env_agent_port_system_configenable_object_reconstructionmetrics_export_port
no_monitortracing_startup_hookray_debugger_externalinclude_log_monitorresource_isolation_configrL   RAY_FAKE_CLUSTER RAY_OVERRIDE_NODE_ID_FOR_TESTINGRAY_CLOUD_INSTANCE_IDRAY_NODE_TYPE_NAMERAY_enable_autoscaler_v2r   ,zjSpecifying {} for external Redis address is deprecated. Please specify environment variable {}={} instead.r   RAY_REDIS_ADDRESSr   r   )external_addresseszj`{}` should not be specified as empty string if external Redis server(s) `{}` points to requires username.r   zj`{}` should not be specified as empty string if external redis server(s) `{}` points to requires password.r   )r   zLocal node IP)redis_shard_portsnum_redis_shardsredis_max_clientszRay is trying to start at z, but is already running at zQ. Please specify a different port using the `--port` flag of `ray start` command.)create_token_if_missing)headshutdown_at_exitspawn_reaperzRay runtime started.-z
Next steps	webui_urlz,To add another node to this Ray cluster, runz {} ray start --address='{}' z=1r   zTo connect to this Ray cluster:z{} rayimportzray{}init({})r   z_node_ip_address{}{}='z+To submit a Ray job using the Ray Jobs CLI:zZ  RAY_API_SERVER_ADDRESS='http://{}' ray job submit --working-dir . -- python my_script.pyzYSee https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html z?for more information on submitting Ray jobs to the Ray cluster.z!To terminate the Ray runtime, runz
  ray stopz&To view the status of the cluster, usez  {}z
ray statusz0To monitor and debug Ray, view the dashboard at z]If connection to the dashboard fails, check your firewall settings and network configuration.zMulti-node Ray clusters are not supported on Windows and OSX. Restart the Ray cluster with the environment variable `{}=1` to proceed anyway.zuMulti-node Ray clusters are not supported on Windows and OSX. Restart the Ray cluster with the environment variable `z=1` to proceed anyway.z>`{}` is a required flag unless starting a head node with `{}`.r   zI`--address` is a required flag unless starting a head node with `--head`.)rN   r   r   z@`{}` should only be specified when starting head node with `{}`.z@ should only be specified when starting head node with `--head`.)r   z$Cannot canonicalize address `{}={}`.z'Cannot canonicalize address `--address=z`.zray_process_exit.logr   zAThis command will now block forever until terminated by a signal.zRunning subprocesses are monitored and a message will be printed if any of them terminate unexpectedly. Subprocesses exit with SIGTERM will be treated as graceful, thus NOT reported.z&Process exit logs will be saved to: {}z&Failed to init process exit logger: {}   c                 0    g | ]\  }}|j         v||fS r   )
returncode)r   process_typeprocessexpected_return_codess      r9   r   zstart.<locals>.<listcomp>  s;     # # #)L'%-BBB w'BBBr;   z*Some Ray subprocesses exited unexpectedly:r   z	exit code)_tagsr  z  z [exit code=]z+Some Ray subprocesses exited unexpectedly:

z%sz$Failed to write process exit log: {}z#Remaining processes will be killed.)check_aliveallow_gracefulr   )]r   resolve_ip_for_localhostr   r   r   r   rW   abortboldr   r   r   warningswarnDeprecationWarningwarningr   r   r3   rE   	parameter	RayParamsrj   RAY_START_HOOKr   r   r   r
   #set_usage_stats_enabled_via_env_varshow_usage_stats_promptnewlineDEFAULT_PORT
redis_portgcs_server_portr   r'   env_varsusage_constantKUBERAY_ENV
setdefaultsplitrr   r   update_if_absentREDIS_DEFAULT_USERNAMEREDIS_DEFAULT_PASSWORDget_node_ip_addresslabeled_valuer   r   find_bootstrap_addressfind_gcs_addressesConnectionErrorr   nodeNoder   successgroupaddress_infoENABLE_RAY_CLUSTERrq   IS_WINDOWS_OR_OSXENABLE_RAY_CLUSTERS_ENV_VARindentedmagentarV   yellow
underlinedgcs_addressitems
ValueErrorcanonicalize_bootstrap_addressget_temp_dir_pathcheck_version_infoflushutilswrite_ray_addressget_logs_dir_pathpathr   r   rx   ry   dead_processessignalSIGTERMerrorr  getattrr   kill_all_processes_exit)Lr   r   rZ   r   r   r   r  r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   dashboard_agent_grpc_portr   blockr   r   r   no_redirect_outputr   system_configr   r   r  r  r  disable_usage_statsr   labels_filer  r   r   r   r   include_node_ip_addresslabels_from_filer\   labels_from_stringlabels_dictr  r   has_ray_client
ray_paramsr  r  default_addressbootstrap_addressrE  startup_msgdashboard_urlhead_only_flagsflagvallogs_dirprocess_exit_log_pathprocess_exit_loggerdeceasedunexpected_deceasedlines_for_filer   r!  rcrc_strfile_msgr"  sL                                                                              @r9   startr  \  s   @ $""&";OLL$Y
B??I	
;KHH 
 
 
0G2[2233FFG@AA	
 	
 	
 	
 	
 	
 	
 	

5f==   	!7!?!?MM"	      	 	 	4,F,,--A899    	& =%<);<K  O8 O O O	
 	
 	
  7";/5	! ! ! #5>dd$O 566$>N '08!&'1 % % %'%(=))o% (% (	%
 *)%  65% 0/% ,+% v% 0/% &~% &~% (% % %  )!%" {#%$ .-%%& *)'%( #<";)%* 5+%, -%. ,+/%0 &~1%2 &~3%4 %@$?5%6 547%8  659%: %};%< &B%A=%> 0/?%@ :A%B 21C%D 43E%F 0/G%H #<";I%JN #rz11<
2:m:;<<ZNNN Y  	A9%@@@)d3333< -D $,,,)111%)
":>>,-- 	24E#J
 &"*44'2:55$
22 J!!"<cBBB(W_ 1 7 7 < <  ##455 E$$+,,   ")s!3!3 %&&!++"))*<Q*?@@@'';M'NNN"#566:!EEE"  G.//GK((   !EEE"  G.//GK((   	##H4P4R4R#SSS *2LMMM 	##/-" 	$ 	
 	
 	
 ?+J,FMMO ( ? I I#444%)D)F)FFF &4 4 41B4 4 4   	%]ERRRR| %%TE & 
 
 !L 	,3[!1!11222;'''3[!1!11222l++ H	 H	 -k:M/ 
 !OPPP  G:;;$6EAEEEE%      >???$&& 
 
 2:h+?+?@@@ #JsOO /*11
333H33N)O)O     
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  "$$$ !NOOO G1 
 "   "$$$ 2    U      @AAARW\22333   EFFFV]]27<+@+@AABBB "$$$ !STTT MM..    "$$$ M1   EH	 H	 H	 H	 H	 H	 H	 H	 H	 H	 H	 H	 H	 H	 H	R "3
 / 	% ABB	   !=     ?P$$!!  
 +   #4#4
 

 )..00 	 	ID#{R!!  
  ! ! !   %Ch
 
 
 $6$$    
 TTTTUUU!2
 	##$89JKK 	$ 	
 	
 	
 	 *2LMMM 	%]ERRRR| %%UU & 
 
 ))++ 	!!!,3[!1!11222;'''3[!1!11222<===..///!---L(()?JJJ J))++ "X7M N Nbgi0011 	+ 	+S   T  
 8"'BW:X:X   +&?@U&V&V## + + +"#KQOOO&*######+	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+&3	JqMMM**,,H V^#fn$	%!# # # #-5# # #
 &''!++"$$$ !MNNN!#(** 	X 	X1D X X-g"( GC$5$566#.G4F0G0G"H   
 %WlDAA!22!6!6&--.V<.V.VV.V.V.VWWWWX	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	XRF))N334  (--dH====  R R R&'MqQQQQQQQQR "$$$ !FGGG ''E%'PPPg3	/J Js   A 
B'AB""B'+B; ;
E+C21E2AE
E	E

EEB,f
B^(f(^,	,f/^,	0Gfff<A#w* v0/w*0
w:ww*ww**w.1w.&B}}	}.}= =
~&~!!~&z-f--forcez1If set, ray will send SIGKILL instead of SIGTERM.)rO   r?   z-gz--grace-period   zThe time in seconds ray waits for processes to be properly terminated. If processes are not terminated within the grace period, they are forcefully terminated after the grace period. forcegrace_periodc                    t           j                            d          d}d}g }dt          dt          dt
          t                   dt          t          t          t
          t          j	                 f         ffd}t          }|d         d         dk    sJ |d	         d         d
k    sJ |d         g|dd	         |d	         gg}|D ]B} || |t          |          z  |          \  }	}
}||	z  }||
z  }|                    |           C|dk    rt          j        d           n|||k    rt          j        d|           n`t          j        d| d| d| dt#          j        d           d| d           t          j        dt#          j        d           d           t&          j        j                                         dS )z1Stop Ray processes manually on the local machine.linuxr   r  r  processes_to_killreturnc           
      .   g }t          j        ddg          D ]T}	 |                    ||                                |                                f           @# t           j        $ r Y Qw xY wg }|D ]\  }}|rHrFt          |          dk    r3d                    dt          |          |          }t          |          g }	|D ]9}
|
\  }}}|r|nt          j
        |          }||v r|	                    |
           :|	D ]p\  }}}t          t          j
        |                    }	 | r|                                 n|                                 | r;t          j        dt!          j        |          t!          j        d                     n:t          j        dt!          j        |          t!          j        d                     |                    |           # t           j        $ r+ t          j        d	t!          j        |                     Y t           j        t(          f$ r@}t          j        d
t!          j        |          t          |                     Y d}~jd}~ww xY wt          j        |d          \  }}||z   }t          |          t/                      fd}t          j        |||          \  }}t          |          }|D ]}|                                 t          j        |d           ||fS )as  Find all processes from `processes_to_kill` and terminate them.

        Unless `force` is specified, it gracefully kills processes. If
        processes are not cleaned within `grace_period`, it force kill all
        remaining processes.

        Returns:
            total_procs_found: Total number of processes found from
                `processes_to_kill` is added.
            total_procs_stopped: Total number of processes gracefully
                stopped from `processes_to_kill` is added.
            procs_not_gracefully_killed: If processes are not killed
                gracefully, they are added here.
        namecmdline   zVThe filter string should not be more than {} characters. Actual length: {}. Filter: {}zKilled `{}` {} z(via SIGKILL)z#Send termination request to `{}` {}z(via SIGTERM)z5Attempted to stop `{}`, but process was already dead.z"Could not terminate `{}` due to {}Nr   timeoutc                                          |            t          j        t                     d dd           d S )N/z	 stopped.)end)addr   rq   rr   )proc
gone_procstotal_founds    r9   on_terminatez.stop.<locals>.kill_procs.<locals>.on_terminate(  sH    NN4   JHH+HHHdSSSSSSr;   )r  callbackr   )psutilprocess_iterr   r  r  Errorrr   rV   rS  
subprocesslist2cmdliner   kill	terminater   r   r   r*  dimmedNoSuchProcessOSErrorr_  
wait_procsset)r  r  r  process_infosr  stoppedkeywordfilter_by_cmdmsgfound	candidateproc_cmd	proc_argscorpusproc_stringexaliveprocs_to_killr  total_stoppedr  r  is_linuxs                       @@r9   
kill_procszstop.<locals>.kill_procs  s   " '(;<< 	 	D$$dDIIKK%HIIII<    &7 4	 4	"G] & &c'llR.?.?@&S\\733  !oo%E* , ,	,5)h	 -UHH:3J93U3U  f$$LL+++-2 # #)h	!*"9)"D"DEE! )		
 ((( "*-GK00Io66    #*AGK00Io66   NN4((((+   &O,,     g.   $<,,B       =#N  *7A>>>%-(( UU
	T 	T 	T 	T 	T 	T  *<,
 
 
 G  	 	DIIKKKK%++++M500s0   <AA.-A..B7G&&6I4I445I//I4rayletr  
gcs_serverr   z&Did not find any active Ray processes.zStopped all {} Ray processes.zStopped only z out of z' Ray processes within the grace period z seconds. Set `r   z+` to see more details. Remaining processes z will be forcefully terminated.zYou can also use `r  zn` to forcefully terminate processes or set higher `--grace-period` to wait longer time for proper termination.N)sysplatformrl   boolrv   r   r   r   r  Processr&   rr   extendr   rq   rG  r.  r   r*  r3   r0   rX  reset_ray_address)r  r  total_procs_foundtotal_procs_stoppedprocs_not_gracefully_killedr  r  bucketsbucketr  r  r  r  s               @r9   stopr    s^   * |&&w//H"$g1g1#&g1;?9g1	sCfn--	.g1 g1 g1 g1 g1 g1` & Q"h....
 R #|3333!!$%'82'>ARSUAV@WXG 2 2 *
5,W2Mv V VwU"w&#**51111 AABBBB"333>@STTTT1 3 1 1=N 1 19E1 11 1 (C1 1 1   &RWY%7%7 & & &   K'')))))r;   z--min-workersz>Override the configured min worker node count for the cluster.z--max-workersz>Override the configured max worker node count for the cluster.--no-restartzaWhether to skip restarting Ray services during the update. This avoids interrupting running jobs.--restart-onlyzcWhether to skip running setup commands and only restart Ray. This cannot be used with 'no-restart'.z--yesz-yzDon't ask for confirmation.z--redirect-command-outputz-Whether to redirect command output to a file.z&--use-login-shells/--use-normal-shellszRay uses login shells (bash --login -i) to run cluster commands by default. If your workflow is compatible with normal shells, this can be disabled for a better user experience.c                 &   |
rt          j        d           |s|rOt          j        ||k    dt	          j        d          t	          j        d                     ||k    s
J d            t          j                            |           j	        dv r	 t          j
                            | d          }|                                }|                     d	          d
         }t          |d          5 }|                    |           ddd           n# 1 swxY w Y   |} nW# t          j        j        $ r@}t          j        dt'          |                     t          j        d           Y d}~nd}~ww xY wt)          | |||||||||	
  
         dS )zCreate or update a Ray cluster.Fz`{}` is incompatible with `{}`.r  r  zACannot set both 'restart_only' and 'no_restart' at the same time!)httphttps   r  r  r  wbNr   z5Could not download remote cluster configuration file.
config_fileoverride_min_workersoverride_max_workers
no_restartrestart_onlyyesrQ   rS   redirect_command_outputuse_login_shells)r
   r2  r   doassertr   r*  urllibparseurlparseschemerequesturlopenreadr<  openwriter_  	HTTPErrorr.  r   r   )rM   min_workersmax_workersr  r  r  rY   rS   r  r  rg  responsecontent	file_namefr\   s                   r9   upr  l  s   l  =5e<<< 	Oz 	OJ&-G$%%GN##		
 	
 	
 J&&&N '&& |0118<MMM		X~--.A1-MMHmmooG+11#66r:Ii&& !!   ! ! ! ! ! ! ! ! ! ! ! ! ! ! !"+|% 	X 	X 	XtSVV,,,VWWWWWWWW	X '((!*' 7)     s=   A D! 1DD! DD! DD! !E556E00E5z--workers-onlyzOnly destroy the workers.z--keep-min-workersz=Retain the minimal amount of workers specified in the config.c                 ,    t          | ||||           dS )zTear down a Ray cluster.N)r%   )rM   r  workers_onlyrY   keep_min_workerss        r9   downr    s,    2 S,>N    r;   )r   z--hardzTerminates the node via node provider (defaults to a 'soft kill' which terminates Ray but does not actually delete the instances).c           	      T    t          j        dt          | |||          z              dS )z3Kills a random Ray node. For testing purposes only.zKilled node with IP N)rT   rU   r"   )rM   r  hardrY   s       r9   kill_random_noder    s9    * 
J+>T<!X!XX    r;   z--linesd   zNumber of lines to tail.c                 (    t          | ||           dS )z+Tails the autoscaler logs of a Ray cluster.N)r#   )rM   linesrY   s      r9   monitorr  !  s     '=====r;   z--startzStart the cluster if needed.--screenzRun the command in screen.--tmuxzRun the command in tmux.z--newz-NzForce creation of a new screen.z--port-forwardzCPort to forward. Use this multiple times to forward multiple ports.)r<   multipler>   r?   c           
      f    d t          |          D             }t          | |||||||           dS )z3Create or attach to a SSH session to a Ray cluster.c                     g | ]}||fS r   r   r   rZ   s     r9   r   zattach.<locals>.<listcomp>_      @@@TT4L@@@r;   )rS   newrR   N)listr   )rM   r  screentmuxrY   rS   r  rR   s           r9   attachr  3  sZ    X A@T,-?-?@@@L'!	 	 	 	 	 	r;   sourcetargetc                 .    t          | |||d           dS )z+Download specific files from a Ray cluster.T)r  N)r$   )rM   r  r  rY   s       r9   
rsync_downr  l  s$     

vv|$GGGGGGr;   z--all-nodesz-Az'Upload to all nodes (workers and head).)rO   r<   r?   c                 \    |rt          j        d           t          | |||d|           dS )z'Upload specific files to a Ray cluster.a  WARNING: the `all_nodes` option is deprecated and will be removed in the future. Rsync to worker nodes is not reliable since workers may be added during autoscaling. Please use the `file_mounts` feature instead for consistent file sync in autoscaling clustersF)r  	all_nodesN)r   r.  r$   )rM   r  r  rY   r  s        r9   rsync_upr  }  s\    *  
O	
 	
 	
 
     r;   ignore_unknown_options)context_settingsz--stopz4Stop the cluster after the command finishes running.zRun the command in a screen.script--argsz4(deprecated) Use '-- --arg1 --arg2' for script args.script_argsr  )nargsz--extra-screen-argszif screen is enabled, add the provided args to it. A useful example usage scenario is passing --extra-screen-args='-Logfile /full/path/blah_log.txt' as it redirects screen output also to a custom fileextra_screen_argsc                 T   t          j        |o| dt          j        d          t          j        d                     t          j        |
o|	 dt          j        d          t          j        d          t          j        d                     |r|r
J d            |
r|	r
J d	            ||st          j        d           |	rtt          j        dt          j        d                     t          j        dt          j        d          t          j        d                     t          j                     |r/|rt          j        d           t          | d
d
ddd||dd
  
         t          j                            |          }t          j                            d|          }t          | ||||d           d|g}|
r|t          |
          z  }n|	||	gz  }d t          |          D             }d                    |          }t!          | |d|||d||||           d
S )zUploads and runs a script on the specified cluster.

    The script is automatically synced to the following location:

        os.path.join("~", os.path.basename(script))

    Example:
        ray submit [CLUSTER.YAML] experiment.py -- --smoke-test
    z`{}` and `{}` are incompatible.r  r  z@`{0}` and `{1}` are incompatible. Use only `{1}`.
Example: `{2}`r  z-- <args ...>z(ray submit script.py -- --arg=123 --flagz+Can specify only one of `screen` or `tmux`.z%Use -- --arg1 --arg2 for script args.NzATo use extra_screen_args, it is required to use the --screen flagz5`{}` is deprecated and will be removed in the future.z Use `{}` instead. Example: `{}`.FTr  ~)rS   r  pythonc                     g | ]}||fS r   r   r  s     r9   r   zsubmit.<locals>.<listcomp>9  r  r;   r  docker)
cmdrun_envr  r  r  r  rQ   rS   rR   r	  )r   r  r   r*  r)  r.  r4  r
   r2  r   r   r[  basenamer   r$   r  r   )rM   r  r  r  r  rY   rS   rR   r  argsr  rg  r	  r  command_partsr  s                   r9   submitr    s   d _)


	   !T"K

  
:;;   O4OO"OOO NNN'NNN%%%O	
 	
 	
  	CRWXEVEV	
 	
 	
 	.GO$$G>??	
 	
 	

 	 
 	A9%@@@ +!%!%".+$)!	
 	
 	
 	
 Wf%%FW\\#v&&F	'    v&M  k***		$@@T,-?-?@@@L
((=
!
!C*'!+     r;   r  z	--run-envautozChoose whether to execute this command in a container or directly on the cluster head. Only applies when docker is configured in the YAML.c                     d t          |	          D             }	|r|
rt          j        d           t          | |||||||||	d           dS )z+Execute a command via SSH on a Ray cluster.c                     g | ]}||fS r   r   r  s     r9   r   zexec.<locals>.<listcomp>  r  r;   FT)
r  r  r  r  r  r  rQ   rS   rR   _allow_uninitialized_stateN)r  r
   r2  r   )rM   r  r  r  r  r  r  rY   rS   rR   rg  s              r9   execr  J  s    D A@T,-?-?@@@L A 	A9%@@@*'!#'     r;   c                 J    t          j        t          | |                     dS )z)Return the head node IP of a Ray cluster.N)rT   rU   r   )rM   rY   s     r9   get_head_ipr    s&     
J 3\BBCCCCCr;   c                 t    t          | |          }t          j        d                    |                     dS )z/Return the list of worker IPs of a Ray cluster.r%  N)r!   rT   rU   r   )rM   rY   
worker_ipss      r9   get_worker_ipsr    s6     %%8,GGJ	Jtyy$$%%%%%r;   c                  N    t          j        d           t          d           dS )zDisable usage stats collection.

    This will not affect the current running clusters
    but clusters launched in the future.
    FenabledzgUsage stats disabled for future clusters. Restart any current running clusters for this to take effect.Nr
   "set_usage_stats_enabled_via_configrq   r   r;   r9   rg  rg    s:     0????		H    r;   c                  N    t          j        d           t          d           dS )zEnable usage stats collection.

    This will not affect the current running clusters
    but clusters launched in the future.
    Tr   zfUsage stats enabled for future clusters. Restart any current running clusters for this to take effect.Nr"  r   r;   r9   enable_usage_statsr%    s:     0>>>>		H    r;   c                  6    d} t          j        | d           dS )z=Take a stack dump of all Python workers on the local machine.a  
pyspy=`which py-spy`
if [ ! -e "$pyspy" ]; then
    echo "ERROR: Please 'pip install py-spy'"         "or 'pip install ray[default]' first."
    exit 1
fi
# Set IFS to iterate over lines instead of over words.
export IFS="
"
# Call sudo to prompt for password before anything has been printed.
sudo true
workers=$(
    ps aux | grep -E ' ray::|default_worker.py' | grep -v raylet | grep -v grep
)
for worker in $workers; do
    echo "Stack dump for $worker";
    pid=`echo $worker | awk '{print $2}'`;
    case "$(uname -s)" in
        Linux*) native=--native;;
        *)      native=;;
    esac
    sudo $pyspy dump --pid $pid $native;
    echo;
done
    T)shellN)r  call)COMMANDs    r9   stackr*    s&    G4 OG4((((((r;   c                  &    ddl m}   |              dS )z6Run a local Ray microbenchmark on the current machine.r   mainN)ray._private.ray_perfr-  r,  s    r9   microbenchmarkr/    s%     +*****DFFFFFr;   z'Override the Ray address to connect to.c                 H   t          j        |           } t                              d|  d           t	          j        |            t          j                                        d          }t          j
                            t          j        j                                        d| d          }t	          j        |           t          j
                            |          }t                              d| d	| d
           t                              d           dS )z1Take a Chrome tracing timeline for a Ray cluster.r   r   r   z%Y-%m-%d_%H-%M-%Szray-timeline-z.json)r   zTrace file written to z (z bytes).z>You can open this with chrome://tracing in the Chrome browser.N)r   r   r   r   r3   r   r   todayr   r   r[  r   r0   rX  get_user_temp_dirtimelinegetsize)r   rx   r   sizes       r9   r4  r4    s     <WEEG
KK;;;;<<<HW>$$%899Dw||++--/Jt/J/J/J H L(####7??8$$D
KKCCCTCCCDDD
KKPQQQQQr;   z
--group-byNODE_ADDRESSSTACK_TRACEzLGroup object references by a GroupByType (e.g. NODE_ADDRESS or STACK_TRACE).z	--sort-by)PIDOBJECT_SIZEREFERENCE_TYPEr:  zfSort object references in ascending order by a SortingType (e.g. PID, OBJECT_SIZE, or REFERENCE_TYPE).z--units)BKBMBGBr<  zFSpecify unit metrics for displaying object sizes (e.g. B, KB, MB, GB).z--no-formatz^Display unformatted results. Defaults to true when terminal width is less than 137 characters.)rO   r>   r=   r?   z--stats-onlyz Display plasma store stats only.z--num-entriesz--nz+Specify number of sorted entries per group.c           	         t          j        |           } t          j                            |           }t          |           t          j                    }dd| dz   dz   }	t          | ||||||          }
t          |	 d|
            dS )z.Print object references held in a Ray cluster.r1  z========z Object references status: r  r%  N)
r   r   r3   _raylet	GcsClientr:   r   nowr   rq   )r   group_bysort_byunits	no_format
stats_onlynum_entriesr6   rx   header	mem_statss              r9   r   r     s    j <WEEG&&w&77Jz"""<>>D<T<<<<wFF I 
V
"
"y
"
"#####r;   z8Experimental: Display additional debuggging information.)r<   rO   r   r?   c                    t          j        |           } t          j                            |           }t          |           t          j        j                            |           |	                    t          j                                                  }|	                    t          j                                                  }t          t          ||||                      dS )z1Print cluster status, including autoscaling info.r1  )r   r   N)r   r   r3   rA  rB  r:   rg   rh   _initialize_internal_kvinternal_kv_getrj   DEBUG_AUTOSCALING_STATUSencodeDEBUG_AUTOSCALING_ERRORrq   r   )r   r   r6   statusr_  s        r9   rR  rR  ^  s     <WEEG&&w&77Jz""" 88DDD''(N(U(U(W(WXXF&&}'L'S'S'U'UVVE	,vugw
G
G
GHHHHHr;   z--streamz-Sz:If True, will stream the binary archive contents to stdout)r<   r>   rO   r=   r?   z--outputz-ozOutput file.z--logs/--no-logsz!Collect logs from ray session dirz--debug-state/--no-debug-statez,Collect debug_state.txt from ray session dirz--pip/--no-pipzCollect installed pip packagesz--processes/--no-processesz!Collect info on running processesz*--processes-verbose/--no-processes-verbosez&Increase process information verbosityz
--tempfilez-TzTemporary file to usestreamoutputlogsdebug_statepip	processesprocesses_verbosetempfilec           
      4    t          | |||||||           dS )zCollect local data and package into an archive.

    Usage:

        ray local-dump [--stream/--output file]

    This script is called on remote nodes to fetch their data.
    rS  rT  rU  rV  rW  rX  rY  rZ  N)r    r\  s           r9   
local_dumpr]  v  s>    F +	 	 	 	 	 	r;   z--hostz-hz,Single or list of hosts, separated by comma.z
--ssh-userz-UzUsername of the SSH user.z	--ssh-keyz-KzPath to the SSH key file.z--dockerz-dz,Name of the docker container, if applicable.z--localz-Lz.Also include information about the local node.z(Collect debug_state.txt from ray log dirr   ssh_userssh_keyr  localc                     t          | |||||||||	|
||          }|rt          j        d|            dS t          j        d           dS )aq  Get log data from one or more nodes.

    Best used with Ray cluster configs:

        ray cluster-dump [cluster.yaml]

    Include the --local flag to also collect and include data from the
    local node.

    Missing fields will be tried to be auto-filled.

    You can also manually specify a list of hosts using the
    ``--host <host1,host2,...>`` parameter.
    )rM   r   r^  r_  r  r`  rT  rU  rV  rW  rX  rY  rZ  zCreated archive: zCould not create archive.N)r   rT   rU   )rM   r   r^  r_  r  r`  rT  rU  rV  rW  rX  rY  rZ  archive_paths                 r9   cluster_dumprc    s    Z ,/+  L  0
5|5566666
./////r;   c                     t          j        |            t           j        j                                         t          d           dS )z9Trigger Python garbage collection on all cluster workers.r1  z&Triggered gc.collect() on all workers.N)r3   r   rE   internal_api	global_gcrq   r1  s    r9   rf  rf  G	  sC     HWL'')))	
233333r;   z	--node-idzVHex ID of the worker node to be drained. Will default to current node if not provided.z--reasonc                 J    g | ] }|d          t           j        k    |d         !S )r   r   )r(   DRAIN_NODE_REASON_UNSPECIFIED)r   items     r9   r   r   `	  s8     	
 	
 	
Aw.FFF GFFFr;   z(The reason why the node will be drained.z--reason-messagez"The detailed drain reason message.z--deadline-remaining-secondsa  Inform GCS that the node to be drained will be force killed after this many of seconds. Default is None which means there is no deadline. Note: This command doesn't actually force kill the node after the deadline, it's the caller's responsibility to do that.r   reasonreason_messagedeadline_remaining_secondsc                    t          j        |           } |%t          j                                                    }d}|9|dk     rt          j        d|           t          j                    dz  |dz  z   }t          j	        
                    |          t          j	                                        k    rt          j        d|           t          j                            |           }t          |           |                    |t           j                            |          ||          \  }}|st          j        d|           dS )	zF
    This is NOT a public API.

    Manually drain a worker node.
    Nr   z5--deadline-remaining-seconds cannot be negative, got i@B i  z"Invalid hex ID of a Ray node, got r1  z#The drain request is not accepted: )r   r   r3   get_runtime_contextget_node_idrT   BadParameterrx   time_nsNodeIDfrom_hexnilrA  rB  r:   
drain_noder(   DrainNodeReasonValuerX   )	r   r   rj  rk  rl  deadline_timestamp_msr6   is_acceptedrejection_error_messages	            r9   ru  ru  R	  sh   j <WEEG)++7799!-%))$414 4   "&7!:&-!
 z7##sz~~'7'777 !Og!O!OPPP&&w&77Jz"""+5+@+@&,,V44	, ,(K(  
"K2IKK
 
 	

 
r;   zkuberay-autoscaler)r  r   z[The name of the Ray Cluster.
Should coincide with the `metadata.name` of the RayCluster CR.z--cluster-namespacezvThe Kubernetes namespace the Ray Cluster lives in.
Should coincide with the `metadata.namespace` of the RayCluster CR.rY   cluster_namespacer  c                 *    ddl m}  || |           dS )zRuns the autoscaler for a Ray cluster managed by the KubeRay operator.

    `ray kuberay-autoscaler` is meant to be used as an entry point in
        KubeRay cluster configs.
    `ray kuberay-autoscaler` is NOT a public CLI.
    r   )run_kuberay_autoscalerN).ray.autoscaler._private.kuberay.run_autoscalerr}  )rY   r{  r}  s      r9   kuberay_autoscalerr  	  s1    0 VUUUUU<):;;;;;r;   zhealth-checkz--componentzNHealth check for a specific component. Currently supports: [ray_client_server]z--skip-version-checkz6Skip comparison of GCS version with local Ray version.c                 \   t          j        |           } t          j                            |           }|st          |           |st          j        d           |                    |	                                t          j                  }|st          j        d           t          j        |          }t          j                    }t          |d                   }||z
  }|t          j        j
        j        k     }	|	rt          j        d           dS t          j        d           dS )zl
    This is NOT a public API.

    Health check a Ray or a specific component. Exit code 0 is healthy.
    r1  r   ra   r   rx   N)r   r   r3   rA  rB  r:   r  exitrN  rP  rj   KV_NAMESPACE_HEALTHCHECKrn   ro   rx   floatrE   HEALTHCHECK_EXPIRATION_S)
r   	componentskip_version_checkr6   
report_strreportcur_timereport_timedeltatime_oks
             r9   healthcheckr  	  s   0 <WEEG&&w&77J ':&&& ++m&L ,  J  Z
##F
 y{{Hv''K {"Ecl0IIG r;   z--show-library-pathz-showz8Show the cpp include path and library path, if provided.z$--generate-bazel-project-template-toz-genzEThe directory to generate the bazel project template to, if provided.c                 r   t           j        dk    rt          j        d          | s|st	          d          t
          j                            t
          j                            t          j
                            }t
          j                            |d          }t
          j                            |d          }t
          j                            |d          }t
          j                            |d          }t
          j                            |          st	          d          | rPt          j        d	t          j        |                      t          j        d
t          j        |                      |r|}t
          j                            |          rt%          j        |           t%          j        ||           dD ]]}t
          j                            ||dd                   }	t%          j        t
          j                            ||          |	           ^t
          j                            |d          }
t%          j        ||
           t
          j                            |d          }t%          j        ||           t          j        dt          j        t
          j                            |                                t          j        d           t          j        t          j        dt
          j                            |           d                     dS dS )zBShow the cpp library path and generate the bazel project template.win32z2Ray C++ API is not supported on Windows currently.zePlease input at least one option of '--show-library-path' and '--generate-bazel-project-template-to'.cppexampleincludelibz:Please install ray with C++ API by "pip install ray[cpp]".zRay C++ include path {} zRay C++ library path {} )
_WORKSPACEz_BUILD.bazelz	_.bazelrcr   Nzthirdparty/includezthirdparty/libz Project template generated to {}z#To build and run this template, runz    cd z && bash run.sh)r  r  rT   rX   rS  r   r[  abspathdirnamer3   __file__r   isdirr   rq   r   r*  existsshutilrmtreecopytreemove)show_library_path"generate_bazel_project_template_toraydircpp_dircpp_templete_dirinclude_dirlib_dirout_dirr   	dest_nameout_include_dirout_lib_dirs               r9   r  r  	  s   $ |w"#WXXX 
%G 
;
 
 	
 W__RW__S\::;;Fgll65))Gw||GY77',,w	22Kgll7E**G7==!! WUVVV L3RW=M5N5NOOO3RW\5J5JKKK) W4 7>>'"" 	#M'"""('222C 	D 	DHWhqrrl;;IKWh77CCCC',,w0DEE_555gll7,<==---.Grww//122	
 	
 	
 	>???!T27??7+C+C!T!T!TUUVVVVV/W Wr;   c                      t           j        dt          fd            } t          j        |                                           }|t           j        k    sJ t          j        d|            dS )zRun a sanity check to check that the Ray installation works.

    This is not a public API and is intended to be used by Ray developers only.
    r  c                      t           j        S r   )r3   r4   r   r;   r9   get_versionz!sanity_check.<locals>.get_versionC
  s
    r;   zSuccess! Ray version: N)r3   remoter   r   r4   r   rG  )r  vs     r9   sanity_checkr  <
  s{     	Z    Z 	""$$%%A33344444r;   metrics)r  c                      d S r   r   r   r;   r9   metrics_groupr  L
  s    Dr;   zlaunch-prometheusc                  ,    t          j                     d S r   )r)   r-  r   r;   r9   launch_prometheusr  Q
  s     %'''''r;   zshutdown-prometheusc                      	 t          j        d           d S # t           j        j        $ r1} t	          d|             t          j        d           Y d } ~ d S d } ~ ww xY w)Nzhttp://localhost:9090/-/quitzAn error occurred: r   )requestspost
exceptionsRequestExceptionrq   r  r  )r\   s    r9   shutdown_prometheusr  V
  ss    455555/   'A''(((s    A&AAzget-auth-tokenz
--generatez#Generate a new token if none existsc                    ddl m} ddlm} |                                }|                    d          sK| r5t          j        dd            |             |                                 nt          j	        d          |
                    d          }t          j        |d	
           t          j        dd           dS )zPrints the Ray authentication token to stdout.

    If --generate is specified, a new token is created and saved to ~/.ray/auth_token if one does not exist.
    r   )generate_and_save_token)AuthenticationTokenLoaderT)ignore_auth_modez&Generating new authentication token...)errzQNo authentication token found. Use ray `get-auth-token --generate` to create one.F)nlr   N)6ray._private.authentication.authentication_token_setupr  ray._rayletr  instance	has_tokenrT   rU   reset_cacherX   get_raw_token)generater  r  loadertokens        r9   get_auth_tokenr  _
  s             
 '//11FT22  	J?TJJJJ##%%%    &c  
   $ 77E 
Ju	Jrtr;   zsymmetric-run)logs_state_cli_groupray_getray_listsummary_state_cli_groupr  r   summaryz0Integrating ray state command line tool failed: )job_cli_groupr   z3Integrating ray jobs command line tool failed with )	serve_cliz4Integrating ray serve command line tool failed with c                      t                      S r   r  r   r;   r9   r-  r-  
  s    55Lr;   __main__r   )FNTTTTFN)NNNNNNNTTTTFN)rn   rB   r   r  r  r]  r  r  rx   r  urllib.parser+  r   typingr   r   r   r   rT   r   r  r3   !ray._common.usage.usage_constantsr0   r1   usage_constantsr9  ray._private.ray_constantsrE   rj   ray._private.servicesr   ray._common.network_utilsr   r	   ray._common.usager
   ray._common.utilsr   r  r   ray._private.internal_apir   ray._private.label_utilsr   r   r   ray._private.logr   r   &ray._private.resource_isolation_configr   ray._private.utilsr   r   "ray.autoscaler._private.cli_loggerr   r   r    ray.autoscaler._private.commandsr   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   !ray.autoscaler._private.constantsr&   5ray.autoscaler._private.fake_multi_node.node_providerr'   ray.core.generatedr(   ray.dashboard.modules.metricsr)   ray.scripts.symmetric_runr*   ray.util.annotationsr+   ray.util.check_open_portsr,   r  	getLogger__name__r   r:   rH  optionLOGGER_LEVELr   LOGGER_LEVEL_HELPLOGGER_FORMATLOGGER_FORMAT_HELPversion_optionrL   commandargumentrv   DEFAULT_DASHBOARD_PORTr]   rw   r   r   r  r   r5  r>  r?  systemDEFAULT_DASHBOARD_IPrV   #DEFAULT_DASHBOARD_AGENT_LISTEN_PORTro   r  r  r  r  r  r  r  r  r  r  r  Choicer  r  r  rg  r%  r*  r/  r4  r   rR  r]  rc  rf  rv  rR  ru  r  r  r  r  r  r  r  r  add_commandray.util.state.state_clir  r  r  r  ImportErrorr\   ray.dashboard.modules.job.clir  rW   ray.serve.scriptsr  r-  r   r;   r9   <module>r     s.     				        



              - - - - - - - - - - - -    



 : : : : : : : : : : : : 2 2 2 2 2 2 2 2 2 ( ( ( ( ( ( ( ( ( B B B B B B B B ' ' ' ' ' ' ( ( ( ( ( (      5 4 4 4 4 4         
 J I I I I I I I J J J J J J        Y X X X X X X X X X                              < ; ; ; ; ; S S S S S S - - - - - - F F F F F F 3 3 3 3 3 3 * * * * * * 6 6 6 6 6 6 		8	$	$	
 	
 	
 &			(   '			)   6 6     6 %3???		0   	0	5   	0	1   	2	          @? >@* c#h *  *  *  * Z     %c0U   	@  h?3 h? h? h? h?   h?V %c8U   kE:VWWW	
$!
$ 
$ 
$	   	
(   	0	=   	0	=   		R   		:	   		8   	
#   	
,   
=   	
/   	
C   	
3	   5s1R   5s1R   		S 8?	))	 	 
  . 	2	   		?	   .
	   	0	C	J	J,
 
   #	=	?	   !		?	   		M	   
U	   		9	   !	 
a	   		8	   	@	   
%   		2   $	O   		O	   	T   	
   
9	   	@	   	
)	 	 	 		Y   	
)	   !
,	 	 	 	
D		 	 	 	
D	   	
C
 
 
 
H H  
 
 	 		 	  	 	                    /                 XW  \
HX 	<	   	B	 	 	 
`* `*C `* `* `*  	 	  &`*F %3???		I	   		I	   	1   	1   T45R   		0   	2	   	8	   ,	=	 	 	 	@	   
2 2   	 	         @? P2j %3???T45R   dE8S   		0   	H	   
        @? . D%3???T45R   
I	   		0       @? & %3???s;U   		0   > >    @? >
 %3???tU1O   e2N   he:TUUU		0   	2	   gtT0QRRR		N   
     SR   VU   @? B0 %3???5s3335s333		0   H H   43 43 @? H
 %3???5s3335s333		0   	2        43 43 @? &* 7>??%3???	?	   tU1O   e2P   he:TUUU		0   	2	   		N   4c222		?	   R(((	@	   
;    (,h h  }h h h    )(  32    VU    @? @?thV %3???3///	m	$	$
M   	?	   tU1O   e2P   he:TUUU		0   	2	   		N   	@	            VU     0/ @? h F %3???		0  D D  @? D
 %3???		0  & &  @? & 
 
 
 
 
 
 ) ) )>    		2	  R R  R %c0U   	~}5	6	6
%	   	>>>	?	?
-	   	---	.	.
	   	
-   D%6X   			6  $ $        V$6 %c0U   	C   IC I$ I I I    I D		E   u3>   	,	   $	7	   dD7W    	,	   0	1	   		     #" SM  	
 
   sm           `> %C@@@		7   		$   		$   		7   		9   u3>   	,	   $	3	   dD7W    	,	   0	1	   		    *."!   #"/0 /0!#/0
3-/0 sm/0 c]	/0
 SM/0 D>/0 SM/0 /0 /0 
/0 /0 /0 sm/0 /0 /0             A@ `/0d D%c0U  4 4  4 D%c0U   		a	   		
 	
&6<<>>	
 	
 	

 
 
4   		-	   "	
3
 
 
,
,
,
 ,
 	,

 !$,
 ,
 ,

 
     N,
^ &t44	
E	   	
J	  <S <S <T < < <   54< ...%c0U   	
	   	A	  % %    /."%P 	C   *
		P   +W +W     +W\ D5 5 5 )	 	 	 /00( ( 10( 122  32 "##	.	     $#D 	                                      ! ! !                     
        	            # $ $ $ " # # #     
      ! ! !     O 4 4 4    I            OOH6O***OOG%O(((OO+)O<<<OO(vO6666 I I I
LLGAGGHHHHHHHHIL;;;;;;OOMO.... L L L
LLJqJJKKKKKKKKLM++++++OOI M M M
LLKKKLLLLLLLLM   zDFFFFF sa   n2A1Ap$ p$Aqp)AqqAqq Aq1 q1Arq6ArrArrAr: r:As"r?AssAs"