%`i<,ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl Z ddl Z ddl Z ddl mZddlmZddlmZddlmZddlmZejeZejdkZeGd d ZeGd d ZdS) N)DictOptional) ray_constants)disable_client_hook)GcsClientOptions)FAKE_HEAD_NODE_ID) DeveloperAPIntcTeZdZdZ d dededefdZ d dZd d ee fd Z d Z dS)AutoscalingClusterzzCreate a local autoscaling cluster for testing. See test_autoscaler_fake_multinode.py for an end-to-end example. Fhead_resourcesworker_node_types autoscaler_v2c N||_|j||fd|i||_||_dS)zCreate the cluster. Args: head_resources: resources of the head node, including CPU. worker_node_types: autoscaler node types config for worker nodes. rN)_head_resources_generate_config_config_autoscaler_v2)selfr rr config_kwargss e/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/cluster_utils.py__init__zAutoscalingCluster.__init__ sQ .,t,    (     ,c tjttjtjtjd}tj |}||d<|idd|dd<|rd|dd<t|dd <| ||S) Nz0autoscaler/_private/fake_multi_node/example.yamlavailable_node_typesr) resources node_config max_workerszray.head.defaultTproviderlaunch_multiple head_node_id) yaml safe_loadopenospathjoindirnameray__file__copydeepcopyrupdate)rr rrr base_config custom_configs rrz#AutoscalingCluster._generate_config6sn  GOOCL11F     k22 0A ,-'E E ,-.@A  J;?M* %&7 88IM* %n 5]+++rN override_envctjgdtj\}}t |d5}|t j|jdddn #1swxYwYddd |dg}d|j vr@| d |j dd |j vr@| d |j d d |j vr@| d |j d |j r?| d t j|j |<| d t j|dtj}|ddd|jr#|dt$t$d|r||tj||dS)z{Start the cluster. After this call returns, you can connect to the cluster with ray.init("auto"). r)stopz--forcewNr)startz--autoscaling-config={}z--headCPUz --num-cpus={}GPUz --num-gpus={}object_store_memoryz--object-store-memory={}z--resources='{}'z--system-config={}),:) separators1)AUTOSCALER_UPDATE_INTERVAL_SRAY_FAKE_CLUSTER)RAY_enable_autoscaler_v2RAY_CLOUD_INSTANCE_ID RAY_OVERRIDE_NODE_ID_FOR_TESTING)env) subprocess check_calltempfilemkstempr$writejsondumpsrformatrappendpopr%environr+r-rr)r_system_configr0_ fake_configfcmdrBs rr5zAutoscalingCluster.startPs 888999!)++; +s # # .q GGDJt|,, - - - . . . . . . . . . . . . . . .   % , ,[ 9 9    D( ( ( JJ--d.B.F.Fu.M.MNN O O O D( ( ( JJ--d.B.F.Fu.M.MNN O O O D$8 8 8 JJ*11(,,-BCC      T JJ)00D8I     % JJ| $ $ $cs++++++s-A66A:=A:c2tjgddS)zTerminate the cluster.r2N)rCrDrs rshutdownzAutoscalingCluster.shutdowns 88899999r)F)NN) __name__ __module__ __qualname____doc__dictboolrrrrr5rUrrr r s$ ,,, , ,,,,.@E4/,/,x~/,/,/,/,b:::::rr c eZdZ ddedededefdZed Zed Zdd Z dd efd Z ddZ dde fdZ dde fdZdZdZdZdS)ClusterFNTinitialize_headconnecthead_node_argsshutdown_at_exitcvtrtdd|_t |_d|_d|_tj j |_ ||_ |s|rtd|r)|pi}|jdi||r|dSdSdS)aOInitializes all services of a Ray cluster. Args: initialize_head: Automatically start a Ray cluster by initializing the head node. Defaults to False. connect: If `initialize_head=True` and `connect=True`, ray.init will be called with the address of this cluster passed in. head_node_args: Arguments to be passed into `start_ray_head` via `self.add_node`. shutdown_at_exit: If True, registers an exit hook for shutting down all started processes. zRay cluster mode is currently experimental and untested on Windows. If you are using it and running into issues please file a report at https://github.com/ray-project/ray/issues.NFz(Cannot connect to uninitialized cluster.r\)cluster_not_supportedloggerwarning head_nodeset worker_nodes redis_address connectedr)_privatestate GlobalState global_state_shutdown_at_exit RuntimeErroradd_noder`)rr_r`rarbs rrzCluster.__init__s( !  NNN    EE!L.::<<!1 K7 KIJJ J  +1rN DM + +N + + +       rc,|jdS|jjSN)rg gcs_addressrTs rruzCluster.gcs_addresss > !4~))rc|jSrtrurTs raddresszCluster.addresss rc|jJ|jrJtj|d|j|j|j}t |d|_dS)z"Connect the driver to the cluster.NT) namespaceignore_reinit_errorrx_redis_username_redis_password)rxrkr)initredis_usernameredis_passwordreinfo)rrz output_infos rr`zCluster.connectsj|'''>!!!h $L / /      K   rwaitc xdddddd}tjjjdi|}|jdi|t 5|j tjj|d|j |j }||_|jj |_ | dtj |_| d tj|_|jj|_t%j|jddd }|j|tjj|jjn||j ||j |d |dd|vrd|_tjj|d |j |j }|j||r||dddn #1swxYwY|S)aAdds a node to the local Ray Cluster. All nodes are by default started with the following settings: cleanup=True, num_cpus=1, object_store_memory=150 * 1024 * 1024 # 150 MiB Args: wait: Whether to wait until the node is alive. node_args: Keyword arguments used in `start_ray_head` and `start_ray_node`. Overrides defaults. Returns: Node object of the added Ray node. ri` )num_cpusnum_gpusr8min_worker_portmax_worker_portNT)headrb spawn_reaperrrF)allow_cluster_id_nilfetch_cluster_id_if_nil)rjrw)include_log_monitor)node_manager_portdashboard_agent_listen_portr\)r)rl parameter RayParamsupdate_if_absentrrgnodeNoderprjgetrREDIS_DEFAULT_USERNAMErREDIS_DEFAULT_PASSWORDr webui_urlrcreateruro_initialize_global_stateutilswrite_ray_addressrriadd_wait_for_node)rr node_argsdefault_kwargs ray_paramsr gcs_optionss rrrzCluster.add_nodes"#4    \+5BB BB # #55n555 " "7 *7 *~%|(--%)%;!%!7 . "&%)^%A"&/mm$m&J''#'0mm$m&J''#"&!9.5$)-,1  !::;GGG "44T^5OPPPP++$:L+MMM++8H+III+++FFF++a+@@@0 AA>BJ:|(--%)%;!%!7 . !%%d+++ * ##D)))o7 *7 *7 *7 *7 *7 *7 *7 *7 *7 *7 *7 *7 *7 *7 *r sG$H//H36H3ctjjjj}|2|j|jkr"tjtd||j |kr%|j d|dd|_ n2| d|d|j || r JddS)zKills all processes associated with worker node. Args: node: Worker node of which all associated processes will be removed. NzRemoving a node that is connected to this Ray client is not allowed because it will break the driver. You can use the get_other_node utility to avoid removing a node that the Ray client is connected.FT) check_aliveallow_gracefulrz3There are zombie processes left over after killing.)r)rlworker global_workerr_raylet_socket_namerU ValueErrordestroy_external_storagergkill_all_processesriremoveany_processes_alive)rrr global_nodes r remove_nodezCluster.remove_nodesl)7<  "';+JJJ  ? %%''' >T ! ! N - -!.t .   "DNN  # #!.t $      $ $T * * *((** A A @ A A * A Artimeoutcftjj|j|j|dS)aWait until this node has appeared in the client table. Args: node (ray._private.node.Node): The node to wait for. timeout: The amount of time in seconds to wait before raising an exception. Raises: TimeoutError: An exception is raised if the timeout expires before the node appears in the client table. N)r)rlservices wait_for_noderuplasma_store_socket_name)rrrs rrzCluster._wait_for_nodeEs: ++    )      rctj}tj|z |kr|j}t|}t||krt ddSt t|d|tjdtj|z |ktd)aWaits for correct number of nodes to be registered. This will wait until the number of live nodes in the client table exactly matches the number of "add_node" calls minus the number of "remove_node" calls that have been made on this cluster. This means that if a node dies without "remove_node" having been called, this will raise an exception. Args: timeout: The number of seconds to wait for nodes to join before failing. Raises: TimeoutError: An exception is raised if we time out while waiting for nodes to join. z!All nodes registered as expected.Nz6 nodes are currently registered, but we are expecting g?z*Timed out while waiting for nodes to join.) timero_live_node_idslenlist_all_nodesredebugsleep TimeoutError)rr start_time live_clientsexpecteds rwait_for_nodeszCluster.wait_for_nodesWs"Y[[ ikkJ&00,;;==L4..0011H<  H,, @AAA <((77,477 3ikkJ&00GHHHrcRt|j}|jr |jg|z}|S)zLists all nodes. TODO(rliaw): What is the desired behavior if a head node dies before worker nodes die? Returns: List of all nodes, including the head node. )listrirg)rnodess rrzCluster.list_all_nodesxs2T&'' > -^$u,E rcXtd|DS)aReturns a bool indicating whether all processes are alive or not. Note that this ignores processes that have been explicitly killed, e.g., via a command like node.kill_raylet(). Returns: True if all processes are alive and false otherwise. c3>K|]}|VdSrt)remaining_processes_alive).0rs r z4Cluster.remaining_processes_alive..s.VV41133VVVVVVr)allrrTs rrz!Cluster.remaining_processes_alives-VV@S@S@U@UVVVVVVrc0t|j}|D]}|||j||jtjjtjj dS)zRemoves all nodes.N) rrirrgr) experimental internal_kv_internal_kv_reset_commonrreset_ray_address)r all_nodesrs rrUzCluster.shutdowns *++  # #D   T " " " " > %   T^ , , , $77999 ++-----r)FFNTrt)T)r)rVrWrXr[rZrpropertyrurxr`rrrfloatrrrrrUr\rrr^r^sc!&#!% (((( (  ((((T**X*   X     RRTRRRRh%A%A%A%AN  E    $IIeIIIIB    W W W.....rr^)r+rHloggingr%rCrErtypingrrr"r)ray._private.services ray._privaterray._private.client_mode_hookr ray._rayletr5ray.autoscaler._private.fake_multi_node.node_providerrray.util.annotationsr getLoggerrVrenamerdr r^r\rrrsz   !!!!!!!! &&&&&&======((((((SSSSSS------  8 $ $4i:i:i:i:i:i:i:i:XX.X.X.X.X.X.X.X.X.X.r