&`i_#dZddlZddlZddlZddlZddlmZddlZddlZddl m Z ddl m Z ddl Z e ddZdefd Zefd Zefd efd Zd eedeefdZejdddddejddedejdedejddejdZedkr edSdS)zSymmetric Run for Ray.N)List) env_integer) GcsClient&RAY_SYMMETRIC_RUN_CLUSTER_WAIT_TIMEOUTreturncbddlmcm}|}t |dkS)Nr)ray._private.services_privateservicesfind_gcs_addresseslen)r running_gcs_addressess m/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/scripts/symmetric_run.pycheck_ray_already_startedrsE,,,,,,,,,%7799 $ % % ))cptj}d}tjdtj|z |krqtjdt tj}||krdSt jd|d|dtj|z |kqdS) z\Wait for all nodes to start. Raises an exception if the nodes don't start in time. T)ignore_reinit_errorzWaiting for nodes to start... /z nodes startedF)timerayinitsleeprnodesclickecho)nnodestimeout start_time current_nodess rcheck_cluster_readyr#s JMH&&&& )++ "W , , 1 CIKK(( F " "4 JWWWWWW    )++ "W , , 5raddressc6tj}t|}tj|z |kr[|gdrtjddStjdtj|z |k[dS)N)r$r)r zRay cluster is ready!TrF)rr check_aliverrr)r$r r! gcs_clients rcheck_head_node_readyr(1sJ7+++J )++ "W , ,  ! !"a ! 0 0  J. / / /4 1 )++ "W , , 5rrun_and_start_argscNtd|d}t|jd}|D]j}|dkrt jd|dkrt jd|d krt jd |d krt jd k|S) N_T)resilient_parsingray_args_and_entrypoint--headz*Cannot use --head option in symmetric_run.z--node-ip-addressz5Cannot use --node-ip-address option in symmetric_run.z--portz*Cannot use --port option in symmetric_run.--blockz+Cannot use --block option in symmetric_run.) symmetric_run make_contextlistparamsrClickException)r)ctx cleaned_argsargs r"curate_and_validate_ray_start_argsr8<s  $ $S*>L V V (??&'STT T % % %&G  (??&'STT T )  &'TUU U  rr0T)ignore_unknown_optionsallow_extra_argsaQCommand to start Ray across all nodes and execute an entrypoint command. USAGE: ray symmetric-run --address ADDRESS [--min-nodes NUM_NODES] [RAY_START_OPTIONS] -- [ENTRYPOINT_COMMAND] DESCRIPTION: This command (1) starts a Ray cluster across all nodes, (2) runs a command on the head node, and (3) stops the Ray cluster. The '--' separator is required to distinguish between Ray start arguments and the entrypoint command. The --min-nodes option is optional and can be used to wait for a specific number of nodes to start. EXAMPLES: # Start Ray with default settings and run a Python script ray symmetric-run --address 127.0.0.1:6379 -- python my_script.py # Start Ray with specific head node and run a command ray symmetric-run --address 127.0.0.1:6379 --min-nodes 4 -- python train_model.py --epochs=100 # Start Ray and run a multi-word command ray symmetric-run --address 127.0.0.1:6379 --min-nodes 4 --num-cpus=4 -- python -m my_module --config=prod RAY START OPTIONS: Most ray start command options are supported. Arguments that are not supported are: --head, --node-ip-address, --port, --block. SEPARATOR REQUIREMENT: The '--' separator is mandatory and must appear between Ray start arguments and the entrypoint command. This ensures clear separation between the two sets of arguments. )namecontext_settingshelp --addresszThe address of the Ray cluster.)requiredtyper=z --min-nodesz4If provided, wait for this number of nodes to start.)r@r=r-)nargsr@c tjdd}|r|ddkr |dd} |d}n"#t$rt jdwxYw|d|||dzd}}t |}|dn|}|st jdtrt jdtj j |}|t jd|d |\} } tj | | tjtj} | dd d} n*#tj$rt jd | wxYwg} t#jD]M\}}|D]E}|jtjjtjjfvr| |jFN|dkr d | D} | | v}d} |rt jd dddd| d| g|}t7j|ddt jdt jd|dkr#t;|st jdt jd|t jdt7j|}t jdnZt jd|dt=|st jdddd|dg|}t7j|dn#t6j$r}t jd|d |j r0t jd!|j !d |j"r0t jd"|j"!d Yd}~n)d}~wtF$rt jd#d YnwxYwt7jdd$g|D|j$dkr;t jd%|j$d tj%|j$dSdSdS#t7jdd$g|C|j$dkr9t jd%|j$d tj%|j$wwwxYw)&Nrrz symmetric-runz--zqNo separator '--' found in arguments. Please use '--' to separate Ray start arguments and the entrypoint command.zNo entrypoint command provided.z$Ray is already started on this node.zInvalid address format: z, should be `host:port`zCould not resolve hostname: c*g|]}|dk|dk|S)z 127.0.0.1z::1).0ips r z!symmetric_run..s'KKK{):):rU{{"{{{rz*On head node. Starting Ray cluster head...rstartr.z--node-ip-address=z--port=T)checkcapture_outputzHead node started.z=======================z+Timed out waiting for other nodes to start.zRunning command on head node: z-On worker node. Connecting to Ray cluster at z...z)Timed out waiting for head node to start.r>r/)rKzFailed to start Ray: )errzstdout: zstderr: zInterrupted by user.stopz Command failed with return code )&sysargvindex ValueErrorrr4r8rr_common network_utils parse_addresssocket getaddrinfo AF_UNSPEC SOCK_STREAMgaierrorpsutil net_if_addrsitemsfamily AddressFamilyAF_INETAF_INET6appendr$r subprocessrunr#r(CalledProcessErrorstdoutdecodestderrKeyboardInterrupt returncodeexit)r$ min_nodesr-all_args separatorr)entrypoint_on_headray_start_args gcs_host_portgcs_hostgcs_portaddrinforesolved_gcs_hostmy_ipsifaceaddrsaddris_headresult ray_start_cmdes rr0r0Psnx|H HQK?22ABB< NN4((    " G     )Q!+ 88JKKN&II F"#DEEE ""K"#IJJJK-;;GDDM" Gw G G G   'HhN% h 0&2D  %QKN1- ?NNN"#L(#L#LMMMNF+--3355,, u , ,D{$,$- dl+++  ,1}}LKvKKK6)G FB( / 6 JC D D D8%688$($$   M N=T J J J J J+ , , , J0 1 1 11}}%8%C%C}*A JE1CEE    J0 1 1 1^$677F J0 1 1 1 1 JSwSSS T T T(11 X*+VWWW   M N= 5 5 5 5  (BBB .1..D9999 8 B J618??#4#466D A A A A 8 B J618??#4#466D A A A A 555 )t4444445 v'''  &"3q"8"8 JM&:KMMSW X X X X HV& ' ' ' ' '  "8"8 v'''  &"3q"8"8 JM&:KMMSW X X X X HV& ' ' ' ' "8sVAA#?E'E5D;MQ&PBOQ&#PQ&PQ&&AS__main__)__doc__rVrcrOrtypingrrrray._private.ray_constantsr ray._rayletrr[CLUSTER_WAIT_TIMEOUTboolrr#strr(r8commandoptionintargument UNPROCESSEDr0__name__rFrrrs   222222!!!!!! "{#KRPP*4****)=*1E349c( 04$OO( ,,,Z$S/P ? )%:KLLLC(C(ML [,,lC(L zMOOOOOr