&`iL dddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZmZddlZddlZddlmZddlmZddlmZmZmZddlZdZd Zd d gZGd d eZGddeZ GddeZ!GddZ"GddZ#GddZ$ d;de$de ee%de%de$fdZ& dd(e e%fd)Z, d?d+e#d%e"d,e%de e%fd-Z-de$d+e#d%e"fd.Z.de$d%e"fd/Z/de$d0ee#d%e"fd1Z0de$d0ee#d%e"fd2Z1d3e%dee e%e%e%e e%e e%ffd4Z2 d@d5e e%d6e e%d7e e%d8e e%d9e e%f d:Z3dS)AN)ThreadPoolExecutor)contextmanager)ListOptionalSequenceTuple) cli_logger)_get_node_provider)NODE_KIND_HEADNODE_KIND_WORKERTAG_RAY_NODE_KINDubuntu~/ray_bootstrap_key.pemz%~/.ssh/ray-autoscaler_2_us-west-2.pemceZdZdS) CommandFailedN__name__ __module__ __qualname__x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/cluster_dump.pyrrDrrceZdZdS)LocalCommandFailedNrrrrrrrrrceZdZdS)RemoteCommandFailedNrrrrrr#rrrcdeZdZ d dedededededeeeeeff d ZdS) GetParametersTNlogs debug_statepip processesprocesses_verboseprocesses_listcZ||_||_||_||_||_||_dSN)r!r"r#r$r%r&)selfr!r"r#r$r%r&s r__init__zGetParameters.__init__(s7 &"!2,r)TTTTTN) rrrboolrrrstrr*rrrr r 's "&;?---- -  -  -!eCI&6!78------rr c DeZdZdZ d dededed eed ef d ZdS) NodezNode (as in "machine")rrNFhostssh_userssh_keydocker_containeris_headcL||_||_||_||_||_dSr()r/r0r1r2r3)r)r/r0r1r2r3s rr*z Node.__init__<s,    0 r)rrNF)rrr__doc__r,rr+r*rrrr.r.9su  !0*.      #3-        rr.ceZdZdZddeefdZedZdZ dZ dZ d Z e dd ed eefd ZdS)ArchiveaArchive object to collect and compress files into a single file. Objects of this class can be passed around to different data collection functions. These functions can use the :meth:`subdir` method to add files to a sub directory of the archive. Nfilec|ptjddd|_d|_t j|_dS)N ray_logs_.tar.gzprefixsuffix)tempfilemkstempr8tar threadingLock_lock)r)r8s rr*zArchive.__init__Ts>UH,K RRRSTU ^%% rc*t|jSr()r+rBr)s ris_openzArchive.is_openYsDH~~rcDtj|jd|_dS)Nzw:gz)tarfileopenr8rBrGs rrKz Archive.open]s< 622rcF|jd|_dSr()rBcloserGs rrMz Archive.close`s rc.||Sr()rKrGs r __enter__zArchive.__enter__ds  rc.|dSr()rM)r)exc_typeexc_valexc_tbs r__exit__zArchive.__exit__hs r/subdirrootc#KtjGfdd}|VdS)a Open a context to add files to the archive. Example: .. code-block:: python with Archive("file.tar.gz") as archive: with archive.subdir("logfiles", root="/tmp/logs") as sd: # Will be added as `logfiles/nested/file.txt` sd.add("/tmp/logs/nested/file.txt") Args: subdir: Subdir to which to add files to. Calling the ``add(path)`` command will place files into the ``subdir`` directory of the archive. root: Root path. Files without an explicit ``arcname`` will be named relatively to this path. Yields: A context object that can be used to add files to the archive. cFeZdZeddedeeffd ZdS) Archive.subdir.._ContextNpatharcnamecbtj|}|p=tjtj|}jj||j dS)Nr\) osr[abspathjoinrelpathrEacquirerBaddrelease)r[r\rWr)rVs rrdz$Archive.subdir.._Context.addswt,,!VRW\\&"'//$PT:U:U%V%V ""$$$ T7 333 ""$$$$$rr()rrr staticmethodr,rrd)rWr)rVsr_ContextrZs^  % %# %  % % % % % % %\ % % %rrgN)r_r[r`)r)rVrWrgs``` rrVzArchive.subdirkst.wt$$ % % % % % % % % % % % %hjjrr()rU)rrrr5rr,r*propertyrHrKrMrOrTrrVrrrr7r7Ks&&Xc]&&&& X333""S" """^"""rr7/tmp/ray/session_latestarchiveexcludesession_log_dirreturncD |js||pg}tjtj|d}|d|5}tj|D]}\}}}|D]t}tj||}tj|| t fd|Dr_| |u~ dddn #1swxYwY|S)aCopy local log files into an archive. Args: archive: Archive object to add log files to. exclude (Sequence[str]): Sequence of regex patterns. Files that match any of these patterns will not be included in the archive. session_dir: Path to the Ray session files. Defaults to ``/tmp/ray/session_latest`` Returns: Open archive object. r!rW)startc3BK|]}tj|VdSr()rematch).0patternrel_paths r z%get_local_ray_logs..s/JJwrx22JJJJJJrN) rHrKr_r[ra expanduserrVwalkrbanyrd) rjrkrlsdrWdirsfilesr8 file_pathrvs @rget_local_ray_logsrs[$ ? mGgll27#5#5o#F#FOOO _ 5 5"!#!9!9 " " D$ " "GLLt44 7??9O?LLJJJJ'JJJJJy!!!!  " """""""""""""""" Ns5BDDD session_dirc|js|tj|}tj|d}tj|std|d|5}| |dddn #1swxYwY|S)zCopy local log files into an archive. Args: archive: Archive object to add log files to. session_dir: Path to the Ray session files. Defaults to ``/tmp/ray/session_latest`` Returns: Open archive object. zlogs/debug_state.txtz No `debug_state.txt` file found.roN) rHrKr_r[rxraexistsrrVrd)rjrdebug_state_filer{s rget_local_debug_staters ? '$$[11Kw||K1GHH 7>>* + +E !CDDD  - -!    !!!!!!!!!!!!!!! Ns CC Cc|js| ddlm}n#t$r ddlm}YnwxYwt jd5}|D]}||dg| | d5}| |j ddddn #1swxYwYdddn #1swxYwY|S)zGet currently installed pip packages and write into an archive. Args: archive: Archive object to add meta files to. Returns: Open archive object. r)freezewt rzpip_packages.txtN) rHrKpip._internal.operationsr ImportErrorpip.operationsr@NamedTemporaryFile writelinesflushrVrdname)rjrfpliner{s rget_local_pip_packagesrs ? *3333333 ***))))))))*  $T * *0bMMOO ( (D MM4, ' ' ' '  ^^B   02 FF27. / / / 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 000000000000000 NsC$77AC%&C C%C C%C C%%C),C)Fr$verbosec Z|sddlm}|}g}tjgdD]} |5d|}||r|n#|ddddd| |j | d |fdddn #1swxYwY#t$r}t||d}~wwxYwi}|D]E\} }|D]=\} } | r | d } ntj|} | | vr| d |vr | || d <>Ft!jd 5} |D]+}| t)j|d g,| |d5}|| j ddddn #1swxYwYdddn #1swxYwY|S)a7Get the status of all the relevant ray processes. Args: archive: Archive object to add process info files to. processes: List of processes to get information on. The first element of the tuple is a string to filter by, and the second element is a boolean indicating if we should filter by command name (True) or command line including parameters (False) verbose: If True, show entire executable command line. If False, show just the first term. Returns: Open archive object. r) RAY_PROCESSES)pidrcmdlinestatus z--r?N) executablerrrrrrrmetazprocess_info.txt)!ray.autoscaler._private.constantsrpsutil process_iteroneshotrarappendsplitrrr Exceptionr subprocess list2cmdliner@rvaluesryamldumprrVrd)rjr$rr process_infosprocessrexcrelevant_processes process_dictkeyword filter_by_cmdcorpusrrr{s rget_local_ray_processesrs<" "CCCCCC! M&'K'K'KLL33 3""  ((7??#4#455$$ '+@''!(tQ!7!7!:3B3!?$+LLNN#*;&-nn&6&6  ))                    3 3 3$S))s 2 3!.GG g&/ G G "G] :%f-#099&  \%%8@R%R%R:F"<#67  G  $T * *0b&--// 3 3D MM49T??D1 2 2 2 2  ^^F # # 0r FF27. / / / 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 000000000000000 NsrC7B%C+ C7+C/ /C72C/ 3C77 DDD7A*H !H = H  H H H H  H$'H$ parametersct|js||jr= t|n+#t$r}t j|Yd}~nd}~wwxYw|jr= t|n+#t$r}t j|Yd}~nd}~wwxYw|j r= t|n+#t$r}t j|Yd}~nd}~wwxYw|j rI t||j |jn+#t$r}t j|Yd}~nd}~wwxYw|S)a'Get all local data. Gets: - The Ray logs of the latest session - The currently installed pip packages Args: archive: Archive object to add meta files to. parameters: Parameters (settings) for getting data. Returns: Open archive object. )rjN)rjr$r)rHrKr!rrr errorr"rr#rr$rr&r%)rjrrs rget_all_local_datar6s ? " " w / / / / /! " " "  S ! ! ! ! ! ! ! ! "" " !' 2 2 2 2 2! " " "  S ! ! ! ! ! ! ! ! "~" " "7 3 3 3 3 3! " " "  S ! ! ! ! ! ! ! ! "" " #$3"4      " " " "  S ! ! ! ! ! ! ! ! " Ns\5 AAA(A99 B!BB!,B== C%C  C%0D D5D00D5'itemsc8|d||S)Nr)ra)rquotess r_wraprhs" /chhuoo /v / //rray remote_node script_pathc ddddd|j|jd|jg}|jr |dd|jgz }|d d g}||jrd gnd gz }||jrd gndgz }||jrdgndgz }||jrdgndgz }|jr||jrdgndgz }|ddt|dgz }|j sdnd}tj d|jtjd|d|jddd }t|d!5} t!j||t$j"n=#t j$r+}t+d#d$||d%}~wwxYw d%d%d%n #1swxYwY|S)&aCreate an archive containing logs on a remote node and transfer. This will call ``ray local-dump --stream`` on the remote node. The resulting file will be saved locally in a temporary file and returned. Args: remote_node: Remote node to gather archive from. script_path: Path to this script on the remote node. parameters: Parameters (settings) for getting data. Returns: Path to a temporary file containing the node's collected data. sshz-o StrictHostKeyChecking=noz-o UserKnownHostsFile=/dev/nullz-o LogLevel=ERRORz-i@dockerexecz local-dumpz--streamz--logsz --no-logsz --debug-statez--no-debug-statez--pipz--no-pipz --processesz--no-processesz--processes-verbosez--no-proccesses-verbosez /bin/bashz-c")rnodeheadz"Collecting data from remote node: ray__r;r<r?wb)stdoutstderrz(Gathering logs from remote node failed: rN)r1r0r/r2r!r"r#r$r%rr3r printr@rArKr check_callsysrCalledProcessErrorrra) rrrcmd collect_cmdcattmprrs r'create_and_get_archive_from_remote_noderlsc& %)   44+"244 C#     (    j9KCH::{mCK (>XO$$EWDXXK @G99ZL@Kj&:RM??AQ@RRK + - " # #+, Ku[=== >>C#+ 7&&CL+:JLLMMM  "B"B"B{/?"B"B"B9 U U UVW XC c4B   !#b D D D D D,   %J388C==JJ   E Js6E2!D('E2(E"7&EE""E22E69E6c Pt||}|js||jsdnd}|dt j|5}||d|d|j dd d d n #1swxYwY|S) a%Create and get data from remote node and add to local archive. Args: archive: Archive object to add remote data to. remote_node: Remote node to gather archive from. parameters: Parameters (settings) for getting data. Returns: Open archive object. rrrrorrr;r^N) rrHrKr3rVr_r[dirnamerdr/)rjrrrrr{s r+create_and_add_remote_data_to_local_archivers 2+z J JC ? #+ 7&&C !5!5 6 6D" sB3BB)9BBBCCCDDDDDDDDDDDDDDD Ns+$BB"Bct5}t||dddn #1swxYwY|js||dt j|j5}| |jddddn #1swxYwYt j |j|S)zCreate and get data from this node and add to archive. Args: archive: Archive object to add remote data to. parameters: Parameters (settings) for getting data. Returns: Open archive object. Nrrozlocal_node.tar.gzr^) r7rrHrKrVr_r[rr8rdremove)rjrlocal_data_archiver{s r*create_and_add_local_data_to_local_archiversK ;(-z:::;;;;;;;;;;;;;;; ?  1C1H!I!I J JEb !&0CDDDEEEEEEEEEEEEEEEI %&&& Ns,00 B44B8;B8 remote_nodesc|js|tt5}|D] }|t |||! dddn #1swxYwY|S)agCreate an archive combining data from the remote nodes. This will parallelize calls to get data from remote nodes. Args: archive: Archive object to add remote data to. remote_nodes (Sequence[Node]): Sequence of remote nodes. parameters: Parameters (settings) for getting data. Returns: Open archive object. ) max_workers)rjrrN)rHrKrMAX_PARALLEL_SSH_WORKERSsubmitr)rjrrexecutorrs rcreate_archive_for_remote_nodesrs ?  (@ A A AX'  K OO;'%       Ns$A""A&)A&c"|js| t||n+#t$r}t j|Yd}~nd}~wwxYwt |||t jdt|d|S)ajCreate an archive combining data from the local and remote nodes. This will parallelize calls to get data from remote nodes. Args: archive: Archive object to add data to. remote_nodes (Sequence[Node]): Sequence of remote nodes. parameters: Parameters (settings) for getting data. Returns: Open archive object. Nz#Collected data from local node and z remote nodes.) rHrKrrr rrrlen)rjrrrs r)create_archive_for_local_and_remote_nodesrs ? 27JGGGG $G\:FFFSc,.?.?SSS Ns. AAAcluster_configc ddlm}tjd|tj|}tjt| }||d}t|d|d tti} tti} fd||zD}|d d }|d d }d }|d d } | r| dd }|dd } ||||| fS)a[Get information from Ray cluster config. Return list of host IPs, ssh user, ssh key file, and optional docker container. Args: cluster_config: Path to ray cluster config. Returns: Tuple of list of host IPs, ssh user name, ssh key file path, optional docker container name, optional cluster name. r)_bootstrap_configz6Retrieving cluster information from ray cluster file: T)no_config_cacheprovider cluster_namec:g|]}|Sr) external_ip)rtrrs r z4get_info_from_ray_cluster_config..@s' N N NDX ! !$ ' ' N N Nrauthr0ssh_private_keyNrcontainer_name) ray.autoscaler._private.commandsrr rr_r[rxr safe_loadrKreadr non_terminated_nodesr r r get) rrconfig head_nodes worker_nodeshostsr0r1r docker_configrrs @r get_info_from_ray_cluster_configr"shCBBBBBU^UUW''77N ^D005577 8 8F  vt < < /RSSJ002CEU1VWWL N N N NJ4M N N NEf~j)HVn./G FJJx..M;""#3T::::nd33L (GV\ 99rclusterr/r0r1rc|s[|sYtjd}tj|r|}t jd|dn!|rtj|}d}|rOt |\}}} } }|p|}|p| }|p| }|r|dn|} | std|n'|r|d} ntd|st}t jd|d |sdtD]\} tj| } tj| r| }t jd |d n]|| ||||fS) ziParse command line arguments. Note: This returns a list of hosts, not a comma separated string! z~/ray_bootstrap_config.yamlz Detected cluster config file at z@. If this is incorrect, specify with `ray cluster-dump `N,z6Invalid cluster file or cluster has no running nodes: z` or `--host`.zUsing default SSH user `z9`. If this is incorrect, specify with `--ssh-user `zAuto detected SSH key file: z6. If this is incorrect, specify with `--ssh-key `) r_r[rxrr warningrrrDEFAULT_SSH_USERDEFAULT_SSH_KEYS)rr/r0r1rbootstrap_configrhukdrcand_key cand_key_files r_info_from_paramsr Ns  . .7--.KLL 7>>* + + &G  /7///    .'$$W--L #CG#L#L 1aL=q,Q1#'. 3Q $VWVV     3 J     # Fx F F F    (  HG..x88Mw~~m,, '"L7LLL   E8Wfl BBr)Nri)ri)NF)r)r)NNNNN)4r_rrrrrJr@rCconcurrent.futuresr contextlibrtypingrrrrrr"ray.autoscaler._private.cli_loggerr !ray.autoscaler._private.providersr ray.autoscaler.tagsr r r rrrr RuntimeErrorrrrr r.r7r,rrrr+rrrrrrrrrr rrrrs  111111%%%%%%222222222222 999999@@@@@@SSSSSSSSSS -/VW     L                -   --------$$CCCCCCCCZ(,4## # hsm $## ####N*C #& :G<37>> >U39-./>>>>>B**]****d00c0000 FK===#0=?B= c]====@ #'5B4 "/4 $,TN@M> $,TN@MF):): 49c3 x} <=):):):):Z""! >C>C c]>C 3->Csm>Cc] >C SM >C>C>C>C>C>Cr