
    &`i7a                     2   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZd dlm	Z	m
Z
 d dlmZ d dlmZmZ d dlmZmZ d dlmZmZ d dlmZmZ d d	lmZ d d
lmZmZmZm Z m!Z!m"Z"m#Z#m$Z$  e j%        e&          Z'dZ(dZ) G d d          Z* G d de*e          Z+dS )    N)Thread)usage_constants	usage_lib)subprocess_output_util)cf
cli_logger)AUTOSCALER_NODE_START_WAIT_SProcessRunnerError)LABELS_ENVIRONMENT_VARIABLERESOURCES_ENVIRONMENT_VARIABLE)CreateClusterEventglobal_event_system)LogTimer)STATUS_SETTING_UPSTATUS_SYNCING_FILESSTATUS_UP_TO_DATESTATUS_UPDATE_FAILEDSTATUS_WAITING_FOR_SSHTAG_RAY_FILE_MOUNTS_CONTENTSTAG_RAY_NODE_STATUSTAG_RAY_RUNTIME_CONFIG      c            	       V    e Zd ZdZddddeddddf	dZd ZddZd Zd	 Z	dd
Z
ddZdS )NodeUpdatera  A process for syncing files and running init commands on a node.

    Arguments:
        node_id: the Node ID
        provider_config: Provider section of autoscaler yaml
        provider: NodeProvider Class
        auth_config: Auth section of autoscaler yaml
        cluster_name: the name of the cluster.
        file_mounts: Map of remote to local paths
        initialization_commands: Commands run before container launch
        setup_commands: Commands run before ray starts
        ray_start_commands: Commands to start ray
        runtime_hash: Used to check for config changes
        file_mounts_contents_hash: Used to check for changes to file mounts
        is_head_node: Whether to use head start/setup commands
        rsync_options: Extra options related to the rsync command.
        process_runner: the module to use to run the commands
            in the CommandRunner. E.g., subprocess.
        use_internal_ip: Wwhether the node_id belongs to an internal ip
            or external ip.
        docker_config: Docker section of autoscaler yaml
        restart_only: Whether to skip setup commands & just restart ray
        for_recovery: True if updater is for a recovering node. Only used for
            metric tracking.
    NFc           	         d                     |          | _        |p.|                    dd          o|o|                    dd           }|                    | j        ||||||          | _        d| _        || _        |                    d          | _        || _        |pi }d |	                                D             | _
        || _        || _        |	| _        || _        || _        |
| _        || _        |pg }d |D             | _        |pi | _        || _        || _        || _        || _        d | _        || _        d S )	NzNodeUpdater: {}: use_internal_ipsFuse_external_head_ipTtypec                 T    i | ]%\  }}|t           j                            |          &S  ospath
expanduser).0remotelocals      s/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/updater.py
<dictcomp>z(NodeUpdater.__init__.<locals>.<dictcomp>x   s;     
 
 
2?&%FBG&&u--
 
 
    c                 L    g | ]!}t           j                            |          "S r!   r"   )r&   r$   s     r)   
<listcomp>z(NodeUpdater.__init__.<locals>.<listcomp>   s5     %
 %
 %
)-BGt$$%
 %
 %
r+   )format
log_prefixgetget_command_runner
cmd_runnerdaemonnode_idprovider_typeprovideritemsfile_mountsinitialization_commandssetup_commandsray_start_commandsnode_resourcesnode_labelsruntime_hashfile_mounts_contents_hashcluster_synced_filesrsync_optionsauth_configis_head_nodedocker_configrestart_onlyupdate_timefor_recovery)selfr4   provider_configr6   rB   cluster_namer8   r9   r:   r;   r>   r?   rC   r<   r=   r@   rA   process_runneruse_internal_iprD   rE   rG   s                         r)   __init__zNodeUpdater.__init__C   s   0 .44W== * 
 2E:: S!4!45KU!S!S 	 #55O
 
 ,0088  "'R
 
CNCTCTCVCV
 
 
 (?$,"4,&()B&
  49r%
 %
1E%
 %
 %
! +0b&(*((r+   c           	         t          j                     }t          j                    r)t          j                    rd}t	          j        |           	 t          | j        d                    | j	                  z             5  | 
                                 d d d            n# 1 swxY w Y   n# t          $ r}| j                            | j        t          t           i           t	          j        dt%          j        t                                t	          j        d           t)          |d          rEt+          |dd          }t	          j        dt%          j        |j                  |j        |           nyt	          j        d	t3          t5          |                               t7          j                    }t	          j        d
|           t	          j        dt3          |                     t	          j        d           t	          j                     t=          |t>          j                   rY d }~d S  d }~ww xY wt          tB          tD          | j	        i}| j#        | j#        |tH          <   | j                            | j        |           t	          j%        dtB                     t          j                     |z
  | _&        d| _'        d S )NzOutput was redirected for an interactive command. Either do not pass `--redirect-command-output` or also pass in `--use-normal-shells`.zApplied config {}zNew status: {}z!!!cmdstderrzNo stderr availablez7Setup command `{}` failed with exit code {}. stderr: {}zException details: {}zFull traceback: {}zError message: {}
New statusr   )(timecmd_output_utildoes_allow_interactiveis_output_redirectedr   abortr   r/   r.   r>   	do_update	Exceptionr6   set_node_tagsr4   r   r   errorr   boldhasattrgetattrrO   
returncodeverbose_errorstrvars	traceback
format_excnewline
isinstanceclickClickExceptionr   r   r?   r   labeled_valuerF   exitcode)rH   update_start_timemsgestderr_outputfull_tracebacktags_to_sets          r)   runzNodeUpdater.run   s    IKK244	"466	"9 
 S!!! 	"5"<"<T=N"O"OO  ! !    ! ! ! ! ! ! ! ! ! ! ! ! ! ! !  	 	 	M''24HI   -rw7K/L/LMMMU###q%   > '85J K K MGAENNL!	    ()@#d1gg,,OOO!*!5!7!7 !5~FFF !4c!ff===U###   !U122 7	<  !2"D$5
 )58<8VK45##DL+>>> /@AAA9;;)::sC   /B.  B!B. !B%%B. (B%)B. .I 9E;H;:H;;I r      c                 V   	 |\  }}g 	t           j        dk    rddg	d	 fd	}t          j        dd|dz   |f	          5   j                                        D ]\  }} |||           |dz  }d d d            n# 1 swxY w Y    j        r{t          j        d
d|dz   |f	          5  t          j        dt           j                              j        D ]} |||d           |dz  }d d d            d S # 1 swxY w Y   d S t          j        dd|dz   |f	           d S )Nr   z~/ray_bootstrap_key.pemz~/ray_bootstrap_config.yamlFc                 \   |r6t           j                            |          st          j        d|           d S t           j                            |          s
J |            t           j                            |          r4|                    d          s|dz  }|                     d          s| dz  } t          j        d	                    ||           z             5  j
        oj
        d         dk    }|sLj                            d	                    t           j                            |                     d            || d	
           | vr:t          j        dt          j        |           t          j        |                     d d d            d S # 1 swxY w Y   d S )Nz"sync: {} does not exist. Skipping./zSynced {} to {}container_name zmkdir -p {}hostrun_envT)docker_mount_if_possiblez
{} from {})r#   r$   existsr   printisdirendswithr   r/   r.   rD   r2   rp   dirnamer   r[   )remote_path
local_pathallow_non_existing_paths	is_dockernolog_pathsrH   sync_cmds       r)   do_syncz-NodeUpdater.sync_file_mounts.<locals>.do_sync   s   ' z0J0J  !EzRRR 7>>*--99z99-w}}Z(( '!**3// &#%J"++C00 '3&K"3":"::{"S"SS    &U4+=>N+OSU+U  ! O''%,,RW__[-I-IJJ & (    [4PPPPk11$$bgk&:&:BGJ<O<O  !                 s   !B3F!!F%(F%zProcessing file mounts[]   	_numberedzProcessing worker file mountszsynced files: {}T)r   zNo worker file mounts to syncF)r   	verbositygroupr8   r7   r@   r}   r`   )
rH   r   step_numbersprevious_stepstotal_stepsr   r   r   r$   r   s
   ``       @r)   sync_file_mountszNodeUpdater.sync_file_mounts   sF   &2#1$$46STK"	 "	 "	 "	 "	 "	 "	 "	J $~7I;0W
 
 
 	  	  ,0+;+A+A+C+C 1 1'ZZ0000aN	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  $ 	!/!!3[A   $ $  !3S9R5S5STTT 5 G GDGD$FFFFF!#$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ /!!3[A     s%   1B  BB.AD  DDc                    t          j        dddt          f          5  t          | j        dz             5  t          j        dt          j        d                     d }	 t          j                    |k    rt          d	          | j
                            | j                  rt          d
          	 | j                            ddd           t          j        d           	 d d d            d d d            dS # t           $ r@}t#          j        ||t&                    }t          j        t&                     Y d }~n[d }~wt          $ rJ}dt+          |          z   dz   }t-          |d          rt/          |j        t*                    r|j        }nyt/          |j        t2                    rd                    |j                  }nDt6                              dt;          |j                   d           t+          |j                  }d                    |j        |          }t          j        dt          j         |          t          j        t+          t&                                         t          j        t&                     Y d }~nd }~ww xY wA# 1 swxY w Y   	 d d d            d S # 1 swxY w Y   d S )Nz#Waiting for SSH to become availabler   r   r   zGot remote shellzRunning `{}` as a test.uptimeTzwait_ready timeout exceeded.z8wait_ready aborting because node detected as terminated.
   rx   )timeoutrz   zSuccess.)retry_interval()rO    ze.cmd type (z) not list or str.z(Exit Status {}): {}z3SSH still not available {}, retrying in {} seconds.)!r   r   NUM_SETUP_STEPSr   r/   r}   r   r[   rR   rX   r6   is_terminatedr4   r2   rp   successr
   rS   handle_ssh_failsREADY_CHECK_INTERVALsleepr`   r\   re   rO   listjoinloggerdebugr   r.   r^   dimmed)rH   deadlinefirst_conn_refused_timerl   	retry_strcmd_s         r)   
wait_readyzNodeUpdater.wait_ready  sp   1dA=W
 
 
 8	9 8	9 $/,>>?? 59 59 !:BGH<M<MNNN*.'29y{{X--'(FGGG}224<@@ '6  
)9++Hb&+QQQ"*:666#!59 59 59 59 59 598	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9( . 9 9 92A2R3+?3 3 3/
 
#788888888$ 9 9 9 %(#a&&L3$6	"1e,, )!%55 2'(u!+AE4!8!8 2'*xx &$R4;;$R$R$R!" !" !" (+15zz(>(E(E !d) )I #(QIi00GC(<$=$=>>   
#788888888;9+2959 59 59 59 59 59 59 59 598	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9 8	9sl   J<A?J#71D)J<
J5E J#JE JJ#JJ##J'	'J<*J'	+J<<K K c           
      N   | j                             | j        t          t          i           t          j        dt                     t          j                    t          z   }| 	                    |           t          j        t          j                   | j                             | j                  }t                              d                    t%          |                               | j        dk    rP| j         j        rDddlm}  || j         j        | j        | j         j                                      | j                   |                    t6                    | j        k    rE| j                            | j        | j        d          }|r|t6          xx         dz  cc<   d| _         | j         rg | _!        |                    t6                    | j        k    rJ| j"        r#|                    tF                    | j"        k    r t          j$        d	d
dtJ          f           nt          j$        dtM          | j                             | j                             | j        t          tN          i           t          j        dtN                     | (                    | j)        dtJ          f           |                    t6                    | j        k    r| j                             | j        t          tT          i           t          j        dtT                     | j+        rBt          j,        dd
dtJ          f          5  t          j        t          j-                   t]          | j/        dz   d          5  | j+        D ]}t          j        t          j-        d|i           	 | j        0                    || j1                            d          d           Z# td          $ rM}|j3        dk    r(t          j4        d           t          j4        d           tk          j6        d          d d }~ww xY w	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   nt          j$        dd
dtJ          f           t          j,        d d
d!tJ          f          5  | j                            | j        | j        d           d d d            n# 1 swxY w Y   | j!        rt          j,        d"d
d#tJ          f          5  t          j        t          j7                   t]          | j/        d$z   d          5  tq          | j!                  }ts          | j!                  D ]\  }}t          j        t          j7        d|i           t
          j:        dk    r3tq          |          d%k    r tw          j<        |d d%                   d&z   }	ntw          j<        |          }	t          j$        d'|	d(||f           	 | j        0                    |d)*           # td          $ rL}|j3        dk    r(t          j4        d           t          j4        d           tk          j6        d+          d }~ww xY w	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   nt          j$        d,d
d#tJ          f           t          j,        d-d
d.tJ          f          5  t          j        t          j=                   t]          | j/        d/z   d          5  | j>        D ]&}i }
| j        r2t          j@                    rd|
t          jB        <   nd|
t          jB        <   | j        d0k    r,| jC        r| jC        |
t          <   | jE        r| jE        |
t          <   	 t          jH                    }t          jI        d           | j        0                    ||
d)1           t          jI        |           # td          $ rL}|j3        dk    r(t          j4        d           t          j4        d           tk          j6        d2          d }~ww xY w	 d d d            n# 1 swxY w Y   t          j        t          jJ                   d d d            d S # 1 swxY w Y   d S )3NrQ   zNode tags: {}awsr   )CloudwatchHelperF)as_headr8   sync_run_yetz-invalidatezYConfiguration already up to date, skipping file mounts, initalization and setup commands.r   z2-6r   zUpdating cluster configuration.)hash)_tagsr   )r   zRunning initialization commands   zInitialization commandsT)show_statuscommandssh_private_keyrx   )ssh_options_override_ssh_keyrz   ssh_command_failedzFailed.zSee above for stderr.zInitialization command failed.z"No initialization commands to run.zInitializing command runnerr   zRunning setup commands   zSetup commands   z...z{}z()autory   zSetup command failed.zNo setup commands to run.zStarting the Ray runtimer   zRay start commandsr(   )environment_variablesrz   zStart command failed.)Kr6   rY   r4   r   r   r   rh   rR   r	   r   r   execute_callbackr   ssh_control_acquired	node_tagsr   r   r.   r`   r5   rI   8ray.autoscaler._private.aws.cloudwatch.cloudwatch_helperr   rJ   update_from_configrC   r0   r   r>   r2   run_initr8   rE   r:   r?   r   r}   r   dictr   r   rsync_upr   r9   r   run_initialization_cmdr   r/   rp   rB   r
   msg_typerZ   rf   rg   run_setup_cmdlen	enumerater   r   r[   start_ray_runtimer;   r   usage_stats_enabledr   USAGE_STATS_ENABLED_ENV_VARr<   r   r=   r   rS   rU   set_output_redirectedstart_ray_runtime_completed)rH   r   r   r   init_requiredrO   rl   totalicmd_to_printenv_varsold_redirecteds               r)   rW   zNodeUpdater.do_updateO  sn   ##L.0FG	
 	
 	
 	 /EFFF9;;!==!!!,-?-TUUUM++DL99	_++C	NN;;<<<&&4=+H&      -t|T]=W   !2333==/00D4EEE !O44) ," 5  M
  *0111]B111$)! 	%"$D ==/00D4EEE. F}}9::-. .
 J8     14CT9U9U9U    M''24HI   $\3GHHH!!$-q/>R!SSS
 }}3448III++L#68I"J   (7HIII/ *#)9#'O"<   $0 $0 ,<.E   & O.GG(,   0 0 (,'C 0 0 3 D$6$M%.$4!" !" !"!0 %)O$7$7(+EIEUEYEY,=F* F* 17 %8 %& %& %& %& (: !0 !0 !0'(z5I'I'I(2(8(C(C(C(2(89P(Q(Q(Q*/*>(H+& +&+/%0!0%0	0 0 0 0 0 0 0 0 0 0 0 0 0 0 0$0 $0 $0 $0 $0 $0 $0 $0 $0 $0 $0 $0 $0 $0 $0L $<#'O"<     %1#Q8   	 	
 O,, $ 1$($4%) -   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 & '#)0#'O"<   !X !X
 ,<.<   & O.>>D   X X %((;$<$<E*3D4G*H*H X X3 3 D$6$DyRUFV!" !" !" $.#71#<#<SB3573ss83D3Du3LLL3573<<L * 0$(,4EBR!" !" !" !"!X$(O$7$7V$7$L$L$L$L'9 !X !X !X'(z5I'I'I(2(8(C(C(C(2(89P(Q(Q(Q*/*>?V*W*W$W!X!X	X X X X X X X X X X X X X X X!X !X !X !X !X !X !X !X !X !X !X !X !X !X !XF $3#'O"<   
 &4O2L
 
 
 (	 (	  01C1UVVV$/,@@dSSS !L !L2  L  LC!H( V$8:: VTUH_%PQQ UVH_%PQ )W44. 4 !% 3 % >  + UDHDTH%@AL)8)M)O)O'=eDDD++x ,    (=nMMMM- L L L:)===&,Y777&,-DEEE#23JKKKL7 L!L !L !L !L !L !L !L !L !L !L !L !L !L !L !LD  0">  M(	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	s&  8Q&+Q25O('Q(
P?	2AP:	:P?	?QQ&Q	Q&Q	Q&&Q*-Q*.(S""S&)S&8Z'CZX*)Z*
Z 	4AY;	;Z 	 ZZ'Z	Z'Z	Z''Z+.Z+/8b'A=a$%A_>=a$>
a	Aa	a	a$b$a(	(b+a(	,!bb!bc                 6   i }||d<   | j                             d          |d<   | j                             d          |d<   | j                            |||           t	          j        dt          j        |          t          j        |                     d S )Nr{   rsync_excludersync_filteroptionsz#`rsync`ed {} (local) to {} (remote))rA   r0   r2   run_rsync_upr   verboser   r[   rH   sourcetargetr{   r   s        r)   r   zNodeUpdater.rsync_up!  s    .F*+#'#5#9#9/#J#J "&"4"8"8"H"H$$VVW$EEE1276??BGFOO	
 	
 	
 	
 	
r+   c                 6   i }||d<   | j                             d          |d<   | j                             d          |d<   | j                            |||           t	          j        dt          j        |          t          j        |                     d S )Nr{   r   r   r   z#`rsync`ed {} (remote) to {} (local))rA   r0   r2   run_rsync_downr   r   r   r[   r   s        r)   
rsync_downzNodeUpdater.rsync_down+  s    .F*+#'#5#9#9/#J#J "&"4"8"8"H"H&&vvw&GGG1276??BGFOO	
 	
 	
 	
 	
r+   )rq   r   )__name__
__module____qualname____doc__
subprocessrM   rp   r   r   rW   r   r   r!   r+   r)   r   r   (   s         P !!-N) N) N) N)`< < <|A A A AF99 99 99vP P Pd
 
 
 

 
 
 
 
 
r+   r   c                       e Zd Zd ZdS )NodeUpdaterThreadc                 f    t          j        |            t          j        | g|R i | d| _        d S )N)r   rM   r   ri   )rH   argskwargss      r)   rM   zNodeUpdaterThread.__init__7  s>    T3D333F333r+   N)r   r   r   rM   r!   r+   r)   r   r   6  s#            r+   r   ),loggingr#   r   rR   rb   	threadingr   rf   ray._common.usager   r   ray.autoscaler._privater   rS   "ray.autoscaler._private.cli_loggerr   r   &ray.autoscaler._private.command_runnerr	   r
   !ray.autoscaler._private.constantsr   r   $ray.autoscaler._private.event_systemr   r   !ray.autoscaler._private.log_timerr   ray.autoscaler.tagsr   r   r   r   r   r   r   r   	getLoggerr   r   r   r   r   r   r!   r+   r)   <module>r      s    				                 8 8 8 8 8 8 8 8 M M M M M M = = = = = = = =               Y X X X X X X X 6 6 6 6 6 6	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 
	8	$	$ K
 K
 K
 K
 K
 K
 K
 K
\    V     r+   