
    &`i                     P   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZmZmZ d dlZd dlmZ d dlmZmZ d dlmZmZmZ d dlmZmZmZmZmZ d d	l m!Z! d d
l"m#Z#m$Z$m%Z% d dl&m'Z'  ej(        e)          Z*dZ+ej,        -                    ej,        .                    ej,        /                    e0                    d          Z1dZ2dZ3dddZ4d Z5d Z6d Z7de8fdZ9de:dee:e;f         fdZ<d Z= G d d          Z> G d de'          Z? G d d e'          Z@dS )!    N)getuserquote)DictListOptionalTuple)%DEFAULT_OBJECT_STORE_MAX_MEMORY_BYTES)cf
cli_logger)AUTOSCALER_NODE_SSH_INTERVAL_SAUTOSCALER_NODE_START_WAIT_S&DEFAULT_OBJECT_STORE_MEMORY_PROPORTION)check_bind_mounts_cmdcheck_docker_imagecheck_docker_running_cmddocker_start_cmdswith_docker_exec)LogTimer)ProcessRunnerErroris_output_redirectedrun_cmd_redirected)CommandRunnerInterface
   z_kubernetes/kubectl-rsync.sh      T)use_login_shellssilent_rsyncc                      t           d         S )Nr   _config     z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/command_runner.pyis_rsync_silentr%   1   s    >""r#   c                     | t           d<   dS )zChoose whether to silence rsync output.

    Most commands will want to list rsync'd files themselves rather than
    print the default rsync spew.
    r   Nr    vals    r$   set_rsync_silentr)   5   s     "GNr#   c                      t           d         S )Nr   r    r"   r#   r$   is_using_login_shellsr+   >   s    %&&r#   r(   c                     | t           d<   dS )a  Choose between login and non-interactive shells.

    Non-interactive shells have the benefit of receiving less output from
    subcommands (since progress bars and TTY control codes are not printed).
    Sometimes this can be significant since e.g. `pip install` prints
    hundreds of progress bar lines when downloading.

    Login shells have the benefit of working very close to how a proper bash
    session does, regarding how scripts execute and how the environment is
    setup. This is also how all commands were ran in the past. The only reason
    to use login shells over non-interactive shells is if you need some weird
    and non-robust tool to work.

    Args:
        val: If true, login shells will be used to run all commands.
    r   Nr    r'   s    r$   set_using_login_shellsr-   B   s    " #&Gr#   cmdenvironment_variablesc                 
   g }|                                 D ]S\  }}t          j        |d          }d                    |t	          |                    }|                    |           Td                    |          }|| z   S )a7  Prepend environment variables to a shell command.

    Args:
        cmd: The base command.
        environment_variables (Dict[str, object]): The set of environment
            variables. If an environment variable value is a dict, it will
            automatically be converted to a one line yaml string.
    ),:)
separatorszexport {}={}; )itemsjsondumpsformatr   appendjoin)r.   r/   
as_stringskeyr(   sall_varss          r$   _with_environment_variablesr?   V   s     J)//11  Sj444""3c

33!wwz""Hc>r#   c                 6    d|  d}ddddt          |          gS )NzEsource ~/.bashrc; export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && ()bashz--loginz-c-ir   )r.   force_interactives     r$   _with_interactiverE   i   s>    	E>A	E 	E 	E  ItT51B+C+CDDr#   c                   "    e Zd ZddZdddZdS )
SSHOptionsNc                     || _         dt          j        ddddd| _        |rBt          j        dk    rn1| j                            dd                    |          d	d
           | j                            |           d S )Nnoyesr   r   )StrictHostKeyCheckingUserKnownHostsFileIdentitiesOnlyExitOnForwardFailureServerAliveIntervalServerAliveCountMaxwin32autoz{}/%C10s)ControlMasterControlPathControlPersist)ssh_keyosdevnullarg_dictsysplatformupdater8   )selfrW   control_pathkwargss       r$   __init__zSSHOptions.__init__r   s     &* #%*# %* $%#$#
 
&  	|w&&$$)/'.~~l'C'C*/    	V$$$$$r#   <   timeoutc                    d                     |          | j        d<   | j        r	d| j        gng }|d d | j                                        D             D             z   S )Nz{}sConnectTimeoutrC   c                     g | ]	}|D ]}|
S r"   r"   ).0yxs      r$   
<listcomp>z2SSHOptions.to_ssh_options_list.<locals>.<listcomp>   sA     !
 !
 !

 !
 !
  !
 !
 !
 !
r#   c              3   P   K   | ]!\  }}|dd                     ||          gV  "d S )Nz-oz{}={}r8   )rh   kvs      r$   	<genexpr>z1SSHOptions.to_ssh_options_list.<locals>.<genexpr>   sG        Aq= w~~a++, === r#   )r8   rZ   rW   r5   )r^   rd   ssh_key_options      r$   to_ssh_options_listzSSHOptions.to_ssh_options_list   s    */,,w*?*?&'15E$--2 !
 !
  M//11  !
 !
 !
 
 	
r#   N)__name__
__module____qualname__ra   rr   r"   r#   r$   rG   rG   q   sG        "% "% "% "%H .0 
 
 
 
 
 
 
r#   rG   c                      e Zd Zd Zd Zd Zd Z	 	 	 ddee         de	de	d	e	fd
Z
	 	 	 	 	 	 	 	 	 	 ddee         dede	deeeeef                           de	deeeef                  dedede	d	e	defdZd ZddZddZd ZdS )SSHCommandRunnerc                    t          j        |                                                                          }t          j        t	                                                                                                }	t
          j        dk    rd }
n0d                    |	d t                   |d t                             }
|| _	        || _
        || _        || _        || _        || _        |                    d          | _        |d         | _        |
| _        d | _        |                    dd           | _        t+          | j        | j        | j                  | _        d S )NrQ   z/tmp/ray_ssh_{}/{}ssh_private_keyssh_userssh_proxy_commandProxyCommand)hashlibsha1encode	hexdigestr   r[   r\   r8   HASH_MAX_LENGTHcluster_name
log_prefixprocess_runnernode_iduse_internal_ipprovidergetrz   r{   ssh_control_pathssh_ipr|   rG   ssh_options)r^   r   r   r   auth_configr   r   r   ssh_control_hashssh_user_hashr   s              r$   ra   zSSHCommandRunner.__init__   sD    #<(;(;(=(=>>HHJJWYY%5%5%7%788BBDD<7""#3::../1ABR?BR1S    )$,. */@AA#J/ 0!,1Dd!K!K% !/
 
 
r#   c                     | j         r| j                            | j                  S | j                            | j                  S rs   )r   r   internal_ipr   external_ipr^   s    r$   _get_node_ipzSSHCommandRunner._get_node_ip   s=     	;=,,T\:::=,,T\:::r#   c           	         |                                  }|t          j        d|           |S t          }t          j        d          5  t          j                    |k     r| j                            | j                  s|                                  }|#t          j        d|           |cd d d            S t          j	        dt          j        t          |                               t          j        |           t          j                    |k     r| j                            | j                  d d d            n# 1 swxY w Y   d S )Nz
Fetched IPzWaiting for IPReceivedz)Not yet available, retrying in {} seconds)r   r   labeled_valuer   grouptimer   is_terminatedr   printr   boldstrsleep)r^   deadlineipintervals       r$   _wait_for_ipzSSHCommandRunner._wait_for_ip   s     >$\2666I1.// 	% 	%)++((1L1L2 2( &&((>,Z<<<	% 	% 	% 	% 	% 	% 	% 	%  ?XAWAW   
8$$$ )++((1L1L2 2(	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% ts   	A#E9A>EE
Ec                    | j         d S t          j                    t          z   }t          | j        dz             5  |                     |          }t          j        |d ud           |
J d            	 d d d            n# 1 swxY w Y   || _         | j        Y	 t          j
        | j        dd           d S # t          $ r-}t          j        dt          |                     Y d }~d S d }~ww xY wd S )NzGot IPzCould not get node IP.zUnable to find IP of nodei  T)modeexist_okz{})r   r   r   r   r   r   r   doassertr   rX   makedirsOSErrorwarningr   )r^   r   r   es       r$   _set_ssh_ip_if_requiredz(SSHCommandRunner._set_ssh_ip_if_required   sT   ;"F 9;;!==do011 	? 	?""8,,B$0HIII>>#>>>>>		? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 
  ,1D1MMMMMM 1 1 1"4Q0000000001 -,s)   9BB	BB: :
C1"C,,C1F	final_cmdwith_outputexit_on_failsilentc                 B   	 |s`t          || j        |t                                t          j                                         t          j                                         S | j                            |          t          j                                         t          j                                         S # t          j	        $ r}d
                    |          }t                      st          dd|j        |          |r(t          j        d                    |                    dd}t!                      r|d	z  }t          j        |          dd}~ww xY w# t          j                                         t          j                                         w xY w)
a  Run a command that was already setup with SSH and `bash` settings.

        Args:
            final_cmd (List[str]):
                Full command to run. Should include SSH options and other
                processing that we do.
            with_output (bool):
                If `with_output` is `True`, command stdout will be captured and
                returned.
            exit_on_fail (bool):
                If `exit_on_fail` is `True`, the process will exit
                if the command fails (exits with a code other than 0).
            silent: If true, the command output will be silenced.

        Raises:
            ProcessRunnerError: If using new log style and disabled
                login shells.
            click.ClickException: If using login shells.
        )r   r   r    zCommand failedssh_command_failed)codecommandzCommand failed:

  {}
NzSSH command failed.z+ See above for the output from the failure.)r   r   r+   r[   stdoutflushstderrcheck_output
subprocessCalledProcessErrorr:   r   
returncodeclickClickExceptionr8   r   )r^   r   r   r   r   r   
joined_cmdfail_msgs           r$   _run_helperzSSHCommandRunner._run_helper   s   4$	  C)#'#6!%:%<%<	  < JJ1 *77	BB. JJ/ , 	? 	? 	?),,J(** ($(&	     ?*/66zBB  1')) N MMH*844$>%	?, JJs*   %B: $B: :E	BEEE   >FNx   rR   r4   r.   rd   port_forwardr/   run_envssh_options_override_ssh_keyshutdown_after_runreturnc           	         |	r|dz  }|r.| j         rt          || j                   }nt          |          }n| j        }t          |t                    s*J d                    t          |                                |                                  t                      rddg}ndg}|rt          j	        d          5  t          |t                    s|g}|D ]Z\  }}t          j        dt          j        |          t          j        |                     |dd	                    ||          gz  }[	 d d d            n# 1 swxY w Y   ||                    |
          z   d                    | j        | j                  gz   }|r:|rt#          ||          }t                      r|t%          |          z  }n||gz  }n|                    d           t          j        dt          j        |                     t          j                    5  t          j        dt          j        d                    |                               d d d            n# 1 swxY w Y   t          j        dk    rFt          j                    5  |                     ||||
          cd d d            S # 1 swxY w Y   d S |                     ||||
          S )N; sudo shutdown -h nowr}   z.ssh_options must be of type SSHOptions, got {}sshz-ttzForwarding portsz+Forwarding port {} to port {} on localhost.z-Lz{}:localhost:{}rc   z{}@{}z while true; do sleep 86400; doneRunning `{}`zFull command is `{}`r   r   r   )r|   rG   r   
isinstancer8   typer   r+   r   r   listverboser   r   rr   r{   r   r?   rE   r9   indentedvery_verboser:   	verbosityr   )r^   r.   rd   r   r   r   r/   r   r   r   r   r   r   localremoter   s                   r$   runzSSHCommandRunner.run@  s     	,++C' 	+% G(0t?U   ))EFF*K
 
 	V 	V;BB4CTCTUU	V 	V 
 	$$&&& "" 	%.CC'C 
	K!"455 	K 	K!,55 2$0>L%1 K KME6&E  
 D"3":":5&"I"IJJCCK	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K --g->>?~~dmT[99:; 	
  
	A$ N1#7LMM$&& #.s333		cU"		 ?@@@>273<<888 "" 	 	#&0C0C(D(D  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 !##$&&  ''{L (                   
 ##I{LQW#XXXs7   >A6EEE;II IJ--J14J1c                     |                     d          pg }|                     d          pg }d |D             }d |D             }d ||z   D             S )Nrsync_excludersync_filterc                     g | ]}d |gS )z	--excluder"   )rh   r   s     r$   rk   z>SSHCommandRunner._create_rsync_filter_args.<locals>.<listcomp>  s*     
 
 
-:[-(
 
 
r#   c                 <    g | ]}d d                     |          gS )z--filterzdir-merge,- {}rm   )rh   r   s     r$   rk   z>SSHCommandRunner._create_rsync_filter_args.<locals>.<listcomp>  s<     
 
 
 )00>>?
 
 
r#   c                     g | ]	}|D ]}|
S r"   r"   )rh   	args_listargs      r$   rk   z>SSHCommandRunner._create_rsync_filter_args.<locals>.<listcomp>  s%    UUU	9UUCUUUUr#   )r   )r^   optionsrsync_excludesrsync_filtersexclude_argsfilter_argss         r$   _create_rsync_filter_argsz*SSHCommandRunner._create_rsync_filter_args  s     _55;N339r
 
>L
 
 

 
 -
 
 
 VU\K%?UUUUr#   c                    |                                   |pi }t          j        dk    rIddg}|| j                            d          z  }||d                    | j        | j        |          gz  }n~dg}|dt          j	        d	g| j                            d          z             gz  }|d
gz  }|| 
                    |          z  }||d                    | j        | j        |          gz  }t          j        dt          j        d                    |                               |                     |t#                                 d S NrQ   scpz-rr   rc   z{}@{}:{}rsyncz--rshr   z-avzr   r   r   r   r   r[   r\   r   rr   r8   r{   r   r   list2cmdliner   r   r   r   r   r:   r   r%   r^   sourcetargetr   r   s        r$   run_rsync_upzSSHCommandRunner.run_rsync_up  s]   $$&&&-R <7""dmGt';;C;HHHG
 1 1$-f U UVVGGiG'Gd.BB3BOOO  G xGt55g5FFFG
 1 1$-f U UVVG>27388G3D3D+E+EFFF):):;;;;;r#   c                    |                                   t          j        dk    rIddg}|| j                            d          z  }|d                    | j        | j        |          |gz  }n~dg}|dt          j	        d	g| j                            d          z             gz  }|d
gz  }|| 
                    |          z  }|d                    | j        | j        |          |gz  }t          j        dt          j        d                    |                               |                     |t#                                 d S r   r   r   s        r$   run_rsync_downzSSHCommandRunner.run_rsync_down  sT   $$&&& <7""dmGt';;C;HHHG
))$-fMMvVVGGiG'Gd.BB3BOOO  G xGt55g5FFFG
))$-fMMvVVG>27388G3D3D+E+EFFF):):;;;;;r#   c                     | j         r&d                    | j         | j        | j                  S d                    | j        | j                  S )Nz&ssh -o IdentitiesOnly=yes -i {} {}@{}
z ssh -o IdentitiesOnly=yes {}@{}
)rz   r8   r{   r   r   s    r$   remote_shell_command_strz)SSHCommandRunner.remote_shell_command_str  sV     	<CC$dmT[   7==t{  r#   )FFF)
Nr   FNFNrR   r4   FFrs   )rt   ru   rv   ra   r   r   r   r   r   boolr   r   intr	   r   objectr   r   r   r   r   r"   r#   r$   rx   rx      s       $
 $
 $
L; ; ;  .1 1 16 ""> >9> > 	>
 > > > >D ""8<!=A,.#(NY NYc]NY NY 	NY
 tE#s(O45NY NY  (S&[(9:NY NY '*NY !NY NY 
NY NY NY NY`V V V< < < <2< < < <0    r#   rx   c                   z   e Zd Zd Z	 	 	 	 	 	 	 	 	 d$dee         ded	ed
eee	eef                           dedee
eef                  dedededefdZd%dZd%dZd Zd Zd Zd&dZdede
eef         defdZdede
eef         defdZdee         dee         fd Zdee         dee         fd!Zd"edefd#ZdS )'DockerCommandRunnerc                     t          di || _        |d         | _        || _        d | _        d| _        |                    dd          }|rdnd| _        d S )Ncontainer_nameF
use_podmanpodmandockerr"   )rx   ssh_command_runnerr   docker_confighome_dirinitializedr   
docker_cmd)r^   r   common_argsr   s       r$   ra   zDockerCommandRunner.__init__  sf    "2"A"A["A"A+,<=* "&&|U;;
&0>((hr#   Nr   FrR   r4   r.   rd   r   r   r   r/   r   r   r   r   c
                    |dk    r6t          |          r|                    | j                  dk    rdn| j        }|rt          ||          }|| j        k    rw|                     |d          }t                      r"d                    t          |                    }t          |g| j	        t                      | j                  d         }|	r|dz  }| j
                            ||||||	          S )
NrR   r   hostT)any_charr   r   with_interactiver  r   )rd   r   r   r   r   )r   findr  r?   _docker_expand_userr+   r:   rE   r   r   r   r   )
r^   r.   rd   r   r   r   r/   r   r   r   s
             r$   r   zDockerCommandRunner.run  s1    f S		%%(XXdo%>%>!%C%C _  ! 	J-c3HIICdo%%**3*>>C$&& 7hh05566"#2!6!8!8?	  
 C  	,++C &**%%#)E + 
 
 	
r#   c                    |pi }t           j                            |                     | j        j                  |                    d                    }t           j                            |                    d                    }t          j
        dk    rB|                    dd          }|                    dd          }|                    dd          }| j                            d| d| j        j         d| t                                 | j                            |||           |                                 r|                    d	d
          st           j                            |          r|dz  }t'          d                    t           j                            |                     |                              g| j        t/                      | j                  d         }| j                            d                    || j        || j        |                     |                    t                                 d S d S d S )N/rQ   \	mkdir -p  && chown -R r   r   r   docker_mount_if_possibleFz/.zmkdir -p {}r  r   z){} && rsync -e '{} exec -i' -avz {} {}:{})rX   pathr:   _get_docker_host_mount_locationr   r   lstripdirnamerstripr[   r\   replacer   r{   r%   r   _check_container_statusr   isdirr   r8   r
  r   r+   r  )r^   r   r   r   host_destinationhost_mount_locationprefixs          r$   r   z DockerCommandRunner.run_rsync_up  s{   -R7<<001H1UVVMM#
 
 !goo.>.E.Ec.J.JKK<7""^^D#..F"5"="=dC"H"H/77cBB##H+ H H&/H H2EH H"$$ 	$ 	
 	
 	
 	,,V5Ew,WWW'')) 	'++&3
 3
 	 w}}V$$ ) !D(  &!(((@(@(H(HII 
  $2!6!8!8?	 	 	 	F #'';BBO$',,V44  '(( ( 	 	 	 	 	+	 	 	 	r#   c                    |pi }t           j                            |                     | j        j                  |                    d                    }t           j                            |                    d                    }|	                    dd          }| j        
                    d| d| j        j         d| t                                 |d         dk    r|dz  }|                    d	d
          so| j        
                    d                    | j        | j        |                     |          |	                    dd                    t                                 | j                            |||           d S )Nr  r  r  r  r   r   .r  Fz,rsync -e '{} exec -i' -avz --delete {}:{} {}r   )rX   r  r:   r  r   r   r  r  r  r  r   r{   r%   r   r8   r  r   r
  r   )r^   r   r   r   host_sourcer  host_mount_location_unixs          r$   r   z"DockerCommandRunner.run_rsync_downI  s   -Rgll001H1UVVMM#
 
 !gook.@.@.E.EFF#6#>#>tS#I#I ##M0 M M&/M M2JM M"$$ 	$ 	
 	
 	

 ":cMF {{5u== 	 #''>EEO',,V44''c 	  '(( ( 
 
 
 	..{FG.TTTTTr#   c                     | j                                                             ddd                              d          }|d                    | j        | j                  z   S )Nr   zssh -tt   
z {} exec -it {} /bin/bash
)r   r   r  stripr8   r  r   )r^   	inner_strs     r$   r   z,DockerCommandRunner.remote_shell_command_strk  sa    #<<>>WUIq))U4[[ 	
 8??OT0
 
 
 	
r#   c                    d}| j                             d| j         d| dd          }|                                                                }||v sd|vrz| j        dk    rg d}nd	d
g}t
                              | j                                         d| j                                         dd                    |          z              d S d S )NNoExistzcommand -v z
 || echo ''Tr   r   )z2curl -fsSL https://get.docker.com -o get-docker.shzsudo sh get-docker.shzsudo usermod -aG docker $USERz sudo systemctl restart docker -fzsudo apt-get updatezsudo apt-get -y install podmanz  not installed. You can install z@ by adding the following commands to 'initialization_commands':
r#  )	r   r   r  decoder$  loggererror
capitalizer:   )r^   no_existoutputcleaned_outputinstall_commandss        r$   _check_docker_installedz+DockerCommandRunner._check_docker_installedu  s"   (,,@$/@@X@@@d - 
 
  ..00~%%)G)G(**$ $ $   *4$ 
 LL?--// E E?5577E E E )),--.     *H)Gr#   c                    | j         rdS | j                            t          | j        | j                  d                              d                                          }d|                                v od|                                vS )NTr)  utf-8truezno such object)	r  r   r   r   r   r  r*  r$  lower)r^   r/  s     r$   r  z+DockerCommandRunner._check_container_status  s     	4#''()<doNN  (   VG__UWW 	 'R,<FLLNN,RRr#   c                 ~   |                     d          }|dk    r| j        V| j                            | j         d| j         dd                              d                                          | _        |r|                    d| j        d	z             S |s"|d
k    r|                    d| j        d          S |S )N~r  z exec z printenv HOMETr)  r4  z~/r  r   r"  )	r	  r   r   r   r  r   r*  r$  r  )r^   stringr  user_poss       r$   r
  z'DockerCommandRunner._docker_expand_user  s    ;;s##b==}$+//? ( ($2E ( ( ($( 0  
 VG__UWW   =~~dDMC,?@@@ =(a--~~c4=!<<<r#   imagecleaned_bind_mountsc                     d}                      t           j         j                  dd                              d                                          }||k    rt          j        d j        ||                                 t           j         j                  dd                              d                                          }	 t          j
        |          }d |D             } fd|                                D             }||z
  }	|	rd}t          j        d	|	           n)# t          j        $ r t          j        d
           Y nw xY w|S )NFTr  )r   r   r4  z[A container with name {} is running image {} instead of {} (which was provided in the YAML)c                 D    h | ]}|d                               d          S )Destinationr  )r$  )rh   mnts     r$   	<setcomp>zLDockerCommandRunner._check_if_container_restart_is_needed.<locals>.<setcomp>  s8     $ $ $25M"((--$ $ $r#   c                 `    h | ]*}                     |                              d           +S )r  )r
  r$  )rh   r   r^   s     r$   rA  zLDockerCommandRunner._check_if_container_restart_is_needed.<locals>.<setcomp>  sE     ' ' ' ((0066s;;' ' 'r#   z~This Docker Container is already running. Restarting the Docker container on this node to pick up the following file_mounts {}z`Unable to check if file_mounts specified in the YAML differ from those on the running container.)r   r   r   r  r*  r$  r   r,  r   r6   loadskeysr   JSONDecodeErrorr   )
r^   r;  r<  re_init_requiredrunning_imagemountsactive_mountsactive_remote_mountsrequested_remote_mountsunfulfilled_mountss
   `         r$   %_check_if_container_restart_is_neededz9DockerCommandRunner._check_if_container_restart_is_needed  s    !HH"4#6HH    
 VG__UWW 	 E!!;#   HH%d&94?KK    
 VG__UWW 		 Jv..M$ $9F$ $ $ ' ' ' '16688' ' '# "9;O!O! #' "H '	   # 	 	 	>    	
  s   AD0 0#EEas_headfile_mountssync_run_yetc                	   ddg}| j                             |rdnd d| j                             d                    }|                                  | j                             dd          r=|s
J d	            |                     d
                    | j        |          d           n*|                     | j         d| d| j         d|            |                                }|D ]}|                    |d            d}|                                 }	d}
|	r[| 	                    ||          }
|
rC| j         d| j
         }t                              d|           |                     |d           |	r|
r|sdS | j                            | j         ddz   |z   d                                                                          }d}	 t!          j        |          D ]4}|                    d          r|                    d          d         } n5n0# t           j        $ r}t+          j        d|            |d }~ww xY w| j                             dg           | j                             |rdnd dg           z   }t/          | j        j        ||| j
        |                     |                     |                    | j        j        || j                  }|                     |d           d}|D ]}||v r|s|                     ||         |           | j                            d                    | j        t:          j                            |                      | j        j                  |          !                    dd          | j
        | "                    |                               	 |                     d | d!|            # tF          $ r |                     d"| d                              d#                                          }|                    d          d$         }|                    d          d%         }|                    d          d&         }|                     d'd                              d#                                          }t+          j$        d(| d)| d*| d+| d,| d-| d| d.           Y w xY wd| _%        |S )/Nz~/ray_bootstrap_config.yamlz~/ray_bootstrap_key.pemheadworker_imager;  pull_before_runTz@Image must be included in config if pull_before_run is specifiedz
{} pull {}r  )r   z image inspect z 1> /dev/null  2>&1 || z pull Fz stop zExecuting Docker command: %sr   z"inspect -f '{{json .Config.Env}}' r)  z/rootzHOME=r"  zHUnable to deserialize `image_env` to Python object. The `image_env` is:
run_options_run_optionsz5rsync -e '{cmd} exec -i' -avz {src} {container}:{dst}r  r  )r.   src	containerdstzcat z1 >/dev/null 2>&1 || sudo chown $(id -u):$(id -g) zls -l r4  r      r   whoamizFile (z) is owned by user:z and group:z with permissions (z). The current user (z) does not have permission to read these files, and Ray may not be able to autoscale. This can be resolved by installing `sudo` in your container, or adding a command like 'chown z' to your `setup_commands`.)&r   r   r2  r   r8   r  copypopr  rM  r   r+  infor   r*  r$  r6   rC  
startswithsplitrE  r   r,  r   r{   _configure_runtime_auto_configure_shmr   r   rX   r  r:   r  r  r
  	Exceptionr   r  )r^   rN  rO  rP  BOOTSTRAP_MOUNTSspecific_imager<  r@  docker_run_executedcontainer_runningrequires_re_initdocker_stop_cmd	image_envhome_directoryenv_varr   user_docker_run_optionsstart_commandmount
lsl_stringpermissionsownerr   current_users                           r$   run_initzDockerCommandRunner.run_init  ss    :;TU+// .vvh6668J8N8Nw8W8W
 
 	$$&&&!!"3T:: 	!  W > HH##DO^DDf      HH? ; ;> ; ;?; ;*8; ;   *..00# 	/ 	/C##C....# 88::  	:#II 3      :%)_"Q"QD<O"Q"Q:OLLL&999! 1	'&6 1	' 
 t '++))):;$% !%	 ,     %N
#z)44  G))'22 )0w)?)?)B '    8,58 8    '+&8&<&<r' '"&&$266(@@@" '#
 .'0##'',,-DEE  '4 M HH]FH333"& & 4	 4	E### A %%k%&8%@@@'++KRR OGLL @@ $ 7 D  "	 
 "' #  #'"5 44U;; S     HH@u @ @8=@ @    !   !1%!1!1tDD  #-"2"23"7"7":K&,,S11!4E&,,S11!4Et<<CCGLLRRTT ! &1 1 15 1 1 1 15@1 1)51 1 0<1 1 ?D1 1 1	 	 	 	 	9 $h  ""s,   A	H I
,II
O  D S$#S$rV  c                    | j                             d          r|S | j                            | j         ddz   d                                                                          }d|v rs	 | j                            dd           |d	gz   S # t          $ rC}t          	                    d
           t          
                    d|            |cY d }~S d }~ww xY w|S )N#disable_automatic_runtime_detectionr   zinfo -f '{{.Runtimes}}' Tr)  znvidia-container-runtimez
nvidia-smiFz--runtime=nvidiaz7NVIDIA Container Runtime is present, but no GPUs found.znvidia-smi error: )r   r   r   r   r  r*  r$  rd  r+  r   debug)r^   rV  runtime_outputr   s       r$   rb  z&DockerCommandRunner._configure_runtime  s   !!"GHH 	 #''?%%%(BBPT (   VXXUWW 	 &77#'++Le+LLL"&8%999 # # #M   5!55666"""""""# s   -!B 
C8CCCc                 n   | j                             d          r|S |D ]'}d|v r!t                              d|            |c S (	 | j                            dd                                                                          }t          d |	                    d          D             d	         	                                d
                   }|dz  }t          |t          z  dz  t                    }|d| dgz   S # t          $ r)}t                              d|            |cY d }~S d }~ww xY w)Ndisable_shm_size_detectionz
--shm-sizez;Bypassing automatic SHM-Detection because of `run_option`: zcat /proc/meminfo || trueTr)  c                     g | ]}d |v |	S )MemAvailabler"   )rh   lns     r$   rk   z;DockerCommandRunner._auto_configure_shm.<locals>.<listcomp>  s"    MMM"8L8L8L8L8Lr#   r#  r   r"  i   g?z--shm-size='zb'z5Received error while trying to auto-compute SHM size )r   r   r+  r_  r   r   r*  r$  r   ra  minr   r
   rd  r   )r^   rV  run_opt
shm_outputavailable_memoryavailable_memory_bytesshm_sizer   s           r$   rc  z'DockerCommandRunner._auto_configure_shm  s   !!">?? 	" 	# 	#Gw&&/%,/ /   #""" '	'++/T ,      #MMj..t44MMM%''!   
 &6%<"'*PPSVV5 H "="="="=!>>> 	 	 	NNVSTVVWWW	s   B8D 
D4D/)D4/D4r   c                 $    ddl m}  ||          S )z0Return the docker host mount directory location.r   )get_docker_host_mount_location)ray.autoscaler.sdkr  )r^   r   r  s      r$   r  z3DockerCommandRunner._get_docker_host_mount_location  s(     	FEEEEE--l;;;r#   )	Nr   FNFNrR   r4   Frs   )F)rt   ru   rv   ra   r   r   r   r   r   r	   r   r   r   r   r   r   r2  r  r
  rM  ru  rb  rc  r  r"   r#   r$   r   r     s:       ? ? ? ""8<!=A,.#(-
 -
c]-
 -
 	-

 tE#s(O45-
 -
  (S&[(9:-
 -
 '*-
 !-
 
-
 -
 -
 -
^2 2 2 2h U  U  U  UD
 
 
  6S S S   ,6 6 /3CH~6 	6  6  6  6 pU#U#-1#s(^U#KOU# U# U# U#nd3i DI    0 tCy  T#Y        D<C <C < < < < < <r#   r   )Ar   r6   loggingrX   r   r[   r   getpassr   shlexr   typingr   r   r   r	   r   ray._private.ray_constantsr
   "ray.autoscaler._private.cli_loggerr   r   !ray.autoscaler._private.constantsr   r   r   ray.autoscaler._private.dockerr   r   r   r   r   !ray.autoscaler._private.log_timerr   .ray.autoscaler._private.subprocess_output_utilr   r   r   ray.autoscaler.command_runnerr   	getLoggerrt   r+  r   r  r:   r  abspath__file__KUBECTL_RSYNCMAX_HOME_RETRIESHOME_RETRY_DELAY_Sr!   r%   r)   r+   r   r-   r   r   r?   rE   rG   rx   r   r"   r#   r$   <module>r     s!      				     



              . . . . . . . . . . . .  L L L L L L = = = = = = = =         
              7 6 6 6 6 6         
 A @ @ @ @ @		8	$	$ GOOBGOOH--..0N    #T
:
:# # #" " "' ' '& & & & &(S c6kAR    &E E E0
 0
 0
 0
 0
 0
 0
 0
ft t t t t- t t tn	g< g< g< g< g<0 g< g< g< g< g<r#   