
    &`iO1                        d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
mZmZmZmZ d dlZd dlmZmZmZmZ d dlmZmZmZmZmZ d dlmZ d d	lmZ d d
lm Z   ej!        e"          Z#ddZ$ G d de           Z%dS )    Nwraps)RLock)
ModuleType)AnyDictListOptionalTuple)bootstrap_gcp&construct_clients_from_provider_configget_node_typetpu_accelerator_config_to_type)GCPTPU
GCPComputeGCPNodeGCPNodeTypeGCPResource)TPUCommandRunner)CommandRunnerInterface)NodeProvider      c                 D     t                      fd            }|S )a!  Retry decorator for methods of GCPNodeProvider.

    Upon catching BrokenPipeError, API clients are rebuilt and
    decorated methods are retried.

    Work-around for https://github.com/ray-project/ray/issues/16072.
    Based on https://github.com/kubeflow/pipelines/pull/5250/files.
    c                     d}|k     rt	  | g|R i |S # t           $ rR t                              d           |dz  }|k     r)|                                  t	          j                   n Y nw xY w|k     rd S d S )Nr   z#Caught a BrokenPipeError. Retrying.r   )BrokenPipeErrorloggerwarning_construct_clientstimesleep)selfargskwargs	try_count	backoff_s	max_triesmethods       }/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/gcp/node_provider.pymethod_with_retriesz#_retry.<locals>.method_with_retries-   s    	)##	vd4T444V444"   DEEEQ	y((++---Jy)))) *) )######s    AA32A3r   )r(   r'   r&   r*   s   ``` r)   _retryr+   #   sB     6]]      ]     c                       e Zd ZdedefdZd ZdedefdZe	defd	            Z
d
efdZd
efdZd
efdZe	d
edefd            Zd
efdZd
efdZe	dedededeeef         fd            Zd
efdZe	d
efd            Ze	d
edefd            Zd
edefdZed             Zedeeef         deeef         fd            Z	 d#ded
edeeef         deded ed!eeeef                  de f fd"Z! xZ"S )$GCPNodeProviderprovider_configcluster_namec                     t          j        | ||           t                      | _        |                                  |                    dd          | _        i | _        d S )Ncache_stopped_nodesF)r   __init__r   lockr   getr2   cached_nodes)r"   r/   r0   s      r)   r3   zGCPNodeProvider.__init__@   s]    dO\BBBGG	!!!#2#6#67Le#T#T  13r,   c                 J   t          | j                  \  }}}}i | _        t          || j        d         | j        d         | j                  | j        t
          j        <   |At          || j        d         | j        d         | j                  | j        t
          j        <   d S d S )N
project_idavailability_zone)	r   r/   	resourcesr   r0   r   COMPUTEr   TPU)r"   _computetpus       r)   r   z"GCPNodeProvider._construct_clientsJ   s    C 
 
1gs :< /9 . !45	/
 /
{*+ ?.4$\2$%89!	/ /DN;?+++ ?r,   	node_namereturnc                 @    | j         t          j        |                   S )zReturn the resource responsible for the node, based on node_name.

        This expects the name to be in format '[NAME]-[UUID]-[TYPE]',
        where [TYPE] is either 'compute' or 'tpu' (see ``GCPNodeType``).
        )r:   r   name_to_type)r"   r@   s     r)   $_get_resource_depending_on_node_namez4GCPNodeProvider._get_resource_depending_on_node_named   s     ~k6yAABBr,   tag_filtersc                     | j         5  g }| j                                        D ]}|                    |          }||z  }d |D             | _        d |D             cd d d            S # 1 swxY w Y   d S )Nc                      i | ]}|d          |S name .0is     r)   
<dictcomp>z8GCPNodeProvider.non_terminated_nodes.<locals>.<dictcomp>v   s     A A A!6A A A Ar,   c                     g | ]
}|d          S rH   rJ   rK   s     r)   
<listcomp>z8GCPNodeProvider.non_terminated_nodes.<locals>.<listcomp>w   s    111!AfI111r,   )r4   r:   valueslist_instancesr6   )r"   rE   	instancesresourcenode_instancess        r)   non_terminated_nodesz$GCPNodeProvider.non_terminated_nodesl   s    Y 		2 		2I N1133 , ,!)!8!8!E!E^+		 !B Ay A A AD11y111		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2 		2s   AA**A.1A.node_idc                     | j         5  |                     |          }|                                cd d d            S # 1 swxY w Y   d S N)r4   _get_cached_node
is_runningr"   rW   nodes      r)   r[   zGCPNodeProvider.is_runningy       Y 	% 	%((11D??$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%   )>AAc                     | j         5  |                     |          }|                                cd d d            S # 1 swxY w Y   d S rY   )r4   rZ   is_terminatedr\   s      r)   ra   zGCPNodeProvider.is_terminated~   s    Y 	( 	(((11D%%''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(r_   c                     | j         5  |                     |          }|                                cd d d            S # 1 swxY w Y   d S rY   )r4   rZ   
get_labelsr\   s      r)   	node_tagszGCPNodeProvider.node_tags   r^   r_   tagsc                     | j         5  |}|                     |          }|                     |          }|                    ||          }|cd d d            S # 1 swxY w Y   d S )N)r]   labels)r4   	_get_noderD   
set_labels)r"   rW   re   rg   r]   rT   results          r)   set_node_tagszGCPNodeProvider.set_node_tags   s    Y 	 	F>>'**D@@IIH((d6(BBF	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AAA!Ac                     | j         5  |                     |          }|                                }|)|                     |          }|                                }|cd d d            S # 1 swxY w Y   d S rY   )r4   rZ   get_external_iprh   r"   rW   r]   ips       r)   external_ipzGCPNodeProvider.external_ip       Y 	 	((11D%%''Bz~~g..))++	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	   AA++A/2A/c                     | j         5  |                     |          }|                                }|)|                     |          }|                                }|cd d d            S # 1 swxY w Y   d S rY   )r4   rZ   get_internal_iprh   rn   s       r)   internal_ipzGCPNodeProvider.internal_ip   rq   rr   base_configcountc                    | j         5  |}t          |          }| j        |         i }| j        rm|d         |d         |d         d}                    |d          d|         }|r5fd|D             }	|                    |	           |t          |          z  }|dk    r8                    |||          }
d	 |
D             }|                    |           ddd           n# 1 swxY w Y   |S )
zCreates instances.

        Returns dict mapping instance id to each create operation result for the created
        instances.
        ray-node-nameray-node-typeray-user-node-type)ry   rz   r{   TNc                 T    i | ]$}|d                               |d                    %S rH   )start_instance)rL   nrT   s     r)   rN   z/GCPNodeProvider.create_node.<locals>.<dictcomp>   sA     ) ) ) &	8#:#:1V9#E#E) ) )r,   r   c                     i | ]\  }}||	S rJ   rJ   )rL   rj   instance_ids      r)   rN   z/GCPNodeProvider.create_node.<locals>.<dictcomp>   s+     & & &,?FKK& & &r,   )r4   r   r:   r2   rR   updatelencreate_instances)r"   rv   re   rw   rg   	node_type	all_nodesfiltersreuse_nodesreused_nodes_dictresultscreated_nodes_dictrT   s               @r)   create_nodezGCPNodeProvider.create_node   s    Y 	5 	5F%k22I~i0HI' .%+O%<%+O%<*01E*F 
 '55gtDDVeVL .) ) ) )!,) ) )% $$%6777S---Eqyy2:2K2K3 3& &CJ& & &"   !3444=	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5 	5@ s   CC((C,/C,c                 d   t                               d                    |                     |                     |          }	 |                    |          }nU# t
          j        j        $ r>}|j        j	        dk    r!t           
                    d| d           d }n|d Y d }~nd }~ww xY w|S )Nz"NodeProvider: {}: Terminating noderW     !Tried to delete the node with id  but it was already gone.)r   infoformatrD   delete_instancegoogleapiclienterrors	HttpErrorrespstatusr   )r"   rW   rT   rj   
http_errors        r)   _thread_unsafe_terminate_nodez-GCPNodeProvider._thread_unsafe_terminate_node   s     	8??HHIII<<WEE	+-- .  FF %/ 	+ 	+ 	+%,,/ / / /    d* 	+ s   A B-/4B((B-c                    | j         5  |                     |          }	 | j        rC|                     |          }|                                r|                    |          }nd }n|                    |          }nS# t          j        j	        $ r<}|j
        j        dk    rt                              d| d           n|d Y d }~nd }~ww xY w|cd d d            S # 1 swxY w Y   d S )Nr   r   r   r   )r4   rD   r2   rZ   r[   stop_instancer   r   r   r   r   r   r   r   )r"   rW   rT   r]   rj   r   s         r)   terminate_nodezGCPNodeProvider.terminate_node   s   Y 	 	@@IIH/+ 	0099D(( &!)!7!7!7!H!H!%%55 ' 6  F #)3 / / /?)S00NN3G 3 3 3   
 %$.    / +	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s;   C!A B ?C! C2CC!CC!!C%(C%c                    |                      i            | j        5  || j        v r| j        |         cd d d            S |                     |          }|                    |          }|cd d d            S # 1 swxY w Y   d S )Nr   )rV   r4   r6   rD   get_instance)r"   rW   rT   instances       r)   rh   zGCPNodeProvider._get_node   s    !!"%%%Y 	 	$+++(1	 	 	 	 	 	 	 	 @@IIH,,W,==H	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   A9 ,A99A= A=c                 X    || j         v r| j         |         S |                     |          S rY   )r6   rh   )r"   rW   s     r)   rZ   z GCPNodeProvider._get_cached_node  s0    d'''$W--~~g&&&r,   c                      t          |           S rY   )r   )cluster_configs    r)   bootstrap_configz GCPNodeProvider.bootstrap_config  s    ^,,,r,   r   c                 X   d| vr| S t          j        |           } | d         }|D ]}||         d         }t          |          t          j        k    rWi }d}d|v r	|d         }nd|v rt          |d                   }|sZd|d| d<   ||         d	                             |           | S )
a  Fill out TPU resources to the cluster config.

        To enable TPU pod autoscaling, we provide the TPU accelerator
        type as a resource that only exists on worker 0 of the pod slice.
        For instance, a v4-16 should have the resource labels:
            worker 0: resources = {"TPU": 4, "TPU-v4-16-head": 1}
            worker 1: resources = {"TPU": 4}

        For the autoscaler to correctly process the demands of
        creating a new TPU pod, then the autoscaler must know what
        a TPU pod is in the form of the TPU accelerator resource.

        Therefore we fill out TPU pods appropriately by providing the
        expected resource which we can deduce from the cluster config.

        available_node_typesnode_config acceleratorTypeacceleratorConfigr   zTPU-z-headr:   )copydeepcopyr   r   r<   r   r   )r   r   r   r   autodetected_resourcesaccelerator_types         r)   &fillout_available_node_types_resourcesz6GCPNodeProvider.fillout_available_node_types_resources  s    ( "77!!~66-.DE- 	 	I.y9-HK[))[_<<)+&#% $33'23D'E$$(K77'E#$78( ($ ( IJ&'E.>'E'E'EF$Y/<CC*   r,   N
log_prefixauth_configprocess_runneruse_internal_ipdocker_configc                    |                      |          }|                    |          }	|||||||d}
t          j        | j        v r*|| j        t          j                 k    rt          d|	| d|
S  t                      j        di |
S )z+Returns a TPU command runner as applicable.)r   r   rW   r   r0   r   r   )r   providerNrJ   )rD   r   r   r<   r:   r   superget_command_runner)r"   r   rW   r   r0   r   r   r   rT   r   common_args	__class__s              r)   r   z"GCPNodeProvider.get_command_runnerB  s     <<WEE((11*$&(,.
 
 Ot~--DN;?;;;#TXTTTTT-577-<<<<<r,   rY   )#__name__
__module____qualname__dictstrr3   r   r   rD   r+   rV   r[   ra   rd   rk   rp   ru   intr   r   r   r   r   rh   rZ   staticmethodr   r   r   r   boolr
   r   r   __classcell__)r   s   @r)   r.   r.   ?   s       3 3C 3 3 3 3  4Cc Ck C C C C 
2 
2 
2 
2 V
2%# % % % %
(S ( ( ( (
% % % % %
 	S 	 	 	 	 V		3 	 	 	 		3 	 	 	 	 &t &4 & &SRVY & & & V&PS    ( c    V0 
 
 
 
 
 V
' ' ' ' ' ' - - \- (S#X(	c3h( ( ( \(d 37= == = #s(^	=
 = #= =  S#X/= 
 = = = = = = = = = =r,   r.   )r   r   )&r   loggingr    	functoolsr   	threadingr   typesr   typingr   r   r	   r
   r   r   "ray.autoscaler._private.gcp.configr   r   r   r    ray.autoscaler._private.gcp.noder   r   r   r   r   .ray.autoscaler._private.gcp.tpu_command_runnerr   ray.autoscaler.command_runnerr   ray.autoscaler.node_providerr   	getLoggerr   r   r+   r.   rJ   r,   r)   <module>r      s                        3 3 3 3 3 3 3 3 3 3 3 3 3 3                             L K K K K K @ @ @ @ @ @ 5 5 5 5 5 5		8	$	$   8_= _= _= _= _=l _= _= _= _= _=r,   