
    &`ii              	       6   d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZmZmZ ddlmZ ddlZdd	lmZ dd
lmZ ddlmZ ddlmZmZ  ej         e!          Z"dZ#dZ$dZ%e%dz  Z&dZ'de%e'fdee(ee(         f         dee)         de*de*fdZ+de,de)de)fdZ- G d de
          Z. G d deej/                  Z0 G d d e0          Z1 G d! d"e0          Z2 G d# d$ej/                  Z3 G d% d&e3          Z4 G d' d(e3          Z5dS ))a  Abstractions around GCP resources and nodes.

The logic has been abstracted away here to allow for different GCP resources
(API endpoints), which can differ widely, making it impossible to use
the same logic for everything.

Classes inheriting from ``GCPResource`` represent different GCP resources -
API endpoints that allow for nodes to be created, removed, listed and
otherwise managed. Those classes contain methods abstracting GCP REST API
calls.
Each resource has a corresponding node type, represented by a
class inheriting from ``GCPNode``. Those classes are essentially dicts
with some extra methods. The instances of those classes will be created
from API responses.

The ``GCPNodeType`` enum is a lightweight way to classify nodes.

Currently, Compute and TPU resources & nodes are supported.

In order to add support for new resources, create classes inheriting from
``GCPResource`` and ``GCPNode``, update the ``GCPNodeType`` enum,
update the ``_generate_node_name`` method and finally update the
node provider.
    N)UserDict)deepcopy)Enumwraps)AnyDictListOptionalTupleUnion)uuid4)AuthorizedHttp)Resource)	HttpError)TAG_RAY_CLUSTER_NAMETAG_RAY_NODE_NAME@            	exceptionregexmax_retriesretry_interval_sc                       fd}|S )zDRetry a function call n-times for as long as it throws an exception.c                 F     t                      fd            }|S )Nc                        fd}t                    D ]7} |            }t          |t                    s nt          j        	           8t          |t                    r||S )Nc                      	  i } | S # t           $ rB}t          |          r$r$t          j        t	          |                    s||cY d }~S d }~ww xY wN)	Exception
isinstanceresearchstr)valueeargsr   funckwargsr   s     t/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/gcp/node.pytry_catch_exczH_retry_on_exception.<locals>.dec.<locals>.wrapper.<locals>.try_catch_excC   s     D$1&11E L    %a33   &(is1vv&>&>   HHHHHHs   	 
A7AAA)ranger"   r!   timesleep)
r(   r*   r,   _retr   r)   r   r   r   s
   ``   r+   wrapperz1_retry_on_exception.<locals>.dec.<locals>.wrapperA   s    	 	 	 	 	 	 	 	 	 ;'' - -#moo!#y11 E
+,,,,#y)) 	J    r   )r)   r2   r   r   r   r   s   ` r+   decz _retry_on_exception.<locals>.dec@   sH    	t	 	 	 	 	 	 	 	 
	* r3    )r   r   r   r   r4   s   ```` r+   _retry_on_exceptionr6   8   s5           2 Jr3   labelsnode_suffixreturnc                     | t                    }t          |          t          t          z
  dz
  k    sJ |t          |          f            | dt	                      j        dt                    d| S )a  Generate node name from labels and suffix.

    This is required so that the correct resource can be selected
    when the only information autoscaler has is the name of the node.

    The suffix is expected to be one of 'compute' or 'tpu'
    (as in ``GCPNodeType``).
       -N)r   lenINSTANCE_NAME_MAX_LENINSTANCE_NAME_UUID_LENr   hex)r7   r8   
name_labels      r+   _generate_node_namerB   \   s{     )*Jz??47MMPQQRRRJURRR OO577;'>(>'>?OO+OOOr3   c                   N    e Zd ZdZdZdZed
d            Zedefd            Z	d	S )GCPNodeTypez'Enum for GCP node types (compute & tpu)computetpunodeGCPNodec                     t          | t                    rt          j        S t          | t                    rt          j        S t          dt          |            d          )z,Return GCPNodeType based on ``node``'s classzWrong GCPNode type .)r"   
GCPTPUNoderD   TPUGCPComputeNodeCOMPUTE	TypeErrortype)rG   s    r+   from_gcp_nodezGCPNodeType.from_gcp_nodes   sY     dJ'' 	#?"dN++ 	'&&;d4jj;;;<<<r3   namec                 R    t          |                     d          d                   S )zProvided a node name, determine the type.

        This expects the name to be in format '[NAME]-[UUID]-[TYPE]',
        where [TYPE] is either 'compute' or 'tpu'.
        r<   )rD   splitrR   s    r+   name_to_typezGCPNodeType.name_to_type|   s!     4::c??2.///r3   N)rG   rH   )
__name__
__module____qualname____doc__rN   rL   staticmethodrQ   r%   rW   r5   r3   r+   rD   rD   m   si        11G
C= = = \= 03 0 0 0 \0 0 0r3   rD   c                        e Zd ZdZdZdZdZdeddddf fdZde	fdZ
de	fd	Zej        defd
            Zej        defd            Zej        defd            ZdefdZ xZS )rH   z(Abstraction around compute and tpu nodesN	base_dictresourceGCPResourcer9   c                      t                      j        |fi | || _        t          | j        t                    sJ d S r    )super__init__r_   r"   r`   )selfr^   r_   r*   	__class__s       r+   rc   zGCPNode.__init__   sG    --f--- $-5555555r3   c                 D    |                      | j                  | j        v S r    )getSTATUS_FIELDRUNNING_STATUSESrd   s    r+   
is_runningzGCPNode.is_running   s    xx)**d.CCCr3   c                 D    |                      | j                  | j        vS r    )rg   rh   NON_TERMINATED_STATUSESrj   s    r+   is_terminatedzGCPNode.is_terminated   s    xx)**$2NNNr3   c                     d S r    r5   rj   s    r+   
get_labelszGCPNode.get_labels       r3   c                     d S r    r5   rj   s    r+   get_external_ipzGCPNode.get_external_ip   rq   r3   c                     d S r    r5   rj   s    r+   get_internal_ipzGCPNode.get_internal_ip   rq   r3   c                 N    d| j         j         d|                     d           dS )N<z: rR   >)re   rX   rg   rj   s    r+   __repr__zGCPNode.__repr__   s,    A4>*AAdhhv.>.>AAAAr3   )rX   rY   rZ   r[   rm   ri   rh   dictrc   boolrk   rn   abcabstractmethodrp   r%   rs   ru   ry   __classcell__)re   s   @r+   rH   rH      sN       22"L6$ 6- 6d 6 6 6 6 6 6
DD D D D DOt O O O O 	D     	     	    B# B B B B B B B Br3   rH   )	metaclassc                   P    e Zd ZdZh dZddhZdhZdZdefdZ	de
fd	Zde
fd
ZdS )rM   z Abstraction around compute nodes>   RUNNINGSTAGINGPROVISIONING
TERMINATED	SUSPENDEDr   statusr9   c                 .    |                      di           S Nr7   rg   rj   s    r+   rp   zGCPComputeNode.get_labels       xx"%%%r3   c                     |                      di g          d                              di g          d                              dd           S )NnetworkInterfacesr   accessConfigsnatIPr   rj   s    r+   rs   zGCPComputeNode.get_external_ip   sG    HH(2$//2S2$''+S$	
r3   c                 b    |                      di g          d                              d          S )Nr   r   	networkIPr   rj   s    r+   ru   zGCPComputeNode.get_internal_ip   s,    xx+bT221599+FFFr3   N)rX   rY   rZ   r[   rm   TERMINATED_STATUSESri   rh   rz   rp   r%   rs   ru   r5   r3   r+   rM   rM      s        ** EDD'5!{L&D & & & &
 
 
 
 
G G G G G G Gr3   rM   c                       e Zd ZdZh dZdhZdZdefdZe	de
fd            Zdee         fdZdd
e
defdZdee         fdZdd
e
defdZdS )rK   zAbstraction around tpu nodes>   READYCREATINGSTARTING
RESTARTINGr   stater9   c                 .    |                      di           S r   r   rj   s    r+   rp   zGCPTPUNode.get_labels   r   r3   c                 J    t          |                     di g                    S NnetworkEndpoints)r=   rg   rj   s    r+   num_workerszGCPTPUNode.num_workers   s!    488.55666r3   c                 0    |                      di g          S r   r   rj   s    r+   get_external_ipszGCPTPUNode.get_external_ips       xx*RD111r3   r   worker_indexc                     |                                  |                             di                               dd           S )NaccessConfig
externalIp)r   rg   rd   r   s     r+   rs   zGCPTPUNode.get_external_ip   s9    !!##L1S$$St$$	
r3   c                 0    |                      di g          S r   r   rj   s    r+   get_internal_ipszGCPTPUNode.get_internal_ips   r   r3   c                 ^    |                                  |                             dd           S )N	ipAddress)r   rg   r   s     r+   ru   zGCPTPUNode.get_internal_ip   s)    $$&&|488dKKKr3   N)r   )rX   rY   rZ   r[   rm   ri   rh   rz   rp   propertyintr   r
   r%   r   rs   r   ru   r5   r3   r+   rK   rK      s       && NMMyL&D & & & & 7S 7 7 7 X72$s) 2 2 2 2
 
C 
 
 
 
 
2$s) 2 2 2 2L LC L L L L L L Lr3   rK   c                   N   e Zd ZdZdededededdf
dZej        d	e	de	fd
            Z
ej        eefdedededefd            Zej        	 	 d"dee         deded         fd            Zej        deddfd            Zej        	 d#dedededefd            Zej        	 d#dedededeeef         fd            Z	 d#dededededeeeef                  f
dZej        d#dededefd            Zej        d#dededefd             Zej        d#dededefd!            ZdS )$r`   z,Abstraction around compute and TPU resourcesr_   
project_idavailability_zonecluster_namer9   Nc                 >    || _         || _        || _        || _        d S r    )r_   r   r   r   )rd   r_   r   r   r   s        r+   rc   zGCPResource.__init__   s(     !$!2(r3   httpc                     dS )@Generate a new AuthorizedHttp object with the given credentials.Nr5   )rd   r   s     r+   get_new_authorized_httpz#GCPResource.get_new_authorized_http   	     	r3   	operation	max_pollspoll_intervalc                     dS )z8Waits a preset amount of time for operation to complete.Nr5   )rd   r   r   r   s       r+   wait_for_operationzGCPResource.wait_for_operation   s	     tr3   Flabel_filtersrn   rH   c                     dS )zReturns a filtered list of all instances.

        The filter removes all terminated instances and, if ``label_filters``
        are provided, all instances which labels are not matching the
        ones provided.
        Nr5   )rd   r   rn   s      r+   list_instanceszGCPResource.list_instances   s	     	r3   node_idc                     dS )zReturns a single instance.Nr5   )rd   r   s     r+   get_instancezGCPResource.get_instance  r   r3   TrG   r7   r   c                     dS )zbSets labels on an instance and returns result.

        Completely replaces the labels dictionary.Nr5   )rd   rG   r7   r   s       r+   
set_labelszGCPResource.set_labels  s	     	r3   base_configc                     dS )zgCreates a single instance and returns result.

        Returns a tuple of (result, node_name).
        Nr5   )rd   r   r7   r   s       r+   create_instancezGCPResource.create_instance  s	     	r3   countc                 l      fdt          |          D             }|r fd|D             }n|}|S )zqCreates multiple instances and returns result.

        Returns a list of tuples of (result, node_name).
        c                 @    g | ]}                     d           S )Fr   )r   ).0ir   r7   rd   s     r+   
<listcomp>z0GCPResource.create_instances.<locals>.<listcomp>/  s>     
 
 
   f OO
 
 
r3   c                 D    g | ]\  }}                     |          |fS r5   r   )r   r   	node_namerd   s      r+   r   z0GCPResource.create_instances.<locals>.<listcomp>5  s@       (Iy ((33Y?  r3   )r-   )rd   r   r7   r   r   
operationsresultss   ```    r+   create_instanceszGCPResource.create_instances$  s}    
 
 
 
 
 
5\\
 
 


  	!   ,6  GG
 !Gr3   c                     dS z'Deletes an instance and returns result.Nr5   rd   r   r   s      r+   delete_instancezGCPResource.delete_instance>  r   r3   c                     dS r   r5   r   s      r+   stop_instancezGCPResource.stop_instanceC  r   r3   c                     dS )z,Starts a single instance and returns result.Nr5   r   s      r+   start_instancezGCPResource.start_instanceH  r   r3   NFT)rX   rY   rZ   r[   r   r%   rc   r|   r}   r   r   	MAX_POLLSPOLL_INTERVALrz   r   r   r   r{   r
   r   r   rH   r   r   r   r   r   r   r   r5   r3   r+   r`   r`      s&       66
)
) 
) 	
)
 
) 

) 
) 
) 
) 	N ~     	 #*	   	
 
    	 )-# ~  
i	    	C I     	FJ %)?C	    	JN )-CG	tSy	    $(   	
 ! 
eD#I	   4 	 s  PT     	 S d d     	 c t t      r3   r`   c                   <   e Zd ZdZdedefdZeefdede	de	defdZ
	 	 ddee         dedee         fdZdedefdZ	 ddedededefdZdeeef         deeef         fdZ	 ddedededeeef         fdZddededefdZddededefdZddededefdZd	S )
GCPComputez'Abstraction around GCP compute resourcer   r9   c                 T    t          |j        t          j                              }|S r   r   r   credentialshttplib2Httprd   r   new_https      r+   r   z"GCPCompute.get_new_authorized_httpQ  "    !$"2IIIr3   r   r   r   c                    t                               d|d          d           t          |          D ]}| j                                                            | j        |d         | j                                      | 	                    | j        j
                            }d|v rt          |d                   |d         dk    r&t                               d	|d          d
            nt          j        |           |S )z/Poll for compute zone operation until finished.z7wait_for_compute_zone_operation: Waiting for operation rR    to finish...)projectr   zoner   errorr   DONEz+wait_for_compute_zone_operation: Operation 
 finished.)loggerinfor-   r_   zoneOperationsrg   r   r   executer   _httpr!   r.   r/   rd   r   r   r   r0   results         r+   r   zGCPCompute.wait_for_operationV  s4    	F%.v%6F F F	
 	
 	

 y!! 	& 	&A,,.. O'//   
 d::4=;NOOPP  &  w000h6))?!*6!2? ? ?   J}%%%%r3   NFr   rn   c                     |pi }|r8dd                     d |                                D                       z   dz   }nd}|rt          j        nt          j        }dd                     d |D                       z   dz   }d                    t           j        	          }d
                    d          }d ||||fD             }d                     |          }	 j        	                                
                     j         j        |	                                                    j        j                            }
|
                    dg           } fd|D             S )N(z AND c                 B    g | ]\  }}d                      ||          S )(labels.{key} = {value})keyr&   format)r   r   r&   s      r+   r   z-GCPCompute.list_instances.<locals>.<listcomp>  s?       &C 399c9OO  r3   ) z OR c                 :    g | ]}d                      |          S )z(status = {status}))r   r   )r   r   s     r+   r   z-GCPCompute.list_instances.<locals>.<listcomp>  s*    TTT&--V-<<TTTr3   r   r   z(NOT labels.{label}:*)	tpu_cores)labelc                     g | ]}||S r5   r5   )r   fs     r+   r   z-GCPCompute.list_instances.<locals>.<listcomp>  s0     	
 	
 	
 	
	
 	
 	
r3   )r   r   filterr   itemsc                 0    g | ]}t          |          S r5   )rM   r   r   rd   s     r+   r   z-GCPCompute.list_instances.<locals>.<listcomp>  s#    ;;;Aq$'';;;r3   )joinr  rM   r   rm   r   r   r   r_   	instanceslistr   r   r   r   r   rg   )rd   r   rn   label_filter_exprstatusesinstance_state_filter_exprcluster_name_filter_exprtpu_negation_filter_exprnot_empty_filtersfilter_exprresponser  s   `           r+   r   zGCPCompute.list_instancesz  s   
 &+ 	#,, *7*=*=*?*?      !# 8N..7 	 kkTT8TTT  	 	# $>#D#D$D,= $E $
 $
  $<#B#B#B#U#U 	
 	
 "*((		
 	
 	
 ll#455 M##%%T+"   
 W$66t}7JKKWLL 	 LL"--	;;;;;;;;r3   r   c                     | j                                                             | j        | j        |                                          }t          ||           S Nr   r   instance)r_   r  rg   r   r   r   rM   rd   r   r  s      r+   r   zGCPCompute.get_instance  sW    M##%%S+    
 WYY 	 h---r3   TrG   r7   r   c                 X   t          |d         fi ||d         d}|d         }| j                                                            | j        | j        ||                              |                     | j        j                            }|r| 	                    |          }n|}|S )Nr7   labelFingerprint)r7   r  rR   )r   r   r  bodyr   )
rz   r_   r  	setLabelsr   r   r   r   r   r   )rd   rG   r7   r   r  r   r   r   s           r+   r   zGCPCompute.set_labels  s     4>44V44 $%7 8
 
 v,M##%%Y+ 	    W$66t}7JKKWLL 	  	,,Y77FFFr3   configuration_dictc                 ^   t          |          }|d         }t          j        d|          s%d                    | j        |d                   |d<   |                    dg           D ]D}|d         }t          j        d|          s%d                    | j        | j        |	          |d<   E|S )
a  Ensures that resources are in their full URL form.

        GCP expects machineType and acceleratorType to be a full URL (e.g.
        `zones/us-west1/machineTypes/n1-standard-2`) instead of just the
        type (`n1-standard-2`)

        Args:
            configuration_dict: Dict of options that will be passed to GCP
        Returns:
            Input dictionary, but with possibly expanding `machineType` and
                `acceleratorType`.
        machineTypez.*/machineTypes/.*z(zones/{zone}/machineTypes/{machine_type})r   machine_typeguestAcceleratorsacceleratorTypez.*/acceleratorTypes/.*z>projects/{project}/zones/{zone}/acceleratorTypes/{accelerator})r   r   accelerator)r   r#   r$   r   r   rg   r   )rd   r  existing_machine_typer$  gpu_types        r+   _convert_resources_to_urlsz%GCPCompute._convert_resources_to_urls  s     &&899 2= Ay-/DEE 	 ;AA+/> B    .112ErJJ 		 		K"#45H95x@@  U[[ O/ ( \   % "!r3   r   c                 t   |                      |          }|                    dd            t          |t          j        j                  }t          |                    di           fi |}|                    t          |fi t          | j
        i|d           |                    dd           }| j                                                            | j        | j        ||                              |                     | j        j                            }|r|                     |          }n|}||fS )NnetworkConfigr7   )r7   rR   sourceInstanceTemplate)r   r   r*  r  r   )r'  poprB   rD   rN   r&   rz   rg   updater   r   r_   r  insertr   r   r   r   r   r   )	rd   r   r7   r   configrR   source_instance_templater   r   s	            r+   r   zGCPCompute.create_instance  sH    00==

?D)))"6;+>+DEEfjj2..99&99vSS*>@Q)RSS 	
 	
 	
* $*::.F#M#M  M##%%V+'?	    W$66t}7JKKWLL 	  	,,Y77FFFt|r3   c                     | j                                                             | j        | j        |                                          }|r|                     |          }n|}|S r  )r_   r  deleter   r   r   r   rd   r   r   r   r   s        r+   r   zGCPCompute.delete_instance@  sn    M##%%V+    
 WYY 	  	,,Y77FFFr3   c                     | j                                                             | j        | j        |                                          }|r|                     |          }n|}|S r  )r_   r  stopr   r   r   r   r2  s        r+   r   zGCPCompute.stop_instanceR  sn    M##%%T+    
 WYY 	  	,,Y77FFFr3   c                    | j                                                             | j        | j        |                              |                     | j         j                            }|r|                     |          }n|}|S )Nr  r   )	r_   r  startr   r   r   r   r   r   r2  s        r+   r   zGCPCompute.start_instancec  s     M##%%U+    
 W$66t}7JKKWLL 	  	,,Y77FFFr3   r   r   )rX   rY   rZ   r[   r   r   r   r   rz   r   r   r   r{   r
   rM   r   r%   r   r   r	   r   r'  r   r   r   r   r   r5   r3   r+   r   r   N  s?       11N ~     #*	" "" " 	"
 
" " " "L )-#C< C<~C< C< 
n		C< C< C< C<J.C .N . . . . NR ",0FJ	   4$""&sCx.$"	c3h$" $" $" $"N KO2 22)-2CG2	tSy	2 2 2 2h s  PT    $ S d d    " c t t      r3   r   c                   F   e Zd ZdZed             ZdedefdZee	fde
dedede
fd	Z	 	 ddee
         dedee         fdZdedefdZ eed          	 ddede
dede
fd            Z	 dde
de
dedee
ef         fdZddedede
fdZddedede
fdZddedede
fdZd
S )GCPTPUz#Abstraction around GCP TPU resourcec                 &    d| j          d| j         S )Nz	projects/z/locations/)r   r   rj   s    r+   pathzGCPTPU.path{  s    O4?OOt7MOOOr3   r   r9   c                 T    t          |j        t          j                              }|S r   r   r   s      r+   r   zGCPTPU.get_new_authorized_http  r   r3   r   r   r   c                 2   t                               d|d          d           t          |          D ]}| j                                                                                                                            |d                                        | 	                    | j        j
                            }d|v rt          |d                   d|v r&t                               d|d          d	            nt          j        |           |S )
z&Poll for TPU operation until finished.z.wait_for_tpu_operation: Waiting for operation rR   r   rV   r   r   r  z"wait_for_tpu_operation: Operation r   )r   r   r-   r_   projects	locationsr   rg   r   r   r   r!   r.   r/   r   s         r+   r   zGCPTPU.wait_for_operation  s/    	F%.v%6F F F	
 	
 	

 y!! 	& 	&A&&((Yv.011d::4=;NOOPP  &  w000V##?!*6!2? ? ?   J}%%%%r3   NFr   rn   c                      j                                                                                                                              j                                                            j         j                            }|	                    dg           } fd|D             }pi  j
        t          <   dt          dt          ffd}t	          t          ||                    }|S )N)parentr   nodesc                 0    g | ]}t          |          S r5   )rK   r	  s     r+   r   z)GCPTPU.list_instances.<locals>.<listcomp>  s#    <<<QZ4((<<<r3   r  r9   c                     |                                  rdS |                                 }r0                                D ]\  }}||vr dS |||         k    r dS dS )NFT)rn   rp   r  )r  r7   r   r&   r   s       r+   filter_instancez.GCPTPU.list_instances.<locals>.filter_instance  s    %%'' u((**F %"/"5"5"7"7 % %JC&(($uus++$uu , 4r3   )r_   r=  r>  rA  r  r:  r   r   r   rg   r   r   rK   r{   r  )rd   r   rn   r  r  rD  s   ``    r+   r   zGCPTPU.list_instances  s     M""$$Y[[UWWTT##W$66t}7JKKWLL 	 LL"--	<<<<)<<<	 &+.2.?*+	j 	T 	 	 	 	 	 	 ;;<<	r3   r   c                 &   | j                                                                                                                             |                              |                     | j         j                            }t          ||           S )NrV   r   )	r_   r=  r>  rA  rg   r   r   r   rK   r  s      r+   r   zGCPTPU.get_instance  sm    M""$$Y[[UWWSgSW$66t}7JKKWLL 	 (D)))r3   zunable to queue the operationTrG   r7   r   c                 |   dt          |d         fi |i}d}| j                                                                                                                            |d         ||                              |                     | j        j                            }|r| 	                    |          }n|}|S )Nr7   rR   )rR   
updateMaskr  r   )
rz   r_   r=  r>  rA  patchr   r   r   r   )rd   rG   r7   r   r  update_maskr   r   s           r+   r   zGCPTPU.set_labels  s    
 d4>44V44
  M""$$Y[[UWWU&\&   
 W$66t}7JKKWLL 	  	,,Y77FFFr3   r   c           	      H   |                                 }|                    dd            t          |t          j        j                  }t          |                    di           fi |}|                    dt          |fi t          | j
        ii           d|vri |d<   d|d         vrd|d         d<   d|v rB|                    d          d         |d<   |d                             d	          |d         d
<   | j                                                                                                                            | j        ||                              |                     | j        j                            }|r|                     |          }n|}||fS )Nr   r7   r)  enableExternalIpsTserviceAccountsr   serviceAccountscopesscope)r@  r  nodeIdr   )copyr+  rB   rD   rL   r&   rz   rg   r,  r   r   r_   r=  r>  rA  creater:  r   r   r   r   )rd   r   r7   r   r.  rR   r   r   s           r+   r   zGCPTPU.create_instance  s    !!##

&---"6;?+@AAfjj2..99&99$vSS*>@Q)RSS	
 	
 	
 &((&(F?#f_&=== <@F?#$78
 &&'-zz2C'D'DQ'GF#$067G0H0L0LX0V0VF#$W- M""$$Y[[UWWVy   
 W$66t}7JKKWLL 	  	,,Y77FFFt|r3   c                 L   | j                                                                                                                             |                              |                     | j         j                            }|r|                     |t                    }n|}|S NrV   r   )r   )
r_   r=  r>  rA  r1  r   r   r   r   r   r2  s        r+   r   zGCPTPU.delete_instance)  s    M""$$Y[[UWWVV!!W$66t}7JKKWLL 	  	,,Y),LLFFFr3   c                 L   | j                                                                                                                             |                              |                     | j         j                            }|r|                     |t                    }n|}|S rT  )
r_   r=  r>  rA  r4  r   r   r   r   r   r2  s        r+   r   zGCPTPU.stop_instance:  s    M""$$Y[[UWWTwTW$66t}7JKKWLL 	  	,,Y),LLFFFr3   c                 L   | j                                                                                                                             |                              |                     | j         j                            }|r|                     |t                    }n|}|S rT  )
r_   r=  r>  rA  r6  r   r   r   r   r   r2  s        r+   r   zGCPTPU.start_instanceJ  s    M""$$Y[[UWWUU  W$66t}7JKKWLL 	  	,,Y),LLFFFr3   r   r   )rX   rY   rZ   r[   r   r:  r   r   MAX_POLLS_TPUr   rz   r   r   r   r{   r
   rK   r   r%   r   r6   r   r   r   r   r   r   r   r5   r3   r+   r8  r8  v  s8       -- P P XPN ~     '*	       	 
 
       H )-#' '~' ' 
j		' ' ' 'R	*C 	*J 	* 	* 	* 	* $CDDIM (,BF	   ED8 KO/ //)-/CG/	tSy	/ / / /b s  PT    " S d d      c t t      r3   r8  )6r[   r|   loggingr#   r.   collectionsr   rQ  r   enumr   	functoolsr   typingr   r	   r
   r   r   r   uuidr   r   google_auth_httplib2r   googleapiclient.discoveryr   googleapiclient.errorsr   ray.autoscaler.tagsr   r   	getLoggerrX   r   r>   r?   r   rW  r   r!   r%   r   r6   rz   rB   rD   ABCMetarH   rM   rK   r`   r   r8  r5   r3   r+   <module>rd     sM   2 


  				                                : : : : : : : : : : : : : : : :        / / / / / / . . . . . . , , , , , , G G G G G G G G		8	$	$  	 A
   )	! !Yi 001!C=! ! 	! ! ! !HP P3 P3 P P P P"0 0 0 0 0$ 0 0 02B B B B Bh#+ B B B BDG G G G GW G G G.L L L L L L L LBk k k k kCK k k k k\e e e e e e e eP	b b b b b[ b b b b br3   