
    &`i                     v   U d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZ ddlmZmZmZmZmZmZmZmZ ddlZddlmZ ddlmc mc mZ ddlmc mc mZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' dd	l(m)Z) dd
l*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZA erddlBmCZC ddlDmEZE  ejF        eG          ZH e#dd          ZI e#dd          ZJ e"dd          ZKeLeMd<   dZNdZOdZPdeQdeQfdZR G d  d!          ZSdeTfd"ZUd#ejV        deWfd$ZXdS )%zThis file includes the Worker class which sits on the client side.
It implements the Ray API functions that are forwarded through grpc calls
to the server.
    N)defaultdictFuture)TYPE_CHECKINGAnyCallableDictListOptionalTupleUnion)%DEFAULT_CLIENT_RECONNECT_GRACE_PERIOD	env_floatenv_integer)upload_py_modules_if_needed)upload_working_dir_if_needed)pickle)GetTimeoutError)	JobConfig)dumps_from_clientloads_from_server)
GRPC_OPTIONSGRPC_UNRECOVERABLE_ERRORS	INT32_MAXOBJECT_TRANSFER_WARNING_SIZEClientActorClassClientActorHandleClientActorRefClientObjectRefClientRemoteFunc
ClientStub)
DataClient)LogstreamClient)log_once)
ActorClass)RemoteFunction'RAY_CLIENT_INITIAL_CONNECTION_TIMEOUT_S   #RAY_CLIENT_MAX_CONNECTION_TIMEOUT_S   (RAY_CLIENT_MAX_BLOCKING_OPERATION_TIME_Sg       @MAX_BLOCKING_OPERATION_TIME_Si   zKhttps://docs.ray.io/en/latest/ray-core/patterns/too-fine-grained-tasks.htmlzRhttps://docs.ray.io/en/latest/ray-core/patterns/closure-capture-large-objects.htmltimeoutreturnc                 4    | dz   } | t           k    rt           } | S )Nr(   )MAX_TIMEOUT_SECr-   s    j/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/util/client/worker.pybackoffr3   O   s     kG  !N    c                      e Zd Z	 	 	 	 	 d]dededeeeef                  ded	ee	j
                 f
d
Zd^d_dZde	j        defdZdedefdZdej        defdZdefdZde	j        fdZd Zdedeej        gdf         ddfdZdddee         defdZdee         defdZdddded ee          fd!Z!	 d`ded"ee          fd#Z"d$dd%d&d'ee         d(eded)edeee         ee         f         f
d*Z#dee$         fd+Z%d,ej&        d(ee         dee$         fd-Z'd.eddfd/Z(d.eddfd0Z)d.eddfd1Z*d2 Z+	 d`d3ed4ee         de fd5Z,d6e d7eddfd8Z-d9ed:ed;eddfd<Z.	 d`d=ej/        j0        dee         fd>Z1d?ed4ee         defd@Z2d?ed4ee         defdAZ3d?edBedCed4ee         def
dDZ4d?edEed4ee         defdFZ5dGed4ee         dee         fdHZ6dIedJeddfdKZ7dLedee8eef                  fdMZ9defdNZ:d`defdOZ;defdPZ<	 d`dQe=dRee8eef                  fdSZ>d6dTdefdUZ?dVdWdefdXZ@d?eddYfdZZAd?edefd[ZBdefd\ZCdS )aWorker FN   conn_strsecuremetadataconnection_retries_credentialsc                    t                      | _        d| j        fg|r|ng z   | _        d| _        d| _        t
          j        j        | _        i | _	        |p3t          j                            dd                                          dv | _        || _        || _        ||| _        d| _        nd| _        t$          | _        dt          j        v r$t)          t          j        d                   | _        | j        dk    | _        d	| _        d	| _        |                                  d| _        d	| _        t5          | | j        | j                  | _        t9          t(                    | _        t=          | | j                  | _        | j                             tB          j"                   d	| _#        d| _$        tK          j&                    | _'        d| _(        dS )
a  Initializes the worker side grpc client.

        Args:
            conn_str: The host:port connection string for the ray server.
            secure: whether to use SSL secure channel or not.
            metadata: additional metadata passed in the grpc request headers.
            connection_retries: Number of times to attempt to reconnect to the
              ray server if it doesn't respond immediately. Setting to 0 tries
              at least once.  For infinite retries, catch the ConnectionError
              exception.
            _credentials: gprc channel credentials. Default ones will be used
              if None.
        	client_idNRAY_USE_TLS01trueT!RAY_CLIENT_RECONNECT_GRACE_PERIODr   F))make_client_id
_client_idr;   channelservergrpcChannelConnectivityIDLE_conn_state
_convertedosenvirongetlower_secure	_conn_str_connection_retriesr=   r   _reconnect_grace_periodint_reconnect_enabled_in_shutdown_has_connected_connect_channel_serverside_ray_initializedr"   data_clientr   reference_countr#   
log_clientset_logstream_levelloggingINFOclosed!total_outbound_message_size_bytes	threadingLock_req_id_lock_req_id)selfr9   r:   r;   r<   r=   s         r2   __init__zWorker.__init__W   s   * )**%t78 (HHb
 3813 
s!C!C!I!I!K!K P
 "
 "#5 # ,DDLL $D'L$."*<<+.
>?, ,D( #'">!"C "#" ,1( &dDOT]KK1<S1A1A)$>>++GL999 23. &N,,r4   r.   c                 R   | j         8| j                             | j                   | j                                          ddlm} d}| j        rY| j        | j        }nJt          j	        
                    dd                                          dv rd}nt          j                    } || j        t          d|          | _         | j                             | j                   t#          j                    }d}t$          }d}|t'          | j        d	          k     s|r|d	z  }| j        rnt#          j                    |z
  }|r*|| j        k    rd
| _        t/          d| j         d          	 t          j        | j                                       |           t5          j        | j                   | _        t;          |                                           }|rnt#          j        |           n# t          j         $ r! tB          "                    d| d           Y n|t          j#        $ rk}	tB          "                    d| d           tB          "                    d|	$                                            t#          j        |           Y d}	~	nd}	~	ww xY wtB          "                    d| d           |stK          |          }|t'          | j        d	          k     ||s9d
| _        tM          d          rtO          j(        d           t/          d          dS )z
        Attempts to connect to the server specified by conn_str. If
        reconnecting after an RPC error, cleans up the old channel and
        continues to attempt to connect until the grace period is over.
        Nr   )init_grpc_channelr@   rA   rB   F)optionsasynchronouscredentials   Tz:Failed to reconnect within the reconnection grace period (zs)r1   zCouldn't connect channel in z seconds, retryingz+Ray client server unavailable, retrying in zs...zReceived when checking init: z8Waiting for Ray to become ready on the server, retry in ray_client_security_groupszRay Client connection timed out. Ensure that the Ray Client port on the head node is reachable from your local machine. See https://docs.ray.io/en/latest/cluster/ray-client.html#step-2-check-ports for more information.zray client connection timeout))rH   unsubscribe_on_channel_state_changecloseray._private.grpc_utilsrl   rS   r=   rO   rP   rQ   rR   rJ   ssl_channel_credentialsrT   r   	subscribetimeINITIAL_TIMEOUT_SECmaxrU   rY   rV   ConnectionErrorchannel_ready_futureresultray_client_pb2_grpcRayletDriverStubrI   boolping_serversleepFutureTimeoutErrorloggerdebugRpcErrordetailsr3   r$   warningswarn)
ri   reconnectingrl   ro   
start_timeconn_attemptsr-   service_readyelapsed_timees
             r2   r[   zWorker._connect_channel   s    <#L$$T%BCCCL   ====== < 	= ,"/s3399;;}LL" #:<< )(N #	
 
 
 	t<=== Y[[
%c$":A>>>>,>QM  9;;3L t/K K K$(!%949 9 9  $ )$,77>>w>OOO 2B4<PP $T%5%5%7%7 8 8  
7####* Y Y YWGWWWXXXXX = $ $ $O'OOO   JQYY[[JJKKK
7########$ LLX7XXX     + "'**W c$":A>>>>,>`  
	C $D455 (   ""ABBB
	C 
	Cs&   0A.G5  G5 5-J$J2A!JJr   c                     | j         sdS | j        rdS |                                t          v rdS |                                t          j        j        k    r|                                }|dk    rdS dS )zz
        Returns True if the RPC error can be recovered from and a retry is
        appropriate, false otherwise.
        FzException serializing request!T)rX   rY   coder   rJ   
StatusCodeINTERNALr   )ri   r   r   s      r2   _can_reconnectzWorker._can_reconnect  s|    
 & 	5 	56688000 56688t///iikkG::: utr4   	stub_namec                 0   | j         s	  t          | j        |          |i |S # t          j        $ r4}|                     |          rt          j        d           Y d}~` d}~wt          $ r t          j        d           Y w xY wt          d          )z
        Calls the stub specified by stub_name (Schedule, WaitObject, etc...).
        If a recoverable error occurrs while calling the stub, attempts to
        retry the RPC.
              ?NClient is shutting down.)
rY   getattrrI   rJ   r   r   rx   r   
ValueErrorr{   )ri   r   argskwargsr   s        r2   
_call_stubzWorker._call_stub#  s     # 	6wt{I66GGGG=   &&q)) JsOOOHHHH   
 
3 8999s    $ B)A"!A""!BBreqc              /   V  K   d}| j         s|dz   |_        	  | j        j        |g|R i |D ]}|j        |k    r,t
                              d|j         d|j         d           9|dz   |j        k    r"t          d|j         d| j	        dz              |j        }|V  ||j
        dz
  k    r dS dS # t          j        $ r4}|                     |          rt          j        d	           Y d}~ d}~wt           $ r t          j        d	           Y w xY wt#          d
          )z
        Calls the stub for GetObject on the underlying server stub. If a
        recoverable error occurs while streaming the response, attempts
        to retry the get starting from the first chunk that hasn't been
        received.
        rp   zReceived a repeated chunk z from request .zReceived chunk z when we expected Nr   r   )rY   start_chunk_idrI   	GetObjectchunk_idr   r   req_idRuntimeErrorlast_seen_chunktotal_chunksrJ   r   r   rx   r   r   r{   )ri   r   r   r   r   chunkr   s          r2   _get_object_iteratorzWorker._get_object_iterator:  s      # #	 "11!4C2T[23HHHHHH  E~88: : :,/J: : :   !&*en<<*:en : :#3a7: :   ',nOKKK&%*<q*@@@ A =   &&q)) JsOOOHHHH   
 
3 8999s*   BB6 3B6 6D)C43C44!DDc                    | j         s|S t          t          j                              }| j        5  | xj        dz  c_        | j        t          k    rd| _        t          | j                  }ddd           n# 1 swxY w Y   |d|fd|fgz   S )aT  
        Adds a unique req_id and the current thread's identifier to the
        metadata. These values are useful for preventing mutating operations
        from being replayed on the server side in the event that the client
        must retry a requsest.
        Args:
            metadata: the gRPC metadata to append the IDs to
        rp   N	thread_idr   )rX   strre   	get_identrg   rh   r   )ri   r;   r   r   s       r2   _add_ids_to_metadatazWorker._add_ids_to_metadataj  s     & 	O	+--..	 	' 	'LLALL|i'' &&F		' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	'
 K3h5GHHHs   <A99A= A=
conn_statec                 N    t                               d|            || _        d S )Nz"client gRPC channel state change: )r   r   rM   )ri   r   s     r2   rs   zWorker._on_channel_state_change~  s+    F*FFGGG%r4   c                     	 | j                                         }n&# t          j        $ r}t	          |          d }~ww xY w|j        |j        |j        |j        dS )N)num_clientspython_versionray_version
ray_commit)	r]   ConnectionInforJ   r   decode_exceptionr   r   r   r   )ri   datar   s      r2   connection_infozWorker.connection_info  st    	&#2244DD} 	& 	& 	&"1%%%	&  +"1+/	
 
 	
s    ?:?refcallbackc                 t    t          j        |j        gd          }| j                            ||           d S )NT)idsrn   )ray_client_pb2
GetRequestidr]   RegisterGetCallback)ri   r   r   r   s       r2   register_callbackzWorker.register_callback  s<    
 'SVH4HHH,,S(;;;;;r4   r1   r-   c                <   t          |t                    r|sg S |}n8t          |t                    r|g}nt          dt	          |          z            |d }nt          j                    |z   }	 |r8t          t          t          |t          j                    z
  d                    }nt          }	 | 
                    ||          }nX# t          $ rJ |rt          j                    |k    r t                              d                    |                     Y nw xY wt          |          t          |          k    r=t          d                    t          |          t          |                              t          |t                    r|d         }|S )Nz>Can't get something that's not a list of IDs or just an ID: %sTgMbP?zInternal retry for get {}z<Mismatched number of items in request ({}) and response ({})r   )
isinstancelistr   	Exceptiontyperx   	monotonicminr,   rz   _getr   r   r   formatlen)ri   valsr-   to_getdeadline
op_timeoutress          r2   rQ   z
Worker.get  s   dD!! 
	 	FFo.. 	VFF026t**=  
 ?HH~'''1H	I ; 14>#3#33U;; 


 ;
Iii
33" I I I  0 08 ; ;8??GGHHHHHI	I v;;#c((""NUUKKS   
 dO,, 	a&C
s   3C
 
ADDc                    t          j        d |D             |          }t                      }	 |                     || j                  }|D ]}|j        sj	 t          j        |j                  }nM# t          j
        t          f$ r4 t                              d                    |j                              w xY w||j        t           k    r:t#          d          r+|j        dz  }t%          j        d|dd	t(          d
           |                    |j                   n&# t.          j        $ r}	t3          |	          d }	~	ww xY wt5          |          S )Nc                     g | ]	}|j         
S  r   ).0rs     r2   
<listcomp>zWorker._get.<locals>.<listcomp>  s    ,?,?,?aQT,?,?,?r4   )r   r-   r;   Failed to deserialize {}#client_object_transfer_size_warningi   @z'Ray Client is attempting to retrieve a z.2fzy GiB object over the network, which may be slow. Consider serializing the object to a file and using S3 or rsync instead.r(   )
stacklevel)r   r   	bytearrayr   r;   validcloudpickleloadserrorr   UnpicklingError	TypeErrorr   	exceptionr   
total_sizer   r$   r   r   UserWarningextendr   rJ   r   r   r   )
ri   r   r-   r   r   respr   errsize_gbr   s
             r2   r   zWorker._get  s   ',?,?3,?,?,?QQQ{{	&,,S4=,IID ( ({ )/<<"2I>   (()C)J)J5;)W)WXXX I#&BBBx9H HB $.6GM9"P9 9 9 $#$    EJ'''')(* } 	& 	& 	&"1%%%	& &&&s1   'D' A21D' 2A
B<<A*D' 'E
6EE
)client_ref_id_ownerr   r   c                    t          |t                    rt          d          t          || j                  }|                     |||          S )NzCalling 'put' on an ObjectRef is not allowed (similarly, returning an ObjectRef from a remote function is not allowed). If you really want to do this, you can wrap the ObjectRef in a list and call 'put' on it (or return it).)r   r   r   r   rG   _put_pickled)ri   valr   r   r   s        r2   putz
Worker.put  sX     c?++ 	3   !do66  }f===r4   ownerc                    t          j        |          }|||_        ||j        j        |_        | j                            |          }|j        sg	 t          j
        |j                  # t          j        t          f$ r4 t                              d                    |j                              w xY wt%          |j                  S )N)r   r   )r   
PutRequestr   	actor_refr   owner_idr]   	PutObjectr   r   r   r   r   r   r   r   r   r   r   )ri   r   r   r   r   r   s         r2   r   zWorker._put_pickled  s     'T222$ -C ?-CL))#..z 	!'
333*I6     !;!B!B4:!N!NOOO tw'''s   A- -A
B7rp   T)num_returnsr-   fetch_localobject_refsr   r   c                   t          |t                    st          dt          |                     |D ]6}t          |t                    st          dt          |                     7d |D             |||nd| j        d}t          j        di |}|                     d|| j	                  }|j
        st          d          d	 |j        D             }	d
 |j        D             }
|	|
fS )Nz/wait() expected a list of ClientObjectRef, got z?wait() expected a list of ClientObjectRef, got list containing c                     g | ]	}|j         
S r   r   )r   
object_refs     r2   r   zWorker.wait.<locals>.<listcomp>  s    GGGZ:=GGGr4   r   )
object_idsr   r-   r?   
WaitObjectr   z.Client Wait request failed. Reference invalid?c                 ,    g | ]}t          |          S r   r   r   r   s     r2   r   zWorker.wait.<locals>.<listcomp>!  s-     #
 #
 #
%(OC  #
 #
 #
r4   c                 ,    g | ]}t          |          S r   r  r  s     r2   r   zWorker.wait.<locals>.<listcomp>$  s-     '
 '
 '
%(OC  '
 '
 '
r4   r   )r   r   r   r   r   rG   r   WaitRequestr   r;   r   r   ready_object_idsremaining_object_ids)ri   r   r   r-   r   r   r   r   r   client_ready_object_idsclient_remaining_object_idss              r2   waitzWorker.wait  s^    +t,, 	U${BSBSUU    	 	Cc?33 7+/997 7   HG;GGG&#*#6wwR	
 
 (00400|S4=IIz 	NLMMM#
 #
,0,A#
 #
 #
'
 '
,0,E'
 '
 '
# ()DEEr4   c                     |                                 }t          ||f| j                  |_        |                                }|dk    rd}|dk    rt          d          |                     ||          S )Ndynamicr   	streamingzdStreaming actor methods (num_returns="streaming") are not currently supported when using Ray Client.)_prepare_client_taskr   rG   r   _num_returnsr   _call_schedule_for_task)ri   instancer   r   taskr   s         r2   call_remotezWorker.call_remote*  s    ,,..%tVndoFF	++--)##K+%%E  
 ++D+>>>r4   r  c                 .   t                               d|j         d|j         d|j                    | j        |_        |d}|dk    rdd t                    D             dt          t          j
        t          f         dd ffd}| j                            ||           | xj        |                                z  c_        | j        t           k    r9t#          d	          r*t%          j        d
t(           dt*           t,                     S )NzScheduling task  rp   r   c                 *    g | ]}t                      S r   r   )r   _s     r2   r   z2Worker._call_schedule_for_task.<locals>.<listcomp>D  s    ???1fhh???r4   r   r.   c                    t          | t                    rEt          | t          j                  rt	          |           } D ]}|                    |            d S | j        }|j        s\	 t          j	        |j
                  }n%# t          j        t          f$ r}|}Y d }~nd }~ww xY wD ]}|                    |           d S t          |j                  k    rYt!          d dt          |j                             }t#          |j                  D ]\  }}|                    |           d S t#          |j                  D ]\  }}|                    |           d S )Nz	Expected z returns but received )r   r   rJ   r   r   set_exceptiontask_ticketr   r   r   r   r   r   r   r   
return_idsr   zip
set_result)	r   futureticketexe_newexcraw_id
id_futuresnum_return_refss	          r2   populate_idsz4Worker._call_schedule_for_task.<locals>.populate_idsF  s   $	** dDM22 2+D11D( / /F((....%F< $*6<88BB.	:   BBBBBB( - -F((,,,,6$%%88 0 0 06,--0 0  '**f6G&H&H . .NFF((----"%j&2C"D"D * *!!&))))* *s   +B B'B""B'%client_communication_overhead_warningaa  More than 10MB of messages have been created to schedule tasks on the server. This can be slow on Ray Client due to communication overhead over the network. If you're running many fine-grained tasks, consider running them inside a single remote function. See the section on "Too fine-grained tasks" in the Ray Design Patterns document for more details: z. If your functions frequently use large objects, consider storing the objects remotely with ray.put. An example of this is shown in the "Closure capture of large / unserializable object" section of the Ray Design Patterns document, available here: )r   r   namer   
payload_idrG   r?   ranger   r   DataResponser   r]   Schedulerd   ByteSizeMESSAGE_SIZE_THRESHOLDr$   r   r   $DESIGN_PATTERN_FINE_GRAIN_TASKS_LINK!DESIGN_PATTERN_LARGE_OBJECTS_LINKr   )ri   r  r   r%  r#  r$  s       @@r2   r  zWorker._call_schedule_for_task9  sY    	Q	QQDIQQQQRRRK%b  O??o(>(>???
	*u^%@)%KL 	*QU 	* 	* 	* 	* 	* 	* 	*< 	!!$555..$--//A..25KKK@AA L M7 "F7 7 57 7     r4   r   c                     | j         rd S | j        |xx         dz  cc<   | j        |         dk    r|                     |           | j        |= d S d S )Nrp   r   )rc   r^   _release_serverri   r   s     r2   call_releasezWorker.call_release}  sp    ; 	FR   A%   #q((  $$$$R((( )(r4   c                     | j         _t                              d|                                            | j                             t          j        |g                     d S d S )Nz
Releasing )r   )r]   r   r   hexReleaseObjectr   ReleaseRequestr2  s     r2   r1  zWorker._release_server  sc    'LL0bffhh00111**>+HbT+R+R+RSSSSS ('r4   c                     t                               d|                                            | j        |xx         dz  cc<   d S )Nz
Retaining rp   )r   r   r5  r^   r2  s     r2   call_retainzWorker.call_retain  sJ    ,"&&((,,---R   A%     r4   c                     d| _         d| _        | j                                         | j                                         d | _        | j        r"| j                                         d | _        d S d S )NT)rY   rc   r]   rt   r_   rI   rH   ri   s    r2   rt   zWorker.close  st        < 	 L   DLLL	  	 r4   r'  	namespacec                    t          j                    }t           j        j        |_        ||_        |pd|_        t          g i f| j                  |_        | 	                    |d          }t          |          dk    sJ t          t          |d         d                    }|j                                        rt          d| d          |S )Nr7   rp   r   T)weak_refzActorID for z	 is empty)r   
ClientTaskNAMED_ACTORr   r'  r<  r   rG   r   r  r   r   r   r   is_nilr   )ri   r'  r<  r  futureshandles         r2   	get_actorzWorker.get_actor  s     (**"-9		"b%r2h@@	..tQ777||q    ">'!*t#L#L#LMM ""$$ 	=;D;;;<<<r4   actor
no_restartc                    t          |t                    s/t          d                    t	          |                              t
          j                                        }|j        j	        |_	        ||_
        t          j        |          }| j        |_        	 | j                            |           d S # t          j        $ r}t#          |          d }~ww xY w)Nz.ray.kill() only supported for actors. Got: {}.)rE  )r   r   r   r   r   r   TerminateRequestActorTerminater   r   rF  rG   r?   r]   	TerminaterJ   r   r   )ri   rE  rF  
term_actortermr   s         r2   terminate_actorzWorker.terminate_actor  s    %!233 	@GGUTT   $4CCEE
*
 *
.Z@@@	&&&t,,,,,} 	& 	& 	&"1%%%	&s   B9 9CCCobjforce	recursivec                    t          |t                    s t          dt          |           d          t          j                                        }|j        |_        ||_        ||_	        t	          j        |          }| j
        |_        	 | j                            |           d S # t          j        $ r}t!          |          d }~ww xY w)Nz<ray.cancel() only supported for non-actor object refs. Got: r   )task_object)r   r   r   r   r   rH  TaskObjectTerminater   rO  rP  rG   r?   r]   rJ  rJ   r   r   )ri   rN  rO  rP  term_objectrL  r   s          r2   terminate_taskzWorker.terminate_task  s     #// 	%S		% % %   %5IIKK! ).;GGG	&&&t,,,,,} 	& 	& 	&"1%%%	&s   B, ,C;C

Creq_typec                 T   t          j                    }||_        | j                            ||| j                  }|                    d          dk    rt          |j        j	                  }|S |                    d          dk    r|j
        S t          j        |j                  S )N)r-   r;   response_typeresource_tableruntime_context)r   ClusterInfoRequestr   rI   ClusterInfor;   
WhichOneofdictrY  tablerZ  jsonr   )ri   rV  r-   r   r   output_dicts         r2   get_cluster_infozWorker.get_cluster_info  s    
 /11{&&sGdm&TT???++/???t2899K___--1BBB''z$)$$$r4   keyc                     t          j        ||          }	 |                     d|| j                  }n&# t          j        $ r}t          |          d }~ww xY w|                    d          r|j        S d S )Nrc  r<  KVGetr   value)	r   KVGetRequestr   r;   rJ   r   r   HasFieldrg  ri   rc  r<  r   r   r   s         r2   internal_kv_getzWorker.internal_kv_get  s    )cYGGG	&??7C$-?HHDD} 	& 	& 	&"1%%%	&==!! 	:t   6 AAAc                     t          j        ||          }	 |                     d|| j                  }n&# t          j        $ r}t          |          d }~ww xY w|j        S )Nre  KVExistsr   )r   KVExistsRequestr   r;   rJ   r   r   existsrj  s         r2   internal_kv_existszWorker.internal_kv_exists  sn    ,	JJJ	&??:sT]?KKDD} 	& 	& 	&"1%%%	&{rl  rg  	overwritec                     t          j        ||||          }|                     | j                  }	 |                     d||          }n&# t
          j        $ r}t          |          d }~ww xY w|j        S )N)rc  rg  rr  r<  KVPutr   )	r   KVPutRequestr   r;   r   rJ   r   r   already_exists)	ri   rc  rg  rr  r<  r   r;   r   r   s	            r2   internal_kv_putzWorker.internal_kv_put  s     )5I
 
 
 ,,T];;	&??7C(?CCDD} 	& 	& 	&"1%%%	&""s   A A0A++A0del_by_prefixc                     t          j        |||          }|                     | j                  }	 |                     d||          }n&# t
          j        $ r}t          |          d }~ww xY w|j        S )N)rc  rx  r<  KVDelr   )	r   KVDelRequestr   r;   r   rJ   r   r   deleted_num)ri   rc  rx  r<  r   r;   r   r   s           r2   internal_kv_delzWorker.internal_kv_del  s     )=I
 
 
 ,,T];;	&??7C(?CCDD} 	& 	& 	&"1%%%	&s   A A/A**A/prefixc                     	 t          j        ||          }|                     d|| j                  j        S # t
          j        $ r}t          |          d }~ww xY w)N)r~  r<  KVListr   )r   KVListRequestr   r;   keysrJ   r   r   )ri   r~  r<  r   r   s        r2   internal_kv_listzWorker.internal_kv_list
  sh    	& .f	RRRC??8S4=?IINN} 	& 	& 	&"1%%%	&s   7: A	AAuriexpiration_sc                 l    t          j        ||          }|                     d|| j                   d S )N)r  r  PinRuntimeEnvURIr   )r   ClientPinRuntimeEnvURIRequestr   r;   )ri   r  r  r   s       r2   pin_runtime_env_urizWorker.pin_runtime_env_uri  sA    :,
 
 
 	*C$-HHHHHr4   all_namespacesc                     t          j        |          }t          j        | j                            |          j                  S )N)r  )r   ClientListNamedActorsRequestr`  r   r]   ListNamedActorsactors_json)ri   r  r   s      r2   list_named_actorszWorker.list_named_actors  s9    9XXXz$*::3??KLLLr4   c                     |                                  r| j        dS | j        s)|                     t          j        j                  | _        | j        S )NF)is_connectedrI   r\   rb  r   ClusterInfoTypeIS_INITIALIZEDr;  s    r2   is_initializedzWorker.is_initialized  s[      "" 	dk&95/ 	
 04/D/D.=0 0D, //r4   c                     | j         Dt                              d           |                     t          j        j        |          }|duS dS )zSimple health check.

        Piggybacks the IS_INITIALIZED call to check if the server provides
        an actual response.
        NzPinging server.r1   F)rI   r   r   rb  r   r  PING)ri   r-   r}   s      r2   r   zWorker.ping_server+  sV     ;"LL*+++**.3W +  F %%ur4   c                      | j          o| j        S N)rY   rZ   r;  s    r2   r  zWorker.is_connected9  s    $$<)<<r4   
job_configray_init_kwargsc                    |i }	 |d}nt          j                    5 }ddlm} |j        pi }t
          j                            |d          dk    }t          |||t                    }t          |||t                    }|                    dd           |                    |d	           ddd           n# 1 swxY w Y   t          j        |          }| j                            t#          j        |t'          j        |          | j        
                    }|j        st-          d|j                   dS # t0          j        $ r}	t5          |	          d}	~	ww xY w)zInitialize the serverNr   ) RAY_RUNTIME_ENV_IGNORE_GITIGNORErA   rC   )scratch_dirinclude_gitignorer   excludesT)validate)r  r  reconnect_grace_periodz$Initialization failure from server:
)tempfileTemporaryDirectoryray._private.ray_constantsr  runtime_envrO   rP   rQ   r   r   r   popset_runtime_envr   dumpsr]   Initr   InitRequestr`  rV   okConnectionAbortedErrormsgrJ   r   r   )
ri   r  r  serialized_job_configtmp_dirr  r  r  responser   s
             r2   _server_initzWorker._server_init<  s    " O.	&!(,%%022 Kg      #-"8">BK 
'GMMQTT & #>#$+*;%	# # #K #?#$+*;%	# # #K  OOJ555..{T.JJJ3K K K K K K K K K K K K K K K6 )/Z(@(@%',,*4$(J$?$?+/+G   H ; ,JHLJJ   
 } 	& 	& 	&"1%%%	&s<   E  BB;/E  ;B??E  B?A;E   E#EE#r%   c                     t          j                    j        }|j        j        }t          ||j                  | j        |<   |S )z@Register a ClientActorClass for the ActorClass and return a UUIDrm   )uuiduuid4r5  __ray_metadata__modified_classr   _default_optionsrN   )ri   rE  rc  clss       r2   _convert_actorzWorker._convert_actorr  s<    jll$3/U=STTT
r4   funcr&   c                 |    t          j                    j        }t          |j        |j                  | j        |<   |S )z@Register a ClientRemoteFunc for the ActorClass and return a UUIDr  )r  r  r5  r    	_functionr  rN   )ri   r  rc  s      r2   _convert_functionzWorker._convert_functiony  s<    jll/ND$9 
  
  
 
r4   r!   c                     | j         |         S )z)Given a UUID, return the converted objectrN   ri   rc  s     r2   _get_convertedzWorker._get_converted  s    s##r4   c                     || j         v S )zACheck if a key UUID is present in the store of converted objects.r  r  s     r2   _converted_key_existszWorker._converted_key_exists  s    do%%r4   c                 ,    t          || j                  S r  )r   rG   )ri   r   s     r2   _dumps_from_clientzWorker._dumps_from_client  s     do666r4   )r7   FNr8   N)F)r.   Nr  )D__name__
__module____qualname__r   r   r
   r   rW   r   rJ   ChannelCredentialsrj   r[   r   r   r   r   r   r   r   r   rK   rs   r   r   r   r*  r   floatrQ   r   bytesr   r   r   r	  r   r  r?  r  r3  r1  r9  rt   rD  rM  rU  r  TypeEnumrb  rk  rq  rw  r}  r  r  r	   r  r  r   r  r   r  r  r  r  r  r  r   r4   r2   r6   r6   V   sM        *."#:>M MM M uS#X'	M
  M t67M M M M^cC cC cC cC cCJ $    0:C :S : : : :..:!,.:	.: .: .: .:`IS I I I I(&43K & & & &

 

 

<< N78$>?< 
	< < < < 7; ) ) )HUO )s ) ) ) )V'_- ' ' ' ' 'D  $.2> > > 	>
 *+> > > >& PT( (#((19:K1L( ( ( (.  $F $F $F/*$F 	$F
 $F $F 
tO$d?&;;	<$F $F $F $FL?V ? ? ? ?B"-B<DSMB	fB B B BH)u ) ) ) ) )T% TD T T T T
&e & & & & &      59 $,SM	   &&%6 &D &T & & & &&"&+/&<@&	& & & &, $(% % 09% %% % % % 	5 	Xe_ 	 	 	 	 	e  D    ##!&#37#DLUO#	# # # #  )- :B5/ 	       &&(0&	e& & & &Is I# I$ I I I IM Md38n9M M M M M0 0 0 0 0 4    =d = = = = RV4& 4&#4&6>tCH~6N4& 4& 4& 4&lL S    &6 3    $# $, $ $ $ $& & & & & &7 7 7 7 7 7 7r4   r6   c                  6    t          j                    } | j        S r  )r  r  r5  r   s    r2   rF   rF     s    	B6Mr4   r   c                     |                                  t          j        j        k    rt	          d|            S t          j        |                                           }t          |          S )NzGRPC connection failed: )	r   rJ   r   ABORTEDr{   base64standard_b64decoder   r   )r   r   s     r2   r   r     sY    vvxx4?*** =!==>>>$QYY[[11DT"""r4   )Y__doc__r  r`  ra   rO   r  re   rx   r  r   collectionsr   concurrent.futuresr   typingr   r   r   r	   r
   r   r   r   rJ   ray.cloudpickler   !ray.core.generated.ray_client_pb2core	generatedr   &ray.core.generated.ray_client_pb2_grpcr~   r  r   r   r   #ray._private.runtime_env.py_modulesr   $ray._private.runtime_env.working_dirr   ray.cloudpickle.compatr   ray.exceptionsr   ray.job_configr   ray.util.client.client_picklerr   r   ray.util.client.commonr   r   r   r   r   r   r   r   r    r!   ray.util.client.dataclientr"   ray.util.client.logsclientr#   ray.util.debugr$   	ray.actorr%   ray.remote_functionr&   	getLoggerr  r   ry   r0   r,   r  __annotations__r-  r.  r/  rW   r3   r6   r   rF   r   r   r   r   r4   r2   <module>r     s    
    				         # # # # # # % % % % % % S S S S S S S S S S S S S S S S S S S S  % % % % % % : : : : : : : : : : : : D D D D D D D D D D D D         
 L K K K K K M M M M M M * ) ) ) ) ) * * * * * * $ $ $ $ $ $ O O O O O O O O                        2 1 1 1 1 1 6 6 6 6 6 6 # # # # # # 3$$$$$$222222		8	$	$!k"KQOO +CRHH (1y.( ( u    $  (u $$x !S S    t7 t7 t7 t7 t7 t7 t7 t7n    
	# 	#) 	# 	# 	# 	# 	# 	#r4   