
    &`i4                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZmZmZ d dlZd dlZd dlZd dlmc mc mZ d dlmc mc mZ d dlmZ d dlm Z m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d d	l(m)Z) d d
l*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3m4Z4m5Z5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z?m@Z@ d dlAmBZB  ejC        eD          ZE e'dd          ZFd ZG G d dejH                  ZId ZJdeKfdZLdejM        deeeKef                  fdZNd)deKdeOfd ZPdeKdeOfd!ZQd*d#ZRd)d$ZSd%eeK         dee/         fd&ZTd' ZUeDd(k    r eU             dS dS )+    N)defaultdict)AnyCallableDictListOptionalSetUnion)cloudpickle)build_addressis_localhost)ray_constants)disable_client_hook)env_integer)setup_logger)%canonicalize_bootstrap_address_or_die)add_port_to_grpc_server)	GcsClient	JobConfig)CLIENT_SERVER_MAX_THREADSGRPC_OPTIONSOBJECT_TRANSFER_CHUNK_SIZEClientServerHandleResponseCache)DataServicer)LogstreamServicer)serve_proxier)dumps_from_serverloads_from_client)current_serverTIMEOUT_FOR_SPECIFIC_SERVER_S   c                 F     t          j                    fd            }|S )z
    Decorator for gRPC stubs. Before calling the real stubs, checks if there's
    an existing entry in the caches. If there is, then return the cached
    entry. Otherwise, call the real function and use the real cache
    c                    t          |                                          d}t          fd|D                       r | ||          S d         }d         }t          d                   }| j        |         }|                    ||          }|_t          |t                    rH|                    t          j
        j                   |                    t          |                     ||S 	  | ||          }	np# t          $ rc}
|                    |||
           |                    t          j
        j                   |                    t          |
                      d }
~
ww xY w|                    |||	           |	S )N)	client_id	thread_idreq_idc              3       K   | ]}|vV  	d S N ).0imetadatas     q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/util/client/server/server.py	<genexpr>z7_use_response_cache.<locals>.wrapper.<locals>.<genexpr><   s(      77Qq 777777    r&   r'   r(   )dictinvocation_metadataanyintresponse_cachescheck_cache
isinstance	Exceptionset_codegrpc
StatusCodeFAILED_PRECONDITIONset_detailsstrupdate_cache)selfrequestcontextexpected_idsr&   r'   r(   response_cachecached_entryresper.   funcs              @r/   wrapperz$_use_response_cache.<locals>.wrapper8   s   335566;7777,77777 	04gw/// [)	[)	Xh'(( -i8%11)VDD#,	22 #  !DEEE##C$5$5666""		4gw//DD 	 	 	 ''	61===T_@AAAA'''	 	##Ivt<<<s   8D 
E3AE..E3)	functoolswraps)rI   rJ   s   ` r/   _use_response_cacherM   1   s;     _T" " " " "H Nr1   c                      e Zd ZdefdZ	 d.dej        dej        fdZe	d.dej
        fd            Zd.dej        fdZe	d.dej        fd	            Zd.dej        fd
Zd.dej        fdZ	 d.dej        fdZd.dej        fdZd.defdZdededefdZd Zd Zd Zd Z e	d.d            Z!	 d.dej"        dede#de$j%        de&ej'                 f
dZ(dej"        fdZ)dej"        defdZ*	 d.dej+        dej,        fdZ-	 d.de.ee/f         dededefd Z0d.dej1        fd!Z2	 d.d"ej3        d#e4e5         d$e6ee5f         dej7        fd%Z8	 d.d"ej3        d#e4e5         d$e6ee5f         dej7        fd&Z9	 d.d"ej3        d#e4e5         d$e6ee5f         dej7        fd'Z:	 d.d"ej3        d#e4e5         d$e6ee5f         dej7        fd(Z;	 d.d"ej3        dej7        fd)Z<deded*e&e6         de=j>        j?        fd+Z@deded*e&e6         fd,ZAd- ZBdS )/RayletServicerray_connect_handlerc                 \   t          t                    | _        t          t                    | _        i | _        i | _        t          t                    | _        i | _        t                      | _	        t          j                    | _        || _        t          t                    | _        dS )zuConstruct a raylet service

        Args:
           ray_connect_handler: Function to connect to ray cluster
        N)r   r2   object_refsclient_side_ref_mapfunction_refs
actor_refssetactor_ownersregistered_actor_classesnamed_actors	threadingLock
state_lockrP   r   r6   )rA   rP   s     r/   __init__zRayletServicer.__init__a   s     CNdBSBSBMdBSBS 8:3>s3C3C(*%EE#.**#6 9D]9S9Sr1   NrB   returnc                 L   |j         r!t          j        |j                   }d|_        nd }d }t	                      5  t          j                    r0t
          j        j        j	        }|j
                                        }nt          j        |j        pd          }	  | j        |fi | nV# t          $ rI}t                               d           t%          j        dd|           cY d }~cd d d            S d }~ww xY wd d d            n# 1 swxY w Y   |t%          j        d          S |                                }|rt+          |j        j        j                  }|                    |j        j        j                   t+          |j        j        j                  }	|	                    |j        j        j                   ||	k    rCt7          |          dk    r0t%          j        dd	|j        j         d
|j        j                   S t%          j        d          S )NTz{}zRunning Ray Init failed:Fz0Call to `ray.init()` on the server failed with: )okmsgr`   r   z.Runtime environment doesn't match request one z current one )
job_configpickleloads_client_jobr   rayis_initialized_privateworkerglobal_workercore_workerget_job_configjsonray_init_kwargsrP   r9   logger	exceptionray_client_pb2InitResponse_get_proto_job_configrV   runtime_env_infourisworking_dir_uriupdatepy_modules_urislen)
rA   rB   rC   rc   current_job_configrj   extra_kwargsrH   job_uriscurrent_job_uriss
             r/   InitzRayletServicer.Initt   s     	g&899J%)J""J! "" 	 	!## ,:%+%7%F%F%H%H""#z'*A*ITJJ,D,ZHH<HHHH    $$%?@@@)6 RqRR       	 	 	 	 	 	 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 !.$7777  5577
  	:6;KLLHOOJ7<LMMM""38H    ##"38H   +++H0A0A%2N#-#>#CN N#5#F#KN N    *d3333sB   ADB*)D*
C=42C8&C='D8C==DDDc                 F   	 t                      5  t          j        j                            |j        |j        |j        |j                  }d d d            n# 1 swxY w Y   n)# t          $ r}t          ||           d}Y d }~nd }~ww xY wt          j        |          S )N)	overwrite	namespaceF)already_exists)r   rg   experimentalinternal_kv_internal_kv_putkeyvaluer   r   r9   return_exception_in_contextrr   KVPutResponse)rA   rB   rC   r   rH   s        r/   KVPutzRayletServicer.KVPut   s    
	#$&&  !$!1!=!N!NKM%/%/	 "O " "                	# 	# 	#'7333"NNNNNN	# +>JJJJs:   A% =AA% AA%  A!A% %
B/BBc                 .   	 t                      5  t          j        j                            |j        |j                  }d d d            n# 1 swxY w Y   n)# t          $ r}t          ||           d}Y d }~nd }~ww xY wt          j
        |          S )Nr   r1   )r   )r   rg   r   r   _internal_kv_getr   r   r9   r   rr   KVGetResponse)rA   rB   rC   r   rH   s        r/   KVGetzRayletServicer.KVGet   s    	$&&  (4EEK7+< F                  	 	 	'7333EEEEEE	 +%8888:   A 1AA AA AA 
A?#A::A?c                 :   	 t                      5  t          j        j                            |j        |j        |j                  }d d d            n# 1 swxY w Y   n)# t          $ r}t          ||           d}Y d }~nd }~ww xY wt          j        |          S )N)del_by_prefixr   r   )deleted_num)r   rg   r   r   _internal_kv_delr   r   r   r9   r   rr   KVDelResponse)rA   rB   rC   r   rH   s        r/   KVDelzRayletServicer.KVDel   s    		$&&  !.:KKK")"7%/ L                  	 	 	'7333KKKKKK	 +DDDDs:   A 7AA AA AA 
B)B  Bc                 .   	 t                      5  t          j        j                            |j        |j                  }d d d            n# 1 swxY w Y   n)# t          $ r}t          ||           g }Y d }~nd }~ww xY wt          j
        |          S )Nr   )keys)r   rg   r   r   _internal_kv_listprefixr   r9   r   rr   KVListResponse)rA   rB   rC   r   rH   s        r/   KVListzRayletServicer.KVList   s    	$&&  '3EENg.? F                  	 	 	'7333DDDDDD	 ,$7777r   c                 .   	 t                      5  t          j        j                            |j        |j                  }d d d            n# 1 swxY w Y   n)# t          $ r}t          ||           d}Y d }~nd }~ww xY wt          j
        |          S )Nr   F)exists)r   rg   r   r   _internal_kv_existsr   r   r9   r   rr   KVExistsResponse)rA   rB   rC   r   rH   s        r/   KVExistszRayletServicer.KVExists   s    	$&&  )5IIK7+< J                  	 	 	'7333FFFFFF	 .f====r   c                     t                      5  t          j                            |j                  }d d d            n# 1 swxY w Y   t          j        t          j        |                    S )N)all_namespaces)actors_json)	r   rg   utillist_named_actorsr   rr   ClientListNamedActorsResponsern   dumps)rA   rB   rC   actorss       r/   ListNamedActorszRayletServicer.ListNamedActors   s     !"" 	W 	WX//w?U/VVF	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W ;
6**
 
 
 	
s   &AAAc                    t          j                    }|j        |_        |j        t           j        j        k    rt                      5  t          j                    }d d d            n# 1 swxY w Y   d |                                D             }|j	        
                    t           j                            |                     n,|j        t           j        j        k    rt                      5  t          j                    }d d d            n# 1 swxY w Y   d |                                D             }|j	        
                    t           j                            |                     n|j        t           j        j        k    r%t           j                                        }t                      5  t          j                    }t          j        j                            |                                          |_        t          j        j                            |                                          |_        |j        |_        |j        |_        |j        |_        |                                |_        d d d            n# 1 swxY w Y   |j        
                    |           nAt                      5  |                     ||          |_         d d d            n# 1 swxY w Y   |S )Nc                 4    i | ]\  }}|t          |          S r+   floatr,   kvs      r/   
<dictcomp>z.RayletServicer.ClusterInfo.<locals>.<dictcomp>   $    IIItq!q%((IIIr1   )tablec                 4    i | ]\  }}|t          |          S r+   r   r   s      r/   r   z.RayletServicer.ClusterInfo.<locals>.<dictcomp>  r   r1   )!rr   ClusterInfoResponsetypeClusterInfoTypeCLUSTER_RESOURCESr   rg   cluster_resourcesitemsresource_tableCopyFromResourceTableAVAILABLE_RESOURCESavailable_resourcesRUNTIME_CONTEXTRuntimeContextget_runtime_context_commonutilshex_to_binary
get_job_idjob_idget_node_idnode_idr   -should_capture_child_tasks_in_placement_groupcapture_client_tasksgcs_addressget_runtime_env_stringruntime_envruntime_context_return_debug_cluster_inforn   )rA   rB   rC   rG   	resourcesfloat_resourcesctxrtcs           r/   ClusterInfozRayletServicer.ClusterInfo   s   133L	<>9KKK$&& 4 4133	4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 JIy7H7HIIIO((2@@@WW    \^;OOO$&& 6 6355	6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 JIy7H7HIIIO((2@@@WW    \^;KKK 4CCEEC$&& 	? 	?-// [.<<S^^=M=MNN
!k/==coo>O>OPP #E ( #&/"%"<"<">">	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?  ))#....$&& N N ;;GWMM	N N N N N N N N N N N N N N NsI   A((A,/A,3DDD=CJJJK,,K03K0c                    d}|j         t          j        j        k    rt	          j                    }n|j         t          j        j        k    rt	          j                    }n|j         t          j        j        k    rt	          j	                    }nl|j         t          j        j
        k    ri }nO|j         t          j        j        k    r&dt          j        j                                        i}nt          d          t!          j        |          S )z9Handle ClusterInfo requests that only return a json blob.Ndashboard_urlzUnsupported cluster info type)r   rr   r   NODESrg   nodesIS_INITIALIZEDrh   TIMELINEtimelinePINGDASHBOARD_URLri   rj   get_dashboard_url	TypeErrorrn   r   )rA   rB   rC   datas       r/   r   z)RayletServicer._return_debug_cluster_info  s    <>9???9;;DD\^;JJJ%''DD\^;DDD<>>DD\^;@@@DD\^;III#S\%8%J%J%L%LMDD;<<<z$r1   r&   idc                    | j         5  || j        v r]|| j        |         v rNt                              d|                                 d|            | j        |         |= 	 d d d            dS || j        v r|| j        |         v rt                              d|                                 d|            | j        |                             |           |                     |          r7t                              d|                                            | j        |= 	 d d d            dS 	 d d d            dS # 1 swxY w Y   d S )NzReleasing object z for TzReleasing actor Deleting reference to actor F)	r\   rR   rp   debughexrW   remove_can_remove_actor_refrU   )rA   r&   r   s      r/   releasezRayletServicer.release(  s   _ 	 	D,,,))444LL!ORVVXX!O!OI!O!OPPP(3B7	 	 	 	 	 	 	 	 D---*9555LL!NBFFHH!N!N9!N!NOOO%i077;;;11"55 0%NBFFHH%N%NOOO OB/	 	 	 	 	 	 	 	  !	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AE/B7E3EEEc                     | j         5  |                     |           |                     |           d d d            n# 1 swxY w Y   t          j                     d S r*   )r\   _release_objects_release_actorsgccollect)rA   r&   s     r/   release_allzRayletServicer.release_all;  s    _ 	, 	,!!),,,  +++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	
s   +?AAc                     t          fd| j                                        D                        }|o| j        vS )Nc              3       K   | ]}|v V  	d S r*   r+   )r,   
actor_listactor_id_bytess     r/   r0   z7RayletServicer._can_remove_actor_ref.<locals>.<genexpr>F  s9       
 
-7Nj(
 
 
 
 
 
r1   )r4   rW   valuesrY   )rA   r   no_owners    ` r/   r   z$RayletServicer._can_remove_actor_refE  sa     
 
 
 
;?;L;S;S;U;U
 
 
 
 
 
 CN$2CCCr1   c                    || j         vrt                              d|            d S t          | j         |                   }| j         |= || j        v r| j        |= || j        v r| j        |= t                              d| d|            d S )Nz%Releasing client with no references: Released all z objects for client )rR   rp   r   rz   rS   r6   )rA   r&   counts      r/   r   zRayletServicer._release_objectsK  s    D,,,LLLLLMMMFD$Y/00Y'000(3,,,$Y/KUKK	KKLLLLLr1   c                 z   || j         vrt                              d|            d S d}| j                             |          }|D ]S}|dz  }|                     |          r7t                              d|                                            | j        |= Tt                              d| d|            d S )Nz!Releasing client with no actors: r      r   r   z actors for client: )rW   rp   r   popr   r   rU   )rA   r&   r   actors_to_removeid_bytess        r/   r   zRayletServicer._release_actorsW  s    D---LLHYHHIIIF,00;;( 	. 	.HQJE))(33 .LHLLNNLLMMMOH-KUKK	KKLLLLLr1   c                    |                     d          dk    r	 | j        |j                 |j        j                 }t                      5  t          j        ||j        j        |j        j	                   d d d            n# 1 swxY w Y   n# t          $ r}t          ||           Y d }~nd }~ww xY w|                     d          dk    r	 | j        |j        j                 }t                      5  t          j        ||j        j                   d d d            n# 1 swxY w Y   n6# t          $ r}t          ||           Y d }~nd }~ww xY wt!          d          t#          j        d          S )	Nterminate_typetask_object)force	recursiveactor)
no_restartzEClient requested termination without providing a valid terminate_typeTrb   )
WhichOneofrR   r&   r  r   r   rg   cancelr  r  r9   r   rU   r  killr  RuntimeErrorrr   TerminateResponse)rA   reqrC   
object_refrH   	actor_refs         r/   	TerminatezRayletServicer.Terminatef  s+   >>*++}<<	8!-cm<S_=OP
(**  J"!o3"%/";                   8 8 8+Aw777777778^^,--888 OCIL9	(** I IHY393GHHHHI I I I I I I I I I I I I I I 8 8 8+Aw777777778 W   /48888sw   0B ,B7B BB 
BB 
B3B..B3%D. 5!D"D. "D&&D. )D&*D. .
E8EEr(   result_queuec                     t          j                  dk    rt          d           j        d         } j                                     |d          }|s;t          j        dt          j        t          d| d                               S 	 t          
                    d	|z             t                      5  d
t          ddf fd}|                    |           	 ddd           dS # 1 swxY w Y   dS # t          $ r2}	t          j        dt          j        |	                    cY d}	~	S d}	~	ww xY w)zAttempts to schedule a callback to push the GetResponse to the
        main loop when the desired object is ready. If there is some failure
        in scheduling, a GetResponse will be immediately returned.
        r   z3Async get() must have exactly 1 Object ID. Actual: r   NFzClientObjectRef with id z not found for client validerrorzasync get: %sresultr^   c                    	 t          |           }t          |          }|dk    s
J d            t          j        |t          z            }t          j        |          D ]s}|t          z  }t          ||dz   t          z            }t          j	        d|||         |||          }t          j
        |          }                    |           td	S # t          $ r^}	t          j	        dt          j        |	                    }t          j
        |          }
                    |
           Y d	}	~	d	S d	}	~	ww xY w)
zPushes GetResponses to the main DataPath loop to send
                    to the client. This is called when the object is ready
                    on the server side.r   &Serialized object cannot be zero bytesr   Tr  r   chunk_idtotal_chunks
total_size)getr(   Fr  N)r   rz   mathceilr   rangestart_chunk_idminrr   GetResponseDataResponseputr9   r   r   )r  
serializedr  r  r  startendget_resp
chunk_respexcrG   r&   r(   rB   r  rA   s              r/   send_get_responsez;RayletServicer._async_get_object.<locals>.send_get_response  s   /%6vy$%O%O
%(__
)A~~~/W~~~'+y&)CC( ( ).g.Dl(S(S 9 9H$,/I$IE"% *X\=W,W# #C (6'A&*%/c	%:)1-9+5( ( (H *8)D$,V* * *J ),,Z88889 9  % / / /#1#="'{/@/E/E$ $ $  .:xPVWWW$((........./s   CC 
E$AD==E)rz   ids
ValueErrorrR   r  rr   r"  r   r   rp   r   r   r   _on_completedr9   )
rA   rB   r&   r(   r  rC   ridrefr+  rH   s
   `````     r/   _async_get_objectz RayletServicer._async_get_object  s    w{q  OgOO   k!ny)--c488 		!-!'.3 . ."+. .     *	WLL3.///$&& % % /c  /d  /  /  /  /  /  /  /  /  /  /D !!"3444K% % % % % % % % % % % % % % % % % %N  	W 	W 	W!-EARSTAUAUVVVVVVVVV	WsB   +D *C>1D >DD DD 
E'E<EEc              #   $  K   t          |                                          }|                    d          }|9t          j        dt          j        t          d                              V  d S |                     ||          E d {V  d S )Nr&   Fz.client_id is not specified in request metadatar  )	r2   r3   r  rr   r"  r   r   r-  _get_object)rA   rB   rC   r.   r&   s        r/   	GetObjectzRayletServicer.GetObject  s      335566LL--	 ,!'OPP         '';;;;;;;;;;;r1   c              #     K   g }|j         D ]z}| j        |                             |d           }|r|                    |           ;t	          j        dt          j        t          d| d|                               V   d S 	 t          
                    d|z             t                      5  t          j        ||j                  }d d d            n# 1 swxY w Y   nB# t          $ r5}t	          j        dt          j        |                    V  Y d }~d S d }~ww xY wt          |||           }t!          |          }	|	dk    s
J d            t#          j        |	t&          z            }
t)          |j        |
          D ]J}|t&          z  }t-          |	|d	z   t&          z            }t	          j        d
|||         ||
|	          V  Kd S )NFzClientObjectRef z is not found for client r  zget: %s)timeoutr   r  r   Tr  )r,  rR   r  appendrr   r"  r   r   r-  rp   r   r   rg   r6  r9   r   rz   r  r  r   r  r   r!  )rA   rB   r&   
objectrefsr/  r0  r   rH   r%  r  r  r  r&  r'  s                 r/   r3  zRayletServicer._get_object  s     
; 	 	C"9-11#t<<C !!#&&&&$0%+"XsXXYXX        	LLZ/000$&& E E
GODDDE E E E E E E E E E E E E E E 	 	 	 ,5@QRS@T@TUUUUUUFFFFF	 'ui>>
__
A~~~G~~~y.H!HIIg4lCC 		 		H99Ej8a<3M"MNNC ,c	*!)%      		 		s<   +C' 3CC' CC' "C#C' '
D&1*D!!D&c                 R    |                      |j        |j        d|j        |          S )z#gRPC entrypoint for unary PutObject )_put_objectr   client_ref_idowner_id)rA   rB   rC   s      r/   	PutObjectzRayletServicer.PutObject  s/     L'/W5Ew
 
 	
r1   r   r<  r=  c                    	 t          ||           }|r| j        |         }nd}t                      5  t          j        ||          }ddd           n# 1 swxY w Y   nZ# t
          $ rM}	t                              d           t          j	        ddt          j        |	                    cY d}	~	S d}	~	ww xY w|| j        |         |                                <   t          |          dk    r"|                                | j        |         |<   t                              d|z             t          j	        |                                d	
          S )a  Put an object in the cluster with ray.put() via gRPC.

        Args:
            data: Pickled data. Can either be bytearray if this is called
              from the dataservicer, or bytes if called from PutObject.
            client_ref_id: The id associated with this object on the client.
            client_id: The client who owns this data, for tracking when to
              delete this reference.
            owner_id: The owner id of the object.
            context: gRPC context.
        N)_ownerzPut failed:r1   F)r   r  r  r   zput: %sT)r   r  )r    rU   r   rg   r$  r9   rp   rq   rr   PutResponser   r   rR   binaryrz   rS   r   )
rA   r   r<  r&   r=  rC   objowner	objectrefrH   s
             r/   r;  zRayletServicer._put_object  s   &	#D$//C 1$&& 7 7GC666	7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 	 	 	]+++!-e;+<Q+?+?        	 ;D#I$4$4$6$67}!!AJAQAQASASD$Y/>Y*+++)Y-=-=-?-?tLLLLsA   0A! A	A! AA! AA! !
B8+AB3-B83B8c                    g }|j         D ]`}|| j        |j                 vrt          dt	          |          z            |                    | j        |j                 |                    a|j        }|j        }	 t                      5  t          j
        |||dk    r|nd           \  }}d d d            n# 1 swxY w Y   nI# t          $ r<}	t                              d|	            t          j        d          cY d }	~	S d }	~	ww xY wt                              dt	          |          dt	          |                     d	 |D             }
d
 |D             }t          j        d|
|          S )Nz4Asking for a ref not associated with this client: %s)num_returnsr6  z
Exception F)r  zwait:  c                 6    g | ]}|                                 S r+   rB  )r,   ready_object_refs     r/   
<listcomp>z-RayletServicer.WaitObject.<locals>.<listcomp>C  s2     
 
 
*:##%%
 
 
r1   c                 6    g | ]}|                                 S r+   rK  )r,   remaining_object_refs     r/   rM  z-RayletServicer.WaitObject.<locals>.<listcomp>F  s4      
  
  
$ !'')) 
  
  
r1   T)r  ready_object_idsremaining_object_ids)
object_idsrR   r&   r9   r?   r7  rH  r6  r   rg   waitrp   r  rr   WaitResponser   )rA   rB   rC   rR   r/  rH  r6  ready_object_refsremaining_object_refsrH   rP  rQ  s               r/   
WaitObjectzRayletServicer.WaitObject+  s"   % 	I 	IC$*7+<===JSQTXXU   t/0AB3GHHHH)/
	<$&&  ;>8 +'."}}GG$< < <8!#8                	< 	< 	<LL)a))***!.U;;;;;;;;;	< 	 !23333S9N5O5O5OP	
 	
 	

 
>O
 
 
 
  
(= 
  
  
 *-!5
 
 
 	
sB   :C #B7+C 7B;;C >B;?C 
D	1D>D	D	taskarglistkwargsc                    t                               d|j        dt          j        j                            |j                             	 t                      5  |j        t          j        j	        k    r| 
                    ||||          }n|j        t          j        j        k    r|                     ||||          }n|j        t          j        j        k    r|                     ||||          }nj|j        t          j        j        k    r|                     ||          }n9t#          dt          j        j                            |j                  z            d|_        |cd d d            S # 1 swxY w Y   d S # t&          $ rN}t                               dd           t          j        dt+          j        |                    cY d }~S d }~ww xY w)	Nz
schedule: rI  z$Unimplemented Schedule task type: %sTzCaught schedule exception)exc_infoFr  )rp   r   namerr   
ClientTaskRemoteExecTypeNamer   r   FUNCTION_schedule_functionACTOR_schedule_actorMETHOD_schedule_methodNAMED_ACTOR_schedule_named_actorNotImplementedErrorr  r9   ClientTaskTicketr   r   )rA   rX  rY  rZ  rC   r  rH   s          r/   SchedulezRayletServicer.ScheduleP  s*    	yyy.3BGG	RRRT	
 	
 	
	$&&  9 9 BBB!44T7FGTTFFY.";"AAA!11$QQFFY.";"BBB!224&'RRFFY.";"GGG!77gFFFF->(3BGG	RRS    $                    	 	 	LL4tLDDD!2;#4Q#7#7        	sD   F DE5(F 5E99F <E9=F 
GAGGGc                 L   | j                             |j                  }|t          d          t	          ||j                  }t          |j                  }| |j        di |} |j        |i |}| 	                    ||j
                  }	t          j        |	          S )Nz7Can't run an actor the server doesn't have a handle for
return_idsr+   )rU   r  
payload_idr9   getattrr]  decode_optionsoptionsremoteunify_and_track_outputsr&   rr   rj  )
rA   rX  rY  rZ  rC   actor_handlemethodoptsoutputr,  s
             r/   rf  zRayletServicer._schedule_methodr  s     **4?;;UVVVty11dl++#V^++d++F2622**64>BB.#>>>>r1   c                 *   |                      |j        |j        t          |j                            }t          |j                  }| |j        di |}t          |           5   |j        |i |}d d d            n# 1 swxY w Y   || j        |j	        
                                <   | j        |j                                     |j	        
                                           t          j        |j	        
                                g          S Nrm  r+   )lookup_or_register_actorro  r&   rq  baseline_optionsrr  r!   rs  rU   	_actor_idrB  rW   addrr   rj  )rA   rX  rY  rZ  rC   remote_classrw  r  s           r/   rd  zRayletServicer._schedule_actor  sC    44OT^^D<Q-R-R
 
 dl++/</77$77LD!! 	< 	<'L';F;;E	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	<49..001$.)--eo.D.D.F.FGGG.5?;Q;Q;S;S:TUUUU   &B  BBc                 v   |                      |j        |j        t          |j                            }t          |j                  }| |j        di |}t          |           5   |j        |i |}d d d            n# 1 swxY w Y   |                     ||j                  }t          j
        |          S rz  )lookup_or_register_funcro  r&   rq  r|  rr  r!   rs  rt  rr   rj  )	rA   rX  rY  rZ  rC   remote_funcrw  rx  r,  s	            r/   rb  z!RayletServicer._schedule_function  s    22OT^^D<Q-R-R
 
 dl++-+-5555KD!! 	< 	<'[';F;;F	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	<**64>BB.#>>>>r  c                    t          |j                  dk    sJ t          j        |j        |j        pd           }|j                                        }|| j        vr
|| j        |<   | j	        |j
                                     |           | j                            |           t          j        |j                                        g          S )Nr   rm  )rz   ro  rg   	get_actorr]  r   r}  rB  rU   rW   r&   r~  rY   rr   rj  )rA   rX  rC   r  bin_actor_ids        r/   rh  z$RayletServicer._schedule_named_actor  s     4?##q((((di)?4@@--//t..,1DOL)$.)--l;;;l+++.5?;Q;Q;S;S:TUUUUr1   rr  c                    t                      5  || j        vr| j        |         |         }t          j        |          }t          j        |          st          d          |t          |          dk    rt          j	        |          | j        |<   n" t          j	        di ||          | j        |<   d d d            n# 1 swxY w Y   | j        |         S )Nz6Attempting to register function that isn't a function.r   r+   )
r   rT   rR   rg   r  inspect
isfunctionr9   rz   rs  )rA   r   r&   rr  funcrefrI   s         r/   r  z&RayletServicer.lookup_or_register_func  s5    !"" 	I 	I+++*95b9ww'')$// #P   ?c'lla&7&7-0Z-=-=D&r**-BSZ-B-B'-B-B4-H-HD&r*	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I !"%%s   B(CC
Cc                    t                      5  || j        vr| j        |         |         }t          j        |          }t          j        |          st          d          |t          |          dk    rt          j	        |          }n t          j	        di ||          }|| j        |<   d d d            n# 1 swxY w Y   | j        |         S )Nz0Attempting to schedule actor that isn't a class.r   r+   )
r   rX   rR   rg   r  r  isclassr9   rz   rs  )rA   r   r&   rr  actor_class_refactor_class	reg_classs          r/   r{  z'RayletServicer.lookup_or_register_actor  s    !"" 
	> 
	>666"&"29"=b"A!go66{33 X#$VWWW?c'lla&7&7 #
; 7 7II 5
 5 5W 5 5k B BI4=-b1
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> ,R00s   B"B==CCc                 $   |g }nt          |t                    r|}n|g}|D ]b}|                                | j        |         v rt                              d|            || j        |         |                                <   cd |D             S )NzAlready saw object_ref c                 6    g | ]}|                                 S r+   rK  )r,   outs     r/   rM  z:RayletServicer.unify_and_track_outputs.<locals>.<listcomp>  s     000

000r1   )r8   listrB  rR   rp   warning)rA   rx  r&   outputsr  s        r/   rt  z&RayletServicer.unify_and_track_outputs  s    >GG%% 	GGhG 	< 	<Czz||t/	:::>>>???8;DY'

55000000r1   r*   )C__name__
__module____qualname__r   r]   rr   InitRequestrs   r   rM   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r?   r   bytesboolr   r   r   r   r   r  
GetRequestr5   queueQueuer   r"  r1  r4  r3  
PutRequestrA  r>  r
   	bytearrayr;  rT  rW  r^  r   r   r   rj  rk  rf  rd  rb  rh  rg   remote_functionRemoteFunctionr  r{  rt  r+   r1   r/   rO   rO   `   s       TH T T T T( <@24 24%124		$24 24 24 24h K Kn.J K K K K	9 	9n.J 	9 	9 	9 	9 E En.J E E E E	8 	8~/L 	8 	8 	8 	8	> 	>1P 	> 	> 	> 	>  $
 
		5
 
 
 
% %N4V % % % %N   3        " % D    &  D D D
M 
M 
MM M M 9 9 9 9> FW FW*FW FW 	FW
 kFW 
.,	-FW FW FW FWP<!: < < < <$>#< $ $ $ $ $N ;?
 
%0
		#
 
 
 
 &M &ME9$%&M &M 	&M
 &M &M &M &MP#
 #
>3N #
 #
 #
 #
T    '  c  S#X	  
	(       N ? ?'? c? S#X	? 
	(? ? ? ?. V V'V cV S#X	V 
	(V V V V0 ? ?'? c? S#X	? 
	(? ? ? ?& 8<V V"-V		(V V V V&&$'&2:4.&			+& & & &"11$'12:4.1 1 1 1"1 1 1 1 1r1   rO   c                     |H|                     t          |                      |                    t          j        j                   d S d S r*   )r>   encode_exceptionr:   r;   r<   ABORTED)errrC   s     r/   r   r     sN    ,S11222 	011111 r1   r^   c                 v    t          j        |           }t          j        |                                          S r*   )r   r   base64standard_b64encodedecode)rq   r   s     r/   r  r    s/    Y''D$T**11333r1   rr  c                 x    | j         sd S t          j        | j                   }t          |t                    sJ |S r*   )pickled_optionsrd   re   r8   r2   )rr  rw  s     r/   rq  rq    s?    " t</00DdD!!!!!Kr1   hostportc                 *   	 ddt           dt          t          t          f         fd}ddlm} |p|} |t          dt          d          }t          |          }t          |          }t                      }t          j        ||           t          j        ||           t          j        ||           t          |           st!          |d	|            t!          ||  d
|            t#          ||||          }	|                                 |	S )Nrc   ro   c                     t                      5  t          j                    st          j        dd| i|cd d d            S 	 d d d            d S # 1 swxY w Y   d S )Nrc   r+   )r   rg   rh   init)rc   ro   s     r/   default_connect_handlerz&serve.<locals>.default_connect_handler  s     !"" 	J 	J%'' JxII:III	J 	J 	J 	J 	J 	J 	J 	JJ	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	Js   &AAAr   )$create_grpc_server_with_interceptorsray_client_serverF)max_workersthread_name_prefixrr  asynchronousz
127.0.0.1::)task_servicerdata_servicerlogs_servicergrpc_serverr*   )r   r   r?   r   ray._private.grpc_utilsr  r   r   rO   r   r   ray_client_pb2_grpc"add_RayletDriverServicer_to_server(add_RayletDataStreamerServicer_to_server'add_RayletLogStreamerServicer_to_serverr   r   r   r&  )
r  r  rP   r  r  serverr  r  r  current_handles
             r/   server    s_    $J JJ9=c3hJ J J J MLLLLL-H1H11-.	  F ##677M //M%''M:=&QQQ@PVWWW?vVVV =(;T(;(;<<<Ft$4$4d$4$4555'###	  N LLNNNr1   c                     t                      5  t          j        i d d d            n# 1 swxY w Y   dfd	}t          | ||          }|fS )Nc                 T    t          j                    rS t          j        d| iS )Nrc   )rg   rh   r  )rc   ro   argsinforZ  s     r/   rP   z+init_and_serve.<locals>.ray_connect_handler&  s6      	DK8DCzCFCCCr1   )rP   r*   )r   rg   r  r  )r  r  r  rZ  rP   server_handler  s     ``  @r/   init_and_server  !  s    			 ) )x((() ) ) ) ) ) ) ) ) ) ) ) ) ) )D D D D D D D D $:MNNNM4  s   044Fc                     |                      d           t                      5  t          j        |           d d d            d S # 1 swxY w Y   d S )Nr   )stopr   rg   shutdown)r  _exiting_interpreters     r/   shutdown_with_serverr  2  s    
KKNNN			 + +)***+ + + + + + + + + + + + + + + + + +s   AA
A
c                 ,     ddt           f fd}|S )Nrc   c                     r0rt          j        d| d| d S t          j        d| d| d S t          j        dd| i| d S )N)address_redis_username_redis_passwordrc   )r  rc   rc   r+   )rg   r  )rc   ro   r  redis_passwordredis_usernames     r/   rP   z/create_ray_handler.<locals>.ray_connect_handler9  s     	? 	T #$2$2)	 
 &     SZSS?SSSSSH>>
>o>>>>>r1   r*   r   )r  r  r  rP   s   ``` r/   create_ray_handlerr  8  sA    ? ?	 ? ? ? ? ? ? ? ? r1   r  c                 @    t          |           } t          |           S )zr
    Try to create a gcs client based on the command line args or by
    autodetecting a running Ray cluster.
    r  )r   r   r  s    r/   try_create_gcs_clientr  K  s"    
 4G<<GW%%%%r1   c                     dd l } |                                 }|                    dt          dd           |                    ddt          dd	           |                    d
t          g dd           |                    ddt          d           |                    ddt          d           |                    ddt          d           |                    ddt          d d           |                                \  }}t          t          j        t          j	                   t          |j        |j        |j                  }t          |j        |j                  }t          |          }|j        r|                    |j        d          }t$                              d| d|            |j        dk    r4t+          |j        |j        |j        |j        |j        |j                  }nt/          |j        |j        |          }	 t0          }	 dt3          j                    i}		 t4          j        j                                        s8t=          |j                  }
t4          j        j                            |
           t4          j        j                             dtC          j"        |	          t          j#                    n[# tH          $ rN}t$          %                    d!|j         d"|j                    t$          &                    |           Y d }~nd }~ww xY wt3          j'        d#           |j        d$k    rb|j(        j)        dk    rt0          }n|d#z  }|dk    rtU                      |d%z  dk    r(|t0          k    rt$                              | d&           # tT          $ r |+                    d           Y d S w xY w)'Nr   z--hostz0.0.0.0zHost IP to bind to)r   defaulthelpz-pz--porti'  zPort to bind toz--mode)proxylegacyspecific-serverr  )r   choicesr  z	--addressFz Address to use to connect to Ray)requiredr   r  z--redis-usernamez username for connecting to Redisz--redis-passwordz Password for connecting to Redisz--runtime-env-agent-addressz8The port to use for connecting to the runtime_env_agent.)r  r   r  r  z****zStarting Ray Client server on z, args )r  r  runtime_env_agent_addressTtimer  r   [z ] Failed to put health check on r   r     z idle checks before shutdown.),argparseArgumentParseradd_argumentr?   r5   parse_known_argsr   r   LOGGER_LEVELLOGGER_FORMATr  r  r  r  r   r  r  replacerp   r  moder   r  r  r"   r  rg   r   r   _internal_kv_initializedr  _initialize_internal_kvr   rn   r   KV_NAMESPACE_HEALTHCHECKr9   r  rq   sleepr  num_clientsKeyboardInterruptr  )r  parserr  _rP   hostportargs_strr  idle_checks_remaininghealth_report
gcs_clientrH   s               r/   mainr  T  s8   OOO$$&&F
sI4H     hS%FWXXX
666	     e#4V     /	     /	     %G     %%''GD!+]-HIII,d)4+>  TY	22H4yyH A##D$7@@
KKLLL(LLMMMyGIIL..&*&D
 
 
 ty$)-@AA% = 	Y	M$'3LLNN U!6t|!D!DJ$0HHTTT ,=='J}--+D >    
  $ $ $Q	QQ4<QQ     ########	$ JqMMMy---'3a77,I)))Q.)(A--+---)A-22-1NNNKK#8 W W WXXXA 	YD    As?   N+ .BK N+ 
L%AL N+  L%%BN+ +OO__main__r*   )F)Vr  rK   r   r  rn   loggingr  rd   r  rZ   r  collectionsr   typingr   r   r   r   r   r	   r
   r;   rg   ray._private.state!ray.core.generated.ray_client_pb2core	generatedrr   &ray.core.generated.ray_client_pb2_grpcr  r   ray._common.network_utilsr   r   ray._privater   ray._private.client_mode_hookr   ray._private.ray_constantsr   ray._private.ray_loggingr   ray._private.servicesr   ray._private.tls_utilsr   ray._rayletr   ray.job_configr   ray.util.client.commonr   r   r   r   r   #ray.util.client.server.dataservicerr   "ray.util.client.server.logservicerr   ray.util.client.server.proxierr   %ray.util.client.server.server_picklerr   r    #ray.util.client.server.server_stubsr!   	getLoggerr  rp   r"   rM   RayletDriverServicerrO   r   r?   r  TaskOptionsrq  r5   r  r  r  r  r  r  r+   r1   r/   <module>r     s        				            # # # # # # B B B B B B B B B B B B B B B B B B  



     : : : : : : : : : : : : D D D D D D D D D D D D       A A A A A A A A & & & & & & = = = = = = 2 2 2 2 2 2 1 1 1 1 1 1 G G G G G G : : : : : : ! ! ! ! ! ! $ $ $ $ $ $              = < < < < < @ @ @ @ @ @ 8 8 8 8 8 8 V V V V V V V V > > > > > >		8	$	$ +,KR P P , , ,^C
1 C
1 C
1 C
1 C
1(= C
1 C
1 C
1L2 2 243 4 4 4 4
N6 8DcN;S    ! ! !3 ! ! ! !H! !C ! ! ! !"+ + + +   &&8C= &Xi5H & & & &a a aH zDFFFFF r1   