
    &`i                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZ d dlmZmZmZmZmZ d dlZd dlZd dlmc mc mZ d dlmc mc m Z  d dl!mc mc m"Z" d dl#m$Z$m%Z%m&Z& d dl'm(Z(m)Z) d d	l*m+Z+ d d
l,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z< d dl=m>Z> d dl?m@Z@ d dlAmBZBmCZCmDZDmEZEmFZF d dlGmHZH d dlIZI ejJ        eK          ZLdZMdZNdZOdZPdZQdZRe G d d                      ZSdeeT         deUfdZV G d  d!          ZW G d" d#e jX                  ZYd$e@de@fd%ZZd&ej[        deej[        e@f         fd'Z\ G d( d)          Z] G d* d+e j^                  Z_ G d, d-e j`                  Zaddddd.d/eTd0ebd1eeT         d2eeT         d3eeT         d4eeT         d5eeT         fd6ZcdS )7    N)futures)	dataclass)chain)EventLockRLockThread)CallableDictListOptionalTuple)build_addressis_ipv6is_localhost) format_authentication_http_error get_auth_headers_if_auth_enabled)disable_client_hook)init_grpc_channel)	RayParams)RuntimeEnvContext)ProcessInfostart_ray_client_server)add_port_to_grpc_server)detect_fate_sharing_support)	GcsClient)pickle)AuthenticationError)	JobConfig)CLIENT_SERVER_MAX_THREADSGRPC_OPTIONSClientServerHandle_get_client_id_from_context_propagate_error_in_context)_get_reconnecting_from_context   iY  i]        c                       e Zd ZU eed<   ej        ed<   ded<   defdZdde	e
         ddfd	Zde	e         fd
ZddZde	e         ddfdZdS )SpecificServerportprocess_handle_futuregrpc._channel.Channelchannelreturnc                 4    | j                                         S )z4Check if the server is ready or not (doesn't block).)r,   doneselfs    r/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/util/client/server/proxier.pyis_readyzSpecificServer.is_readyD   s    )..000    Ntimeoutc                 ^    | j                             |          }|t          d          dS )z;
        Wait for the server to actually start up.
        r7   NzServer startup failed.)r,   resultRuntimeError)r3   r7   ress      r4   
wait_readyzSpecificServer.wait_readyH   s8     (///@@;7888 ;r6   c                     	 | j                             d          }||j                                        S dS # t          j        $ r Y dS w xY w)z Check if the process has exited.皙?r9   N)r,   r:   processpollr   TimeoutErrorr3   procs     r4   rA   zSpecificServer.pollQ   sg    	-44S4AAD|((***  # 	 	 	FF	s   5: AAc                     	 | j                             d          }||j                                         dS dS # t          j        $ r Y dS w xY w)z)Try to send a KILL signal to the process.r?   r9   N)r,   r:   r@   killr   rB   rC   s     r4   rF   zSpecificServer.killZ   sm    	-44S4AAD!!#####  # 	 	 	DD	s   6< AArD   c                 f    |                                  s| j                            |           dS dS )z?Set the result of the internal future if it is currently unset.N)r5   r,   
set_resultrC   s     r4   rH   zSpecificServer.set_resultd   s9    }} 	8&11$77777	8 	8r6   Nr/   N)__name__
__module____qualname__int__annotations__r   Futureboolr5   r   floatr=   rA   rF   r   rH    r6   r4   r*   r*   >   s         
III">)))$$$$1$ 1 1 1 19 9(5/ 9T 9 9 9 9hsm       8x4 8 8 8 8 8 8 8r6   r*   commandr/   c                 4    d                     |           }d|v S )z
    Detects if the main process in the given command is the RayClient Server.
    This works by ensuring that the command is of the form:
        <py_executable> -m ray.util.client.server <args>
     z-m ray.util.client.server)join)rT   	flatteneds     r4   _match_running_client_serverrY   j   s      !!I&)33r6   c                   l   e Zd Zddddddee         dedee         dee         dee         d	efd
Zej        fdedefdZ	e
defd            Ze
dej        j        j        fd            ZdedefdZdededefdZdededefdZdedee         fdZdedefdZdeded         fdZd ZddZdS )ProxyManagerNr   )session_dirredis_usernameredis_passwordruntime_env_agent_portaddressruntime_env_agent_addressr\   r]   r^   r_   c                   t                      | _        t                      | _        || _        || _        || _        t          t          t          t                              | _        || _        t          | j        d          | _        | j                                         t#          t%                                | _        d | _        t+          j        | j                   d S )NT)targetdaemon)dictserversr   server_lock_address_redis_username_redis_passwordlistrangeMIN_SPECIFIC_SERVER_PORTMAX_SPECIFIC_SERVER_PORT_free_ports_runtime_env_agent_addressr	   _check_processes_check_threadstartrQ   r   
fate_share_nodeatexitregister_cleanup)r3   r`   ra   r\   r]   r^   r_   s          r4   __init__zProxyManager.__init__u   s     37&& 77--&**,DEE'
 '
 +D'#4+@NNN  """:<<==7;
&&&&&r6   familyr/   c                 H   | j         5  t          | j                  }t          |          D ]}| j                            d          }t          j        |t
          j                  }	 |                    d|f           n># t          $ r1 | j        	                    |           Y |
                                 w xY w	 |
                                 n# |
                                 w xY w|c cddd           S 	 ddd           n# 1 swxY w Y   t          d          )zB
        Search for a port in _free_ports that is unused.
        r    Nz-Unable to succeed in selecting a random port.)rg   lenro   rl   popsocketSOCK_STREAMbindOSErrorappendcloser;   )r3   rz   	num_ports_r+   ss         r4   _get_unused_portzProxyManager._get_unused_port   s     	 	D,--I9%% 
 
'++A..M&&*<==FFB:&&&&   $++D111GGIIII	 '
 GGIIIIAGGIIII	 	 	 	 	 	 	 	
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 JKKKsN   AD
(B ?C $B;$C%D
:B;;C?D
C**D

DDc                     | j         r| j         S t          j                    }|d         | _         |d         | _        | j         S )zW
        Returns the provided Ray bootstrap address, or creates a new cluster.
        r`   r\   )rh   rayinit_session_dir)r3   connection_tuples     r4   r`   zProxyManager.address   sC    
 = 	!= 8::(3,];}r6   c                     | j         r| j         S t          | j                  }t          j        j                            |dddd          | _         | j         S )zGets a 'ray.Node' object for this node (the head node).
        If it does not already exist, one is created using the bootstrap
        address.
        )gcs_addressFT)headshutdown_at_exitspawn_reaperconnect_only)ru   r   r`   r   _privatenodeNode)r3   
ray_paramss     r4   r   zProxyManager.node   sb     : 	:4<888
\&++" , 
 

 zr6   	client_idc                    | j         5  | j                            |          J d|             d}|                     t	          |          rt
          j        nt
          j                  }t          |t          j
                    t          t          ||          t                              }|| j        |<   |cddd           S # 1 swxY w Y   dS )z
        Create, but not start a SpecificServer for a given client. This
        method must be called once per client.
        Nz#Server already created for Client: 	127.0.0.1)options)r+   r,   r.   )rg   rf   getr   r   r   AF_INET6AF_INETr*   r   rP   r   r   r!   )r3   r   hostr+   servers        r4   create_specific_serverz#ProxyManager.create_specific_server   s(   
  	 	  ++33@Y@@ 433 D((#*4==Dfn D $&-n&6&6)!$--|    F '-DL#%	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   B4C		CCserialized_runtime_envruntime_env_configspecific_serverc                 L   t                               d|j         d| d           t          | j                  dk    s
J d            t          j        ||d|j                             d          d	          }d}d
}d}d}||k    r	 t          j	        
                    | j        d          }	|                                }
ddi} |j        d#i t          |           t          j                            |	|
d|          }t          j                            |d          }|                                }t          j                    }|                    |           |j        t
          j        j        k    r|j        S |j        t
          j        j        k    rt3          d|j                   J d|j         d            # t          j        j        $ r}d}	 |                                                    dd          }n6# t<          $ r) t?          |d          r|j         ntC          |          }Y nw xY wtE          |j#        |pd          }|rtI          |          ||}t           %                    d|j#         d|p| d| d||z
   d	           Y d}~nMd}~wt          j        j&        $ r3}|}t           %                    d| d| d||z
   d           Y d}~nd}~ww xY wtO          j(        |           |dz  }|d z  }||k    tS          d!| d"|           )$zIncrease the runtime_env reference by sending an RPC to the agent.

        Includes retry logic to handle the case when the agent is
        temporarily unreachable (e.g., hasn't been started up yet).
        z7Increasing runtime env reference for ray_client_server_z.Serialized runtime env is .r   z!runtime_env_agent_address not setray_client_server_zutf-8client_server)r   r   job_idsource_processr'         ?Nz/get_or_create_runtime_envzContent-Typezapplication/octet-streamPOST)datamethodheadersr9   zEFailed to create runtime_env for Ray client server, it is caused by:
FzUnknown status: r|   ignorereasonz/GetOrCreateRuntimeEnv request failed with HTTP : z. Retrying after zs. z retries remaining.z&GetOrCreateRuntimeEnv request failed:    r(   z+GetOrCreateRuntimeEnv request failed after z attempts. Last exception: rS   )*loggerinfor+   r}   rp   runtime_env_agent_pb2GetOrCreateRuntimeEnvRequestencodeurllibparseurljoinSerializeToStringupdater   requestRequesturlopenreadGetOrCreateRuntimeEnvReplyParseFromStringstatusAgentRpcStatusAGENT_RPC_STATUS_OKserialized_runtime_env_contextAGENT_RPC_STATUS_FAILEDr;   error_messageerror	HTTPErrordecode	Exceptionhasattrr   strr   coder   warningURLErrortimesleeprB   )r3   r   r   r   create_env_requestretriesmax_retrieswait_time_slast_exceptionurlr   r   reqresponseresponse_datarebodyformatted_errors                      r4   _create_runtime_envz ProxyManager._create_runtime_env   sc    	C!0!5C C)?C C C	
 	
 	
 /001444. 544 3O#91>(<>>EEgNN*	
 
 
 $$4l**35Q  *;;==)+EFKK!A'!J!JKKKn,,d67 -   ">11#t1DD ()DFF!!-00084CWWW;;H,;ST T 'G56_G G  
 A"@QX"@"@"@@@5<)   H6688??7H==DD  H H H'.q(';';G188QDDDH #C164:SU"V"V" F-o>>AE "#@af @ @PTPYXY @ @&1@ @"7*@ @ @        <(   !"@Q @ @&1@ @"7*@ @ @        J{###qLG1Ku $$x 1+ 1 1 .1 1
 
 	
sQ   C7G AG K)J(HJ0H;8J:H;;AJK)6)K$$K)
job_configc                    |                      |          }|sJ d|             | j                            d|j         d          \  }}|                                }|                                }|r|dk    r!t                                                      }n|                     |||          }t          | j
        d|j        ||| j        d|| j        | j        	
  
        }	|	j        j        }
t           j        d
k    rt%          j        |
          }nd}||	j                                        t*                              d|            nT|                                }t1          |          rn0t*                              d           t5          j        d           ||                    |	           t*                              d|j         d|
 d|            |	j                                        du S )z
        Start up a RayClient Server for an incoming client to
        communicate with. Returns whether creation was successful.
        z!Server has not been created for: r   T)uniquez{})r   r   r   r   zspecific-server)stdout_filestderr_filert   server_typer   r]   r^   win32Nz*SpecificServer startup failed for client: z6Waiting for Process to reach the actual client server.r   z SpecificServer started on port: z with PID: z for client: )_get_server_for_clientr   get_log_file_handlesr+   _get_serialized_runtime_env_get_proto_runtime_env_configr   	serializer   r   r`   rt   ri   rj   r@   pidsysplatformpsutilProcessrA   r   r   cmdlinerY   debugr   r   rH   r   )r3   r   r   r   outputr   r   r   r   rD   r   psutil_proccmds                r4   start_specific_serverz"ProxyManager.start_specific_server?  sG   
 55i@@OO OI O OOO	667!577 7 
 
 ",!G!G!I!I'EEGG% 		)?4)G)G .?-@-@-J-J-L-L**-1-E-E'=#5 / .F . .* 'L )+I//
 
 
 l<7"" .--KKK%|  "".U)UUVVV%%''C+C00 LLQRRRJsOOO % 	""4(((7/C 7 77 7+47 7	
 	
 	
 |  ""d**r6   c                     | j         5  | j                            |          }|t                              d|            |cd d d            S # 1 swxY w Y   d S )Nz#Unable to find channel for client: )rg   rf   r   r   r   )r3   r   clients      r4   r   z#ProxyManager._get_server_for_client}  s     	 	\%%i00F~N9NNOOO		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   ;AAAc                 \    |                      |          }|dS |                                S )NF)r   r5   r3   r   r   s      r4   has_channelzProxyManager.has_channel  s/    ,,Y77>5   r6   r-   c                 6   |                      |          }|dS |                                 	 t          j        |j                                      t                     |j        S # t          j        $ r! t          	                    d|            Y dS w xY w)z~
        Find the gRPC Channel for the given client_id. This will block until
        the server process has started.
        Nr9   z Timeout waiting for channel for )
r   r=   grpcchannel_ready_futurer.   r:   CHECK_CHANNEL_TIMEOUT_SFutureTimeoutErrorr   	exceptionr   s      r4   get_channelzProxyManager.get_channel  s     ,,Y77>4	%fn55<</ =    >!& 	 	 	K	KKLLL44	s   8A( (,BBc                    	 | j         5  t          | j                                                  D ]e\  }}|                                Lt
                              d| d|j                    | j        |= | j        	                    |j                   f	 ddd           n# 1 swxY w Y   t          j        t                     )zX
        Keeps the internal servers dictionary up-to-date with running servers.
        TNzSpecific server z( is no longer running, freeing its port )rg   rk   rf   itemsrA   r   r   r+   ro   r   r   r   CHECK_PROCESS_INTERVAL_S)r3   r   r   s      r4   rq   zProxyManager._check_processes  s5   	1! 	F 	F26t|7I7I7K7K2L2L F F.I&++--9Iy I I2A2FI I   !L3(//0DEEEF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F J/000	1s   BB##B'*B'c                 f    | j                                         D ]}|                                 dS )z
        Forcibly kill all spawned RayClient Servers. This ensures cleanup
        for platforms where fate sharing is not supported.
        N)rf   valuesrF   )r3   r   s     r4   rx   zProxyManager._cleanup  s:    
 l))++ 	 	FKKMMMM	 	r6   rJ   )rK   rL   rM   r   r   rN   ry   r   r   r   propertyr`   r   r   r   r   r*   r   r   r   rQ   r   r   r   r  rq   rx   rS   r6   r4   r[   r[   t   s1        &*(,(,&'' ' '#' $''
 c]' !' !' !$' ' ' '8 .4^ L Ls L L L L L& 
 
 
 
 X
 cl',    X&     2_
 #_
  _
 (	_
 _
 _
 _
B<+s <+	 <+d <+ <+ <+ <+| 8P    !S !T ! ! ! ! 
)	*   ,1 1 1$     r6   r[   c                      e Zd ZdedefdZdedeej	                 fdZ
d Zddej        fd	Zddej        fd
Zddej        fdZddej        fdZddej        fdZddej        fdZ	 ddej        fdZ	 ddej        fdZddej        fdZddZddZ 	 ddej!        dej"        fdZ#ddej$        fdZ%ddej&        fdZ'dS )RayletServicerProxyray_connect_handlerproxy_managerc                 "    || _         || _        d S rI   )r  r  )r3   r  r  s      r4   ry   zRayletServicerProxy.__init__  s    *#6   r6   r   r/   c                    t          |          }| j                            |          }|sDt                              d| d           |                    t          j        j                   d S t          j
        |          }	 d|fg}|r|                                } t          ||          ||          S # t          $ r9}t                              d| d           t          ||           Y d }~d S d }~ww xY w)NzChannel for Client: z not found!r   metadatazProxying call to z failed!)r#   r  r  r   r   set_coder   
StatusCode	NOT_FOUNDray_client_pb2_grpcRayletDriverStubinvocation_metadatagetattrr   r  r$   )	r3   r   contextr   r   chanstubr  r   s	            r4   _call_inner_functionz(RayletServicerProxy._call_inner_function  s&    088	!--i88 	LLF	FFFGGGT_67774"3D99	4$i01H 9"6688(74((8DDDD 	4 	4 	4AAAABBB'7333333333	4s   5B; ;
C>.C99C>c                 T    t          |          }| j                            |          S rI   )r#   r  r   )r3   r  r   s      r4   _has_channel_for_requestz,RayletServicerProxy._has_channel_for_request  s&    /88	!--i888r6   Nc                 0    |                      ||d          S )NInitr  r3   r   r  s      r4   r"  zRayletServicerProxy.Init  s    (('6BBBr6   c                 <   |                      |          r|                     ||d          S t                      5  t          j        j                            |j        |j        |j	                  }ddd           n# 1 swxY w Y   t          j        |          S )a=  Proxies internal_kv.put.

        This is used by the working_dir code to upload to the GCS before
        ray.init is called. In that case (if we don't have a server yet)
        we directly make the internal KV call from the proxier.

        Otherwise, we proxy the call to the downstream server as usual.
        KVPut)	overwriteN)already_exists)r   r  r   r   experimentalinternal_kv_internal_kv_putkeyvaluer'  ray_client_pb2KVPutResponse)r3   r   r  r(  s       r4   r&  zRayletServicerProxy.KVPut  s     ((11 	H,,WgwGGG "" 	 	 -9JJW]g6G K  N	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 +>JJJJs   7A>>BBc                 "   |                      |          r|                     ||d          S t                      5  t          j        j                            |j                  }ddd           n# 1 swxY w Y   t          j	        |          S )a=  Proxies internal_kv.get.

        This is used by the working_dir code to upload to the GCS before
        ray.init is called. In that case (if we don't have a server yet)
        we directly make the internal KV call from the proxier.

        Otherwise, we proxy the call to the downstream server as usual.
        KVGetN)r-  )
r   r  r   r   r)  r*  _internal_kv_getr,  r.  KVGetResponse)r3   r   r  r-  s       r4   r1  zRayletServicerProxy.KVGet  s     ((11 	H,,WgwGGG "" 	O 	O$0AA'+NNE	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O+%8888   *A11A58A5c                    |                      |          r|                     ||d          S t                      5  t          j        j                            |j                   ddd           n# 1 swxY w Y   t          j	                    S )a@  Proxies internal_kv.delete.

        This is used by the working_dir code to upload to the GCS before
        ray.init is called. In that case (if we don't have a server yet)
        we directly make the internal KV call from the proxier.

        Otherwise, we proxy the call to the downstream server as usual.
        KVDelN)
r   r  r   r   r)  r*  _internal_kv_delr,  r.  KVDelResponser$  s      r4   r6  zRayletServicerProxy.KVDel  s     ((11 	H,,WgwGGG "" 	G 	G(99'+FFF	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G+---r4  c                 "   |                      |          r|                     ||d          S t                      5  t          j        j                            |j                  }ddd           n# 1 swxY w Y   t          j	        |          S )a>  Proxies internal_kv.list.

        This is used by the working_dir code to upload to the GCS before
        ray.init is called. In that case (if we don't have a server yet)
        we directly make the internal KV call from the proxier.

        Otherwise, we proxy the call to the downstream server as usual.
        KVListN)keys)
r   r  r   r   r)  r*  _internal_kv_listprefixr.  KVListResponse)r3   r   r  r;  s       r4   r:  zRayletServicerProxy.KVList  s     ((11 	I,,WgxHHH "" 	R 	R#/AA'.QQD	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R,$7777r4  c                 "   |                      |          r|                     ||d          S t                      5  t          j        j                            |j                  }ddd           n# 1 swxY w Y   t          j	        |          S )a@  Proxies internal_kv.exists.

        This is used by the working_dir code to upload to the GCS before
        ray.init is called. In that case (if we don't have a server yet)
        we directly make the internal KV call from the proxier.

        Otherwise, we proxy the call to the downstream server as usual.
        KVExistsN)exists)
r   r  r   r   r)  r*  _internal_kv_existsr,  r.  KVExistsResponse)r3   r   r  rA  s       r4   r@  zRayletServicerProxy.KVExists  s     ((11 	K,,WgzJJJ "" 	S 	S%1EEgkRRF	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S.f====r4  c                 ,   |                      |          r|                     ||d          S t                      5  t          j        j                            |j        |j                   ddd           n# 1 swxY w Y   t          j
                    S )aM  Proxies internal_kv.pin_runtime_env_uri.

        This is used by the working_dir code to upload to the GCS before
        ray.init is called. In that case (if we don't have a server yet)
        we directly make the internal KV call from the proxier.

        Otherwise, we proxy the call to the downstream server as usual.
        PinRuntimeEnvURI)expiration_sN)r   r  r   r   r)  r*  _pin_runtime_env_uriurirF  r.  ClientPinRuntimeEnvURIResponser$  s      r4   rE  z$RayletServicerProxy.PinRuntimeEnvURI/  s     ((11 	S,,Wg?QRRR "" 	 	(=='*> >   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 <>>>s   1A88A<?A<c                 0    |                      ||d          S )NListNamedActorsr#  r$  s      r4   rK  z#RayletServicerProxy.ListNamedActorsC  s     ((';LMMMr6   c                     |j         t          j        j        k    r)t          j        t          j        i                     }|S |                     ||d          S )N)jsonClusterInfo)typer.  ClusterInfoTypePINGClusterInfoResponserM  dumpsr  )r3   r   r  resps       r4   rN  zRayletServicerProxy.ClusterInfoH  sO     <>9>>>!54:b>>JJJDK(('=IIIr6   c                 0    |                      ||d          S )N	Terminater#  )r3   r   r  s      r4   rV  zRayletServicerProxy.TerminateQ  s    ((g{CCCr6   c              #      K   	 |                      ||d          E d {V  d S # t          $ r5}t                              d           t	          ||           Y d }~d S d }~ww xY w)N	GetObjectz"Proxying call to GetObject failed!)r  r   r   r  r$   )r3   r   r  r   s       r4   rX  zRayletServicerProxy.GetObjectT  s      	400';OOOOOOOOOOO 	4 	4 	4ABBB'7333333333	4s   # 
A"*AA"r   c                 0    |                      ||d          S )N	PutObjectr#  r$  s      r4   rZ  zRayletServicerProxy.PutObject\  s     ((';GGGr6   c                 0    |                      ||d          S )N
WaitObjectr#  r$  s      r4   r\  zRayletServicerProxy.WaitObjecta  s    (('<HHHr6   c                 0    |                      ||d          S )NScheduler#  )r3   taskr  s      r4   r^  zRayletServicerProxy.Scheduled  s    ((w
CCCr6   rI   )(rK   rL   rM   r
   r[   ry   r   r   r  r  r  r   r.  InitResponser"  r/  r&  r3  r1  r8  r6  r>  r:  rC  r@  rI  rE  ClientListNamedActorsResponserK  rR  rN  rV  rX  
PutRequestPutResponserZ  WaitResponser\  ClientTaskTicketr^  rS   r6   r4   r  r    s       7H 7\ 7 7 7 74(+4	%6	74 4 4 4*9 9 9C C^-H C C C CK Kn.J K K K K$9 9n.J 9 9 9 9 . .n.J . . . . 8 8~/L 8 8 8 8 > >1P > > > >"  $? ?		6? ? ? ?*  $N N		5N N N N
J JN4V J J J JD D D D4 4 4 4 ;?H H%0H		#H H H H
I I>3N I I I ID Dn.M D D D D D Dr6   r  r   c                     | S rI   rS   )r   s    r4   ray_client_server_env_preprg  h  s    r6   init_requestc                    |                      d          }|dk    sJ d| d            | j        }t                      }|j        rt	          j        |j                  }t          |          }t          j        t	          j	        |          | j        j
        | j        j                  }| j                            |           | |fS )zu
    Extract JobConfig and possibly mutate InitRequest before it is passed to
    the specific RayClient Server.
    rO  r   z!Received initial message of type z, not 'init'.)r   ray_init_kwargsreconnect_grace_period)
WhichOneofr   r   r   r   loadsrg  r.  InitRequestrS  rj  rk  CopyFrom)rh  	init_typer   r   new_job_configmodified_init_reqs         r4   prepare_runtime_init_reqrs  l  s     ''//IGyGGG  
CJ
~ 2\#.11
/
;;N&2<//$)9+0G   0111.))r6   c                        e Zd Zd Zd Zd ZdS )RequestIteratorProxyc                     || _         d S rI   )request_iterator)r3   rw  s     r4   ry   zRequestIteratorProxy.__init__  s     0r6   c                     | S rI   rS   r2   s    r4   __iter__zRequestIteratorProxy.__iter__  s    r6   c                     	 t          | j                  S # t          j        $ rC}t	          |          t          j        ur|t
                              d           t          d }~ww xY w)NzEStop iterating cancelled request stream with the following exception:)nextrw  r   RpcErrorrO  r   r  StopIteration)r3   r   s     r4   __next__zRequestIteratorProxy.__next__  sv    	 -...} 	  	  	  Awwdm++W    	 s    A(>A##A(N)rK   rL   rM   ry   ry  r~  rS   r6   r4   ru  ru    sA        1 1 1           r6   ru  c                   D    e Zd ZdefdZdej        dej        fdZd ZdS )DataServicerProxyr  c                     d| _         i | _        i | _        t                      | _        || _        t                      | _        d S )Nr   )num_clientsclients_last_seenreconnect_grace_periodsr   clients_lockr  r   stopped)r3   r  s     r4   ry   zDataServicerProxy.__init__  s=    359;$ FF*wwr6   	init_respr/   c                     |                     d          }|dk    r|S t          j                    }|                    |           | j        5  | j        |j        _        ddd           n# 1 swxY w Y   |S )z
        Modify the `num_clients` returned the ConnectionInfoResponse because
        individual SpecificServers only have **one** client.
        rO  connection_infoN)rl  r.  DataResponsero  r  r  r  )r3   r  rp  modified_resps       r4   modify_connection_info_respz-DataServicerProxy.modify_connection_info_resp  s     ((00	)))&355y))) 	I 	I8<8HM)5	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	Is   A++A/2A/c              #     K   t          |          }d}t          j                    }t          |          }|dk    rd S t          |          }|r| j        5  || j        vrG|                    t          j        j	                   |
                    d           	 d d d            d S || j        |<   d d d            n# 1 swxY w Y   | j                            |          }| j                            |          }|}	nS| j                            |          }| j        5  || j        |<   | xj        dz  c_        d d d            n# 1 swxY w Y   	 |st                               d| d           t%          |          }
| j        5  |
j        j        | j        |<   d d d            n# 1 swxY w Y   	 t-          |
          \  }}| j                            ||          s9t                               d| d| d	           t3          d
|j         d          | j                            |          }|5t                               d|            t3          d|j         d          n# t6          $ r t9          j        t9          j        dt?          j                                         }|
j!        |_!        |V  Y | j        "                    |          }|s|| j#        $                    |           | j        5  || j        vr+t                               | d           	 d d d            d S | j        |         }t                               | d| d| d           ||k    r(t                               d           	 d d d            d S t           %                    d|            | xj        dz  c_        | j        |= || j        v r| j        |= |&                    d            d d d            d S # 1 swxY w Y   d S w xY wtO          |g|          }	tQ          j)        |          }d|fdtU          |          fg}|+                    |	|          }|D ]6}|,                    d          }|dk    rd}| -                    |          V  7nE# t6          $ r8}t           .                    d           t_          ||          }|sd}Y d }~nd }~ww xY w| j        "                    |          }|s|| j#        $                    |           | j        5  || j        vr+t                               | d           	 d d d            d S | j        |         }t                               | d| d| d           ||k    r(t                               d           	 d d d            d S t           %                    d|            | xj        dz  c_        | j        |= || j        v r| j        |= |&                    d            d d d            d S # 1 swxY w Y   d S # | j        "                    |          }|s|| j#        $                    |           | j        5  || j        vr-t                               | d           	 d d d             Y d S | j        |         }t                               | d| d| d           ||k    r*t                               d           	 d d d             Y d S t           %                    d|            | xj        dz  c_        | j        |= || j        v r| j        |= |&                    d            d d d            w # 1 swxY w Y   w xY w)NFr|   zAAttempted to reconnect a session that has already been cleaned upr   z New data connection from client r   z"Server startup failed for client: z, using JobConfig: !z9Starting Ray client server failed. See ray_client_server_z.err for detailed logs.zChannel not found for zYProxy failed to Connect to backend! Check `ray_client_server.err` and `ray_client_server_zz.err` on the head node of the cluster for the relevant logs. By default these are located at /tmp/ray/session_latest/logs.)okmsg)r   r9   z not found. Skipping clean up.z last started stream at z. Current stream started at r   z%Client reconnected. Skipping cleanup.zClient detached: r   reconnectingr  rO  connection_cleanupTzProxying Datapath failed!)0ru  r   r#   r%   r  r  r  r   r  r  set_detailsr  r   r  r   r  r   r   r{  r   rk  r  rs  r   r   r;   r+   r   r.  r  r`  	traceback
format_excreq_idr   r  waitr   rH   r   r  RayletDataStreamerStubr   Datapathrl  r  r  r$   )r3   rw  r  cleanup_requested
start_timer   r  r   r.   new_iterinit_reqrr  r   r  cleanup_delay	last_seenr  r  resp_streamrT  	resp_typer   recoverables                          r4   r  zDataServicerProxy.Datapath  sM     /0@AA!Y[[
/88	??F5g>> 	&" 
? 
?D$::: $$T_%>???''*   
? 
? 
? 
? 
? 
? 
? 
? 5?&y1
? 
? 
? 
? 
? 
? 
? 
? 
? 
? 
? 
? 
? 
? 
? '>>yIIF(44Y??G'HH '>>yIIF" & &4>&y1  A%  & & & & & & & & & & & & & & &W	( *HLyLLLMMM 011& = = !< 0!= = = = = = = = = = = = = = =! 4LX4V4V1%z-CC!:   > > >0:> > >   +-17- - -  
 #0<<YGGG%Ii%I%IJJJ*<28+< < <   ' !       . ;+8$)*>*@*@  ! ! !I
 (0I$#OOO* !8<<YGGM$ 9)B !!-!888" ( (D$:::KK9 L L LMMM	( ( ( ( ( ( ( (
 !29=	  7 7) 7 7)37 7 7   z))KK GHHH( ( ( ( ( ( ( ( <<<===  A%  *95 <<<4Y?!!$''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (G  !"3!46FGG&=gFFD$i0>3|CTCT2UVH--8-DDK# = = OOF33	 444(,%66t<<<<<<=  	) 	) 	)89995aAAK )$(!	) !8<<YGGM$ 9)B !!-!888" ( (D$:::KK9 L L LMMM	( ( ( ( ( ( ( (
 !29=	  7 7) 7 7)37 7 7   z))KK GHHH( ( ( ( ( ( ( ( <<<===  A%  *95 <<<4Y?!!$''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( !8<<YGGM$ 9)B !!-!888" ( (D$:::KK9 L L LMMM	( ( ( ( ( ( ( ( ( (
 !29=	  7 7) 7 7)37 7 7   z))KK GHHH( ( ( ( ( ( ( ( ( ( <<<===  A%  *95 <<<4Y?!!$''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s%  AB='
B==CC EEE7R 
F+R +F//R 2F/3R 7B7I0 .R 0APR (P;APAPPPPBR X* 
S'.SX* SX* "(XAX5AXX!$X!*A]5+(])]5"A])3]5A])]5)]--]50]-1]5N)	rK   rL   rM   r[   ry   r.  r  r  r  rS   r6   r4   r  r    sm        l    '4		$    w( w( w( w( w(r6   r  c                   *     e Zd Zdef fdZd Z xZS )LogstreamServicerProxyr  c                 V    t                                                       || _        d S rI   )superry   r  )r3   r  	__class__s     r4   ry   zLogstreamServicerProxy.__init__3  s'    *r6   c              #     K   t          |          }t          |          }|dk    rd S t                              d| d           d }t	          t
                    D ]Z}| j                            |          }| n;t                              d|dz    d           t          j
        t                     [|?|                    t          j        j                   |                    d| d           d S t#          j        |          }|                    |d	|fg
          }	 |D ]}|V  d S # t(          $ r t                              d           Y d S w xY w)Nr|   z%New logstream connection from client r   zRetrying Logstream connection. r   z attempts failed.z6Logstream proxy failed to connect. Channel for client z not found.r   r  zProxying Logstream failed!)ru  r#   r   r   rl   LOGSTREAM_RETRIESr  r  r   r   r   LOGSTREAM_RETRY_INTERVAL_SECr  r   r  r  r  r  RayletLogStreamerStub	Logstreamr   r  )	r3   rw  r  r   r.   ir  r  rT  s	            r4   r  z LogstreamServicerProxy.Logstream7  s     /0@AA/88	??FJYJJJKKK ()) 	5 	5A(44Y??G"NNSQqSSSSTTTJ34444?T_6777** * *   4"8AAnni(@'A % 
 
	;#  



  	; 	; 	;9::::::	;s   (	D3 3$EE)rK   rL   rM   r[   ry   r  __classcell__)r  s   @r4   r  r  2  sS        +l + + + + + +#; #; #; #; #; #; #;r6   r  )r]   r^   r\   ra   r   r+   r   r]   r^   r\   ra   c                j   |4t          |          }t          j        j                            |           ddlm}  |t          dt          d          }	t          |||||          }
t          d |
          }t          |
          }t          |
          }t          j        ||	           t          j        ||	           t          j        ||	           t#          |           st%          |	d|            t%          |	|  d	|            |	                                 t)          ||||	
          S )N)r`   r   )$create_grpc_server_with_interceptorsray_client_proxierF)max_workersthread_name_prefixr   asynchronous)r\   r]   r^   ra   z
127.0.0.1::)task_servicerdata_servicerlogs_servicergrpc_server)r   r   r)  r*  _initialize_internal_kvray._private.grpc_utilsr  r    r!   r[   r  r  r  r  "add_RayletDriverServicer_to_server(add_RayletDataStreamerServicer_to_server'add_RayletLogStreamerServicer_to_serverr   r   rs   r"   )r   r+   r   r]   r^   r\   ra   gcs_clir  r   r  r  r  r  s                 r4   serve_proxierr  ]  sj    K000$<<WEEELLLLLL11-/	  F !%%";  M (m<<M%m44M*=99M:=&QQQ@PVWWW?vVVV =(;T(;(;<<<Ft$4$4d$4$4555
LLNNN###	   r6   )drv   rM  loggingr   r   r   r  r   
concurrentr   dataclassesr   	itertoolsr   	threadingr   r   r   r	   typingr
   r   r   r   r   r   r   !ray.core.generated.ray_client_pb2core	generatedr.  &ray.core.generated.ray_client_pb2_grpcr  (ray.core.generated.runtime_env_agent_pb2r   ray._common.network_utilsr   r   r   5ray._private.authentication.http_token_authenticationr   r   ray._private.client_mode_hookr   r  r   ray._private.parameterr    ray._private.runtime_env.contextr   ray._private.servicesr   r   ray._private.tls_utilsr   ray._private.utilsr   ray._rayletr   ray.cloudpickle.compatr   ray.exceptionsr   ray.job_configr   ray.util.client.commonr    r!   r"   r#   r$   #ray.util.client.server.dataservicerr%   r   	getLoggerrK   r   r  rm   rn   r  r  r  r*   r   rQ   rY   r[   RayletDriverServicerr  rg  DataRequestrs  ru  RayletDataStreamerServicerr  RayletLogStreamerServicerr  rN   r  rS   r6   r4   <module>r     s=       



             ! ! ! ! ! !       0 0 0 0 0 0 0 0 0 0 0 0 8 8 8 8 8 8 8 8 8 8 8 8 8 8  



 : : : : : : : : : : : : D D D D D D D D D D D D H H H H H H H H H H H H J J J J J J J J J J        > = = = = = 5 5 5 5 5 5 , , , , , , > > > > > > F F F F F F F F : : : : : : : : : : : : ! ! ! ! ! ! ) ) ) ) ) ) . . . . . . $ $ $ $ $ $              O N N N N N 		8	$	$          (8 (8 (8 (8 (8 (8 (8 (8V4$s) 4 4 4 4 4E E E E E E E EP
iD iD iD iD iD-B iD iD iDX9     * ,*
>%y01* * * *4               0Q( Q( Q( Q( Q(+F Q( Q( Q(h(; (; (; (; (;0J (; (; (;` %)$(!%/30 0 0
0
0 #0
 SM0 SM0 #0  (}0 0 0 0 0 0r6   