
    &`i4             2          U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+ erd dl,Z,d dl-Z-d dl.Z.d dl/Z.d dl0Z.d dl1m2c m3Z3 d dl4m2c m5Z5 d dl6m2c m7Z7 d dl8m2c m9Z9 d dl:Z.d dl;Z.d dl<Z.d dl=m>Z? d dl@Z.d dlAZ.d d	l.mBZBmCZCmDZDmEZE d d
lFmGZG d dlHmIZI d dlJmKZK d dlLmMZM d dlNmOZO d dlPmQZQ d dlRmSZS d dlTmUZU d dlVmWZWmXZXmYZYmZZZ d dl[m\Z\ d dl]m^Z^ d dl_m`Z` d dlambZb d dlcmdZd d dlemfZf d dlgmhZh d dlimjZjmkZkmlZl d dl<mmZm d dlnmoZompZpmqZqmrZr d dlsmtZt d dlumvZv d dlwmxZxmyZymzZzm{Z{ d d l|m}Z} d d!l~mZ d d"lmZmZmZ d d#lmZ d d$lmZ d d%lmZ d d&lmZ d d'lmZ d Zd(Zd)Zd*Zd+Z ej        e          Z e'd,          Z e'd-          Z e'd.          Z e'd/          Z e'd0          Z e'd1          Z e'd2          Z e'd3          Z e'd4          Z e'd5          Z e'd6          Z e'd7          Z e'd8          Z e            Zeed9<    e'd:d;<          Z G d= d;e#          Z G d> d?eee                   Z G d@ dAeeeef                   Z G dB dCeeeeef                   Z G dD dEeeeeeef                   Z G dF dGeeeeeeef                   Z G dH dIeeeeeeeef                   Z G dJ dKeeeeeeeeef                   Z G dL dMeeeeeeeeeef                   Z G dN dOeeeeeeeeeeef	                   Z G dP dQeeeeeeeeeeeef
                   Z G dR dSeeeeeeeeeeeeef                   ZdTefdUZ G dV dW          Z e	j                    ZdXedYefdZZeeOdYe(e e         e e         f         fd[                        Z ed\d]^          d_             Z ed`a          db             Zdce"e         dYefddZ G de dfeg          Ze G dh diee                      Z e            Z	 da	 dje"eeef                  dkedYeeef         fdlZeeO	 dddddddmdmde5j        ddd]e5j        ddddddmdddndoe"e         dpe"e         dqe"e         dre"eeef                  dse"eeef                  dte"e         duedvedwe"e         dxedye"e         dzd{d|ed}ed~e"e         de"e\         de"e         de"e         dje"e(eeef         df                  dede"e         de"e         dYef.d                        Zg ZeeOeddefd                                    Z e j        ed]           ej        Zd Zee_        defdZ e
j
                    Zdmade e         dYee         fdZdYefdZd]Z	 ddeeef         dedefdZd Zd Zd Zd ZeeOdYefd                        Zeedmeddddd d dmddddfdedede"e         de"e         dededededededefd            ZddZed             Zeddededefd            Zdmae)ddddde"e         dYe e         fd            Ze)ddddde"e         dYe e         fd            Ze)ddddde"e         dYefd            Ze)ddde$ev         de"e         dYe e         fd            Ze)dddevde"e         dYefd            ZeeOdddde(de$d         eve$ev         f         de"e         de"e         dYe(ee e         f         fd                        ZeeOddddede"d         dedYdfd                        ZdmaeeOd(dd]dde e(eEejf                  dede"e         dedYe%e e(eEejf                  e e(eEejf                  f         f
dĄ                        ZeeOddede"e         dYdfdƄ                        ZeeOd]dǜdddefdʄ                        ZeeOdmd]d˜de(d         dededYdfdЄ                        ZefdфZd҄ Z G dӄ de#          Ze)de&e         dYeme         fdք            Ze)deg ef         dYee         fd؄            Ze)deegef         dYeeef         fdل            Ze)deeegef         dYeeeef         fdڄ            Ze)deeeegef         dYeeeeef         fdۄ            Ze)deeeeegef         dYeeeeeef         fd܄            Ze)deeeeeegef         dYeeeeeeef         fd݄            Ze)deeeeeeegef         dYeeeeeeeef         fdބ            Ze)deeeeeeeegef         dYeeeeeeeeef         fd߄            Ze)deeeeeeeeegef         dYeeeeeeeeeef	         fd            Ze)deeeeeeeeeeg	ef         dYeeeeeeeeeeef
         fd            Ze)deeeeeeeeeeeg
ef         dYeeeeeeeeeeeef         fd            Ze)eeeeeeeeeeeeeeedde(ee!d         f         dpe(eef         dqe(eef         dreeef         dede(eef         dededededjeeef         dede(de!d         e!d         ef         deeef         de eeef                  dYef d            ZedYe(e.j        j        e.j        jm        f         fd            ZdS )    N)ABCMetaabstractmethod)Mapping)contextmanager)	dataclasswraps)TYPE_CHECKINGAnyAnyStrCallableDictGenericIteratorListLiteralOptionalProtocolSequenceTupleTypeTypeVarUnionoverload)urlparse)ActorIDJobIDLanguage	ObjectRef)ray_option_utils)*RAY_WARN_BLOCKING_GET_INSIDE_ASYNC_ENV_VAR)
load_class)ensure_token_if_auth_enabled)client_mode_hook)TensorTransportEnum)FunctionActorManager)	is_cython)"global_worker_stdstream_dispatchersetup_loggerstderr_deduplicatorstdout_deduplicator)LoggingConfig)ResourceIsolationConfig)RAY_JOB_CONFIG_JSON_ENV_VAR)upload_py_modules_if_needed)*upload_worker_process_setup_hook_if_needed)upload_working_dir_if_needed)get_ray_doc_version)ObjectRefGeneratorTaskID(raise_sys_exit_with_custom_error_message)
ActorClass)ObjectStoreFullErrorRayErrorRaySystemErrorRayTaskError)tqdm_ray)CompiledDAGRef)_initialize_internal_kv_internal_kv_get_internal_kv_initialized_internal_kv_reset)RAY_TQDM_MAGIC)_merge_runtime_env)
DeprecatedDeveloperAPI	PublicAPI)log_once) PlacementGroupSchedulingStrategy)_import_from_string)Template)repr_with_fallback            TT0T1T2T3T4T5T6T7T8T9RDAGNode	UndefinedRF
HasOptions)boundc                       e Zd ZdedefdZdS )r^   selfreturnc                     d S N )ra   task_optionss     g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/worker.pyoptionszHasOptions.options   s        N)__name__
__module____qualname__r]   rh   re   ri   rg   r^   r^      s6        b R      ri   c                   F    e Zd Zdeg ef         ddfdZ	 	 d	dZ	 	 d
dZdS )RemoteFunctionNoArgsfunctionrb   Nc                     d S rd   re   ra   ro   s     rg   __init__zRemoteFunctionNoArgs.__init__       ri   ObjectRef[R]c                     d S rd   re   ra   s    rg   remotezRemoteFunctionNoArgs.remote   	     	ri   
DAGNode[R]c                     d S rd   re   rv   s    rg   bindzRemoteFunctionNoArgs.bind   rx   ri   )rb   rt   )rb   ry   )rj   rk   rl   r   rZ   rr   rw   r{   re   ri   rg   rn   rn      sv        "a% T    	   
	     ri   rn   c                   P    e Zd Zdeegef         ddfdZ	 	 	 	 ddZ	 	 	 	 ddZdS )RemoteFunction0ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction0.__init__   rs   ri   _RemoteFunction0__arg0Union[T0, ObjectRef[T0]]rt   c                     d S rd   re   ra   r   s     rg   rw   zRemoteFunction0.remote   	     	ri   Union[T0, DAGNode[T0]]ry   c                     d S rd   re   r   s     rg   r{   zRemoteFunction0.bind   r   ri   )r   r   rb   rt   )r   r   rb   ry   )	rj   rk   rl   r   rP   rZ   rr   rw   r{   re   ri   rg   r}   r}      s        2$'!2 t    * 
   ( 
     ri   r}   c                   Z    e Zd Zdeeegef         ddfdZ	 	 	 	 	 	 dd
Z	 	 	 	 	 	 ddZ	dS )RemoteFunction1ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction1.__init__   rs   ri   _RemoteFunction1__arg0r   _RemoteFunction1__arg1Union[T1, ObjectRef[T1]]rt   c                     d S rd   re   ra   r   r   s      rg   rw   zRemoteFunction1.remote   	    
 	ri   r   Union[T1, DAGNode[T1]]ry   c                     d S rd   re   r   s      rg   r{   zRemoteFunction1.bind   r   ri   )r   r   r   r   rb   rt   )r   r   r   r   rb   ry   )
rj   rk   rl   r   rP   rQ   rZ   rr   rw   r{   re   ri   rg   r   r      s        2r(A+!6 4    * + 
	   ( ) 
	     ri   r   c                   d    e Zd Zdeeeegef         ddfdZ	 	 	 	 	 	 	 	 ddZ		 	 	 	 	 	 	 	 ddZ
dS )RemoteFunction2ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction2.__init__   rs   ri   _RemoteFunction2__arg0r   _RemoteFunction2__arg1r   _RemoteFunction2__arg2Union[T2, ObjectRef[T2]]rt   c                     d S rd   re   ra   r   r   r   s       rg   rw   zRemoteFunction2.remote   	     	ri   r   r   Union[T2, DAGNode[T2]]ry   c                     d S rd   re   r   s       rg   r{   zRemoteFunction2.bind   r   ri   )r   r   r   r   r   r   rb   rt   )r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rZ   rr   rw   r{   re   ri   rg   r   r      s        2r2,/!: t    * + +	
 
   ( ) )	
 
     ri   r   c                   n    e Zd Zdeeeeegef         ddfdZ		 	 	 	 	 	 	 	 	 	 ddZ
	 	 	 	 	 	 	 	 	 	 ddZdS )RemoteFunction3ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction3.__init__   rs   ri   _RemoteFunction3__arg0r   _RemoteFunction3__arg1r   _RemoteFunction3__arg2r   _RemoteFunction3__arg3Union[T3, ObjectRef[T3]]rt   c                     d S rd   re   ra   r   r   r   r   s        rg   rw   zRemoteFunction3.remote   	     	ri   r   r   r   Union[T3, DAGNode[T3]]ry   c                     d S rd   re   r   s        rg   r{   zRemoteFunction3.bind   r   ri   )
r   r   r   r   r   r   r   r   rb   rt   )
r   r   r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rZ   rr   rw   r{   re   ri   rg   r   r      s        2r2r*:A*=!> 4    * + +	
 + 
   ( ) )	
 ) 
     ri   r   c                   x    e Zd Zdeeeeeege	f         ddfdZ
	 	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 	 	 ddZdS )RemoteFunction4ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction4.__init__   rs   ri   _RemoteFunction4__arg0r   _RemoteFunction4__arg1r   _RemoteFunction4__arg2r   _RemoteFunction4__arg3r   _RemoteFunction4__arg4Union[T4, ObjectRef[T4]]rt   c                     d S rd   re   ra   r   r   r   r   r   s         rg   rw   zRemoteFunction4.remote   	     	ri   r   r   r   r   Union[T4, DAGNode[T4]]ry   c                     d S rd   re   r   s         rg   r{   zRemoteFunction4.bind  r   ri   )r   r   r   r   r   r   r   r   r   r   rb   rt   )r   r   r   r   r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rT   rZ   rr   rw   r{   re   ri   rg   r   r      s        2r2r2*>*A!B t    * + +	
 + + 
   ( ) )	
 ) ) 
     ri   r   c                       e Zd Zdeeeeeee	ge
f         ddfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZdS )RemoteFunction5ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction5.__init__  rs   ri   _RemoteFunction5__arg0r   _RemoteFunction5__arg1r   _RemoteFunction5__arg2r   _RemoteFunction5__arg3r   _RemoteFunction5__arg4r   _RemoteFunction5__arg5Union[T5, ObjectRef[T5]]rt   c                     d S rd   re   ra   r   r   r   r   r   r   s          rg   rw   zRemoteFunction5.remote  	     	ri   r   r   r   r   r   Union[T5, DAGNode[T5]]ry   c                     d S rd   re   r   s          rg   r{   zRemoteFunction5.bind  r   ri   )r   r   r   r   r   r   r   r   r   r   r   r   rb   rt   )r   r   r   r   r   r   r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rT   rU   rZ   rr   rw   r{   re   ri   rg   r   r     s        2r2r2r*BA*E!F 4    	*	 +	 +		
 +	 +	 +	 
	 	 	 		(	 )	 )		
 )	 )	 )	 
	 	 	 	 	 	ri   r   c            	           e Zd Zdeeeeeee	e
gef         ddfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZdS ) RemoteFunction6ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction6.__init__,  rs   ri   _RemoteFunction6__arg0r   _RemoteFunction6__arg1r   _RemoteFunction6__arg2r   _RemoteFunction6__arg3r   _RemoteFunction6__arg4r   _RemoteFunction6__arg5r   _RemoteFunction6__arg6Union[T6, ObjectRef[T6]]rt   c                     d S rd   re   ra   r   r   r   r   r   r   r   s           rg   rw   zRemoteFunction6.remote/  	     	ri   r   r   r   r   r   r   Union[T6, DAGNode[T6]]ry   c                     d S rd   re   r   s           rg   r{   zRemoteFunction6.bind;  r   ri   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   rb   rt   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rT   rU   rV   rZ   rr   rw   r{   re   ri   rg   r   r   +  s       2r2r2r2*F*I!J t    
*
 +
 +	

 +
 +
 +
 +
 

 
 
 

(
 )
 )	

 )
 )
 )
 )
 

 
 
 
 
 
ri   r   c            
           e Zd Zdeeeeeee	e
egef         ddfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!dZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"d ZdS )#RemoteFunction7ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction7.__init__I  rs   ri   _RemoteFunction7__arg0r   _RemoteFunction7__arg1r   _RemoteFunction7__arg2r   _RemoteFunction7__arg3r   _RemoteFunction7__arg4r   _RemoteFunction7__arg5r   _RemoteFunction7__arg6r   _RemoteFunction7__arg7Union[T7, ObjectRef[T7]]rt   c	                     d S rd   re   	ra   r   r   r   r   r   r   r   r   s	            rg   rw   zRemoteFunction7.remoteL  	     	ri   r   r   r   r   r   r   r   Union[T7, DAGNode[T7]]ry   c	                     d S rd   re   r   s	            rg   r{   zRemoteFunction7.bindY  r   ri   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rb   rt   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rT   rU   rV   rW   rZ   rr   rw   r{   re   ri   rg   r   r   H  s*       2r2r2r2r*JA*M!N SW    * + +	
 + + + + + 
   ( ) )	
 ) ) ) ) ) 
     ri   r   c                       e Zd Zdeeeeeee	e
eeg	ef         ddfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d$dZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d%d#ZdS )&RemoteFunction8ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction8.__init__h  	     	ri   _RemoteFunction8__arg0r   _RemoteFunction8__arg1r   _RemoteFunction8__arg2r   _RemoteFunction8__arg3r   _RemoteFunction8__arg4r   _RemoteFunction8__arg5r   _RemoteFunction8__arg6r   _RemoteFunction8__arg7r   _RemoteFunction8__arg8Union[T8, ObjectRef[T8]]rt   c
                     d S rd   re   
ra   r   r   r   r   r   r   r   r   r   s
             rg   rw   zRemoteFunction8.remotem  	     	ri   r   r   r   r   r   r   r   r   Union[T8, DAGNode[T8]]ry   c
                     d S rd   re   r   s
             rg   r{   zRemoteFunction8.bind{  r   ri   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rb   rt   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rZ   rr   rw   r{   re   ri   rg   r   r   g  sA        "b"b"b"b"!Eq!HI	   
* + +	
 + + + + + + 
   ( ) )	
 ) ) ) ) ) ) 
     ri   r   c                       e Zd Zdeeeeeee	e
eeeg
ef         ddfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d'dZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d(d&ZdS ))RemoteFunction9ro   rb   Nc                     d S rd   re   rq   s     rg   rr   zRemoteFunction9.__init__  r   ri   _RemoteFunction9__arg0r   _RemoteFunction9__arg1r   _RemoteFunction9__arg2r   _RemoteFunction9__arg3r   _RemoteFunction9__arg4r   _RemoteFunction9__arg5r   _RemoteFunction9__arg6r   _RemoteFunction9__arg7r   _RemoteFunction9__arg8r   _RemoteFunction9__arg9Union[T9, ObjectRef[T9]]rt   c                     d S rd   re   ra   r   r   r   r  r  r  r  r  r  r  s              rg   rw   zRemoteFunction9.remote  	     	ri   r   r   r   r   r   r   r   r   r   Union[T9, DAGNode[T9]]ry   c                     d S rd   re   r
  s              rg   r{   zRemoteFunction9.bind  r  ri   )r   r   r   r   r   r   r  r   r  r   r  r   r  r   r  r   r  r   r  r  rb   rt   )r   r   r   r   r   r   r  r   r  r   r  r   r  r   r  r   r  r   r  r  rb   ry   )rj   rk   rl   r   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   rr   rw   r{   re   ri   rg   r   r     sW        "b"b"b"b"b!I1!LM	   
* + +	
 + + + + + + + 
   ( ) )	
 ) ) ) ) ) ) ) 
     ri   r   ec                 @    t                               d|             d S )NzAUnhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): )loggererror)r  s    rg   _unhandled_error_handlerr    s-    
LLOAOO    ri   c                      e Zd ZdZd ZedEd            Zed             ZdefdZ	ed	             Z
ed
             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zed             Zd Zd Zed             Z ed             Z!d edd!fd"Z"e#e$         fdFd#Z%e#e$         fdFd$Z&d%e'd&e(fd'Z)d%e'd&e(fd(Z*de+fd)Z,de+fd*Z-de(fd+Z.de(fd,Z/d- Z0d. Z1d/ Z2d0 Z3d!d1e4j5        j6        fd2e7d3e#e+         d4ed5e+fd6Z8d7 Z9	 dGd8e#e+         fd9Z:	 	 	 	 dHd:e;d;e#e<         d<ed=ed5e#e+         de=e>e?j@                 eAf         fd>ZBd? ZCd@ ZDdAe+dBe+deEe>e+         e>e(         f         fdCZFdD ZGd!S )IWorkera  A class used to define the control flow of a worker process.

    Note:
        The methods in this class are considered unexposed to the user. The
        functions outside of this class are considered exposed.

    Attributes:
        node (ray._private.node.Node): The node this worker is attached to.
        mode: The mode of the worker. One of SCRIPT_MODE, LOCAL_MODE, and
            WORKER_MODE.
    c                    d| _         d| _        i | _        d| _        t          j        j                                        | _        i | _	        t          |           | _        t          j                    | _        d| _        d| _        d| _        d| _        t&          j        | _        d| _        d| _        d| _        t          j                    | _        d| _        d| _        d| _        d| _        dS )zInitialize a Worker object.Nri   FT)nodemodeactors_gpu_object_managerray_privateutilsget_visible_accelerator_ids original_visible_accelerator_idsserialization_context_mapr&   function_actor_manager	threadingEventthreads_stoppeddebugger_breakpointdebugger_get_breakpointray_debugger_external_load_code_from_localray_constants$RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING_enable_record_actor_task_log_file_rotation_enabled_out_filepath_err_filepathRLocklock_filter_logs_by_job_debugger_port_cached_job_id_is_connectedrv   s    rg   rr   zWorker.__init__  s    		 $( 
 L::<< 	-
 *,&&:4&@&@#  )00 $'  (+$ &+"%*" > 	* ',#!! O%%	
 $( " # $)ri   rb   !ray.experimental.GPUObjectManagerc                 H    | j         ddlm}  |            | _         | j         S )Nr   )GPUObjectManager)r  ray.experimentalr6  )ra   r6  s     rg   gpu_object_managerzWorker.gpu_object_manager  s9    #+ :99999'7'7'9'9D$''ri   c                     | j         S )z7bool: True if Ray has been started and False otherwise.r3  rv   s    rg   	connectedzWorker.connected
  s     !!ri   is_connectedc                     || _         d S rd   r:  )ra   r<  s     rg   set_is_connectedzWorker.set_is_connected  s    )ri   c                 B    |                                   | j        j        S rd   )check_connectedr  node_ip_addressrv   s    rg   rA  zWorker.node_ip_address  s    y((ri   c                 8    |                                   | j        S rd   )r@  r'  rv   s    rg   load_code_from_localzWorker.load_code_from_local  s    ))ri   c                     | j         | j         S t          | d          r| j                                        S t	          j                    S Ncore_worker)r2  hasattrrF  get_current_job_idr   nilrv   s    rg   current_job_idzWorker.current_job_id  sG    *&&T=)) 	9#66888y{{ri   c                 z    t          | d          r| j                                        S t          j                    S rE  )rG  rF  get_actor_idr   rI  rv   s    rg   actor_idzWorker.actor_id$  s5    4'' 	3#00222{}}ri   c                 ~    t          | d          r,| j                                                            d          S d S )NrF  utf-8)rG  rF  get_actor_namedecoderv   s    rg   
actor_namezWorker.actor_name*  s=    4'' 	E#2244;;GDDDtri   c                 4    | j                                         S rd   )rF  get_current_task_idrv   s    rg   current_task_idzWorker.current_task_id0      33555ri   c                 4    | j                                         S rd   )rF  get_current_task_namerv   s    rg   current_task_namezWorker.current_task_name4  s    55777ri   c                 4    | j                                         S rd   )rF  get_current_task_function_namerv   s    rg   current_task_function_namez!Worker.current_task_function_name8  s    >>@@@ri   c                 4    | j                                         S rd   )rF  get_current_node_idrv   s    rg   current_node_idzWorker.current_node_id<  rV  ri   c                 4    | j                                         S rd   )rF  get_task_depthrv   s    rg   
task_depthzWorker.task_depth@  s    ..000ri   c                 >    | j                                         j        S rd   )rF  get_job_configray_namespacerv   s    rg   	namespacezWorker.namespaceD  s    ..00>>ri   c                 4    | j                                         S rd   )rF  get_placement_group_idrv   s    rg   placement_group_idzWorker.placement_group_idH  s    66888ri   c                 X    | j                                                                         S rd   )rF  get_worker_idbinaryrv   s    rg   	worker_idzWorker.worker_idL  s#    --//66888ri   c                 4    | j                                         S rd   )rF  -should_capture_child_tasks_in_placement_grouprv   s    rg   ro  z4Worker.should_capture_child_tasks_in_placement_groupP  s    MMOOOri   c                     t          | j        j        t          j                  sJ t          | j        t          j                  sJ | j        j        | j        fS )z1Get the current session index and job id as pair.)
isinstancer  
cluster_idr  	ClusterIDrJ  r   rv   s    rg   current_cluster_and_jobzWorker.current_cluster_and_jobT  sO     $).>>>>>$-sy99999y#T%888ri   c                 4    | j                                         S )z"Get the runtime env in json format)rF  get_current_runtime_envrv   s    rg   runtime_envzWorker.runtime_env[  s     77999ri   c                 |    | j                                         }t          j        j                            |          S )z%Get the debugger port for this worker)rF  rk  r  r  stateget_worker_debugger_portra   rm  s     rg   debugger_portzWorker.debugger_port`  s1     $2244	|!::9EEEri   c                     t          | d          sdS | j                                        }|j        sdS t	          j        |j                  }|S )z,Get the job's logging config for this workerrF  N)rG  rF  rd  serialized_py_logging_configpickleloads)ra   
job_configlogging_configs      rg   job_logging_configzWorker.job_logging_configf  sW     t]++ 	4%4466
6 	4j&MNNri   c                     | j         j        S rd   )r  node_labelsrv   s    rg   current_node_labelszWorker.current_node_labelsq  s     y$$ri   c                     | j                                         }t          j        j                            ||           d S rd   )rF  rk  r  r  ry  update_worker_debugger_port)ra   portrm  s      rg   set_debugger_portzWorker.set_debugger_portv  s7    $2244	66y$GGGGGri   c                     || _         dS )z2Set the cached job id to speed `current_job_id()`.N)r2  )ra   job_ids     rg   set_cached_job_idzWorker.set_cached_job_idz  s    $ri   c              #     K   	 | j                             t          j                                                    d           dV  | j                             t          j                                                    d           dS # | j                             t          j                                                    d           w xY w)z(Use while the task is paused by debuggerTNF)rF  update_task_is_debugger_pausedr  get_runtime_context_get_current_task_idrv   s    rg   task_paused_by_debuggerzWorker.task_paused_by_debugger~  s      	;;'))>>@@$   EEE;;'))>>@@%    D;;'))>>@@%   s   AB A Cc              #   .  K   	 | j                                         }t          j        j                            |d           dV  t          j        j                            |d           dS # t          j        j                            |d           w xY w)z]
        Updates the worker num paused threads when the worker is paused by debugger
        rK   N)rF  rk  r  r  ry   update_worker_num_paused_threadsr{  s     rg   worker_paused_by_debuggerz Worker.worker_paused_by_debugger  s      
	O(6688IL??	1MMMEEEL??	2NNNNNCL??	2NNNNs   AA- -'Brotation_enabledNc                     || _         dS )z8Set whether rotation is enabled for outfile and errfile.N)r+  )ra   r  s     rg   set_file_rotation_enabledz Worker.set_file_rotation_enabled  s    &6###ri   c                     || _         dS )z7Set the worker's err file where stderr is redirected toNr-  )ra   err_filepaths     rg   set_err_filezWorker.set_err_file      )ri   c                     || _         dS )z7Set the worker's out file where stdout is redirected toNr,  )ra   out_filepaths     rg   set_out_filezWorker.set_out_file  r  ri   task_idattempt_numberc           	      N   | j         s| j                                        sdS t          | d          sdS | j        rdS | j                            |||                                 |                                 | 	                                | 
                                           dS )z[Record the task log info when task starts executing for
        non concurrent actor tasks.NrF  )r*  rM  is_nilrG  r+  rF  record_task_log_startget_out_file_pathget_err_file_pathget_current_out_offsetget_current_err_offsetra   r  r  s      rg   r  zWorker.record_task_log_start  s     1 	$-:N:N:P:P 	
 Ft]++ 	F& 	F..""$$""$$''))''))	
 	
 	
 	
 	
ri   c                    | j         s| j                                        sdS t          | d          sdS | j        rdS | j                            |||                                 |                                            dS )z]Record the task log info when task finishes executing for
        non concurrent actor tasks.NrF  )	r*  rM  r  rG  r+  rF  record_task_log_endr  r  r  s      rg   r  zWorker.record_task_log_end  s     1 	$-:N:N:P:P 	
 Ft]++ 	F & 	F,,''))''))		
 	
 	
 	
 	
ri   c                 "    | j         | j         ndS )zGet the out log file pathN r  rv   s    rg   r  zWorker.get_out_file_path      %)%7%Ct!!Kri   c                 "    | j         | j         ndS )zGet the err log file pathNr  r  rv   s    rg   r  zWorker.get_err_file_path  r  ri   c                 \    | j         $t          j                            | j                   S dS )z:Get the current offset of the out file if seekable, else 0Nr   )r,  ospathgetsizerv   s    rg   r  zWorker.get_current_out_offset  '    )7??4#5666qri   c                 \    | j         $t          j                            | j                   S dS )z:Get the current offset of the err file if seekable, else 0Nr   )r-  r  r  r  rv   s    rg   r  zWorker.get_current_err_offset  r  ri   c                 $   | j         }| j        }| j        5  ||vrVt          j                    |v r*|                    t          j                              ||<   nt          j        |           ||<   ||         cddd           S # 1 swxY w Y   dS )zGet the SerializationContext of the job that this worker is processing.

        Returns:
            The serialization context of the given job.
        N)rJ  r  r/  r   rI  popserializationSerializationContext)ra   r  context_maps      rg   get_serialization_contextz Worker.get_serialization_context  s     $4Y 	' 	'[((9;;+--*5//%)++*F*FK''*7*LT*R*RK'v&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	's   A"BB	B	c                 2    | j         st          d          dS )zCheck if the worker is connected.

        Raises:
          Exception: An exception is raised if the worker is not connected.
        zBRay has not been started yet. You can start Ray with 'ray.init()'.N)r;  r9   rv   s    rg   r@  zWorker.check_connected  s-     ~ 	 T  	 	ri   c                     || _         dS )a  Set the mode of the worker.

        The mode SCRIPT_MODE should be used if this Worker is a driver that is
        being run as a Python script or interactively in a shell. It will print
        information about task failures.

        The mode WORKER_MODE should be used if this Worker is not a driver. It
        will not print information about tasks.

        The mode LOCAL_MODE should be used if this Worker is a driver and if
        you want to run the driver in a manner equivalent to serial Python for
        debugging purposes. It will not send remote function calls to the
        scheduler and will instead execute them in a blocking fashion.

        Args:
            mode: One of SCRIPT_MODE, WORKER_MODE, and LOCAL_MODE.
        Nr  )ra   r  s     rg   set_modezWorker.set_mode  s    $ 			ri   c                     || _         d S rd   )r'  )ra   rC  s     rg   set_load_code_from_localzWorker.set_load_code_from_local  s    %9"""ri   Fvalueowner_address_is_experimental_channel_tensor_transportc                 B   t          |t                    rt          d          d}ddlm}m}  ||          } ||d           	 |t          j        j        k    r+| 	                                
                    |          \  }	}n'| 	                                                    |          }	n}# t          $ rp}
t          j                    }t          j                            ||           dt#          |           d|                                 }t          |          |
d}
~
ww xY w| }t          j        }|t          j        j        k    rt          j        }| j                            |	||d	||j        
          }|r| j                            |||           |S )a  Put value in the local object store.

        If the plasma store is full, the worker will automatically
        retry up to DEFAULT_PUT_OBJECT_RETRIES times. Each retry
        will delay for an exponentially doubling amount of time,
        starting with DEFAULT_PUT_OBJECT_DELAY. After this, exception
        will be raised.

        Args:
            value: The value to put in the object store.
            owner_address: The serialized address of object's owner.
            _is_experimental_channel: An experimental flag for mutable
                objects. If True, then the returned object will not have a
                valid value. The object must be written to using the
                ray.experimental.channel API before readers can read.
            _tensor_transport: [Alpha] The tensor transport backend to use. Currently, this supports "object_store" and "nixl".
        Returns:
            ObjectRef: The object ref the object was put under.

        Raises:
            ray.exceptions.ObjectStoreFullError: This is raised if the attempt
                to store the object fails because the object store is full even
                after multiple retries.
        zCalling 'put' on an ray.ObjectRef is not allowed. If you really want to do this, you can wrap the ray.ObjectRef in a list and call 'put' on it.Nr   )'normalize_and_validate_tensor_transportvalidate_one_sidedray.put)
print_filez"Could not serialize the put value z:
T)
pin_objectr  inline_small_objectr  tensor_transport_val)rq  r   	TypeError(ray.experimental.gpu_object_manager.utilr  r  r%   OBJECT_STOREnamer  serialize_gpu_objects	serializeioStringIOr  utilinspect_serializabilityreprgetvalueDIRECT_TRANSPORTrF  
put_objectr  r8  )ra   r  r  r  r  tensorsr  r  tensor_transportserialized_valuer  siomsgr  tensor_transport_enumrets                   rg   r  zWorker.put_object  s   @ eY'' 	@  
 	
 	
 	
 	
 	
 	
 	
 	

 CBCTUU+Y777	(#6#C#HHH 2244JJ5QQ$G $(#A#A#C#C#M#Me#T#T  	( 	( 	(+--CH,,Us,CCC$;;$ $<<>>$ $ 
 C..a'	( 21
 3 @2?DDD$7$H! ))!' $%=!6!< * 
 
  	O#..s4DgNNN
s   A'B/ /
D)9A+D$$D)c                 z    |                      ||          }dt          j        v rd S |D ]}t          |           d S )NRAY_IGNORE_UNHANDLED_ERRORS)deserialize_objectsr  environr  )ra   serialized_objectsobject_refsoutr  s        rg   raise_errorszWorker.raise_errorsz  sT    &&'9;GG(BJ66F 	( 	(A$Q''''	( 	(ri   tensor_transport_hintc                 p   i }t          ||          D ]S\  }\  }}}||t          j        k    r|                                }||vr| j                            ||          ||<   T| j        j        5  |                                 }	|		                    |||          cd d d            S # 1 swxY w Y   d S )N)r  )
zipr%   r  hexr8  get_gpu_objectr   r/  r  r  )
ra   r  r  r  gpu_objectsobj_ref_r  	object_idcontexts
             rg   r  zWorker.deserialize_objects  s+    8:14[BT1U1U 	 	-G-a, (#':'GGG I++
 *.)@)O)O0E *P * *I& (- 	 	4466G.."K 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   3+B++B/2B/r  timeoutreturn_exceptionsskip_deserializationc                    |D ]*}t          |t                    st          d| d          +||dk    rt          |dz            nd}| j                            ||          }d}	|D ]|\  }
}}
|rt|                    d          }t          |          dk    rL|d	                             t          j
                  r'|d	         t          t          j
                  d         }	}|rd|	fS |d
dlm}  ||          }|                     |||          }|st          |          D ]\  }}t          |t                    rt          |t           j        j                  r=t          |t           j        j                  st(          j                                         t          |t,                    r|                                |||	fS )a  Get the values in the object store associated with the IDs.

        Return the values from the local object store for object_refs. This
        will block until all the values for object_refs have been written to
        the local object store.

        Args:
            object_refs: A list of the object refs
                whose values should be retrieved.
            timeout: The maximum amount of time in
                seconds to wait before returning.
            return_exceptions: If any of the objects deserialize to an
                Exception object, whether to return them as values in the
                returned list. If False, then the first found exception will be
                raised.
            skip_deserialization: If true, only the buffer will be released and
                the object associated with the buffer will not be deserialized.
            _tensor_transport: [Alpha] The tensor transport to use to fetch `torch.Tensors` found in the Ray Direct Transport object. Currently, this supports "object_store" and "nixl".
        Returns:
            list: List of deserialized objects or None if skip_deserialization is True.
            bytes: UUID of the debugger breakpoint we should drop
                into or b"" if there is no breakpoint.
        z&Attempting to call `get` on the value z , which is not an ray.ObjectRef.Nr    ri      ,rL   rK   r   )r  )r  )rq  r   r  intrF  get_objectssplitlen
startswithr(  OBJECT_METADATA_DEBUG_PREFIXr  r  r  	enumerater8   r  
exceptionsObjectLostErrorOwnerDiedErrorglobal_workerlog_plasma_usager:   as_instanceof_cause)ra   r  r  r  r  r  
object_ref
timeout_msr  r$  r  metadatametadata_fieldsr  valuesir  s                    rg   r  zWorker.get_objects  sd   @ & 	 	Jj)44 5Z 5 5 5   $+#67b==C$b 	
 ((
 
 	 "0 	 	NAx "*.."6"6''1,,1C1N1N!>2 2, +:!*<MFGGII+'   	-,,,(      !H G!! ! ))CT * 
 
 ! 	$%f-- 	$ 	$5eX.. $!s~=  E(0MNNE &1BBDDD!%66 $#77999#$ ***ri   c                     d }t           j        j                            |           | j                                         t          j        d           dS )z9The main loop a worker runs to receive and execute tasks.c                 (    t          dd           d S )NzThe process receives a SIGTERM.rK   )	exit_code)r5   signumframes     rg   sigterm_handlerz)Worker.main_loop.<locals>.sigterm_handler   s(    41Q     ri   r   N)r  r  r  set_sigterm_handlerrF  run_task_loopsysexit)ra   r  s     rg   	main_loopzWorker.main_loop  sR    	 	 	 	..???&&(((ri   c                    | j         }|                                 t          j        j        }t          j                    }	 d}| j                                        }	 | j	        
                                r	 |                                 dS |                                }|d}J| j        r|d         r|d         |k    rd}h||d<   t          j        |           d|cxk    o
|j        k     nc }|rt"                              d           |j        }# t&          |f$ r'}t"                              d|            Y d}~nd}~ww xY w	 |                                 dS # |                                 w xY w)	z>Prints log messages from workers on all nodes in the same job.r   TNjob	localhostd   zThe driver may not be able to keep up with the stdout/stderr of the workers. To avoid forwarding logs to the driver, use 'ray.init(log_to_driver=False)'.zprint_logs: )gcs_log_subscriber	subscriber  r  RpcErrorservicesget_node_ip_addressrJ  r  r#  is_setclosepollr0  r(   emitlast_batch_sizer  warningOSErrorr  )	ra   
subscriberexception_typer  last_polling_batch_size
job_id_hexdatalaggingr  s	            rg   
print_logszWorker.print_logs
  s   ,
0022	-	 '(#,0022JE'..00 D A "((<./+ ,U Uz11./+$-[!27===!8UUUU:;UUUUU NN;   +5*D'?EB ( 	- 	- 	-LL+++,,,,,,,,	-, Js1   5D B	D E	"E?E# E		E# #E9resource_nameresource_regexc                    | j                                         }t                      }ddl}|                                D ]>\  }}||k    s|                    ||          r|D ]\  }}	|                    |           ?| j                            |d          | j        |         fd|D             }| j	        t          k    rm|t          j        k    r| j                                        j        }
n2| j                                        j                            |d          }
|
r
d|
         }t#          |          S )a  Get the accelerator IDs that are assigned to the given accelerator resource.

        Args:
            resource_name: The name of the resource.
            resource_regex: The regex of the resource.

        Returns:
            (List[str]) The IDs that are assigned to the given resource pre-configured.
            (List[int]) The IDs that are assigned to the given resource.
        r   Nc                 :    h | ]}t          |                   S re   )str).0r  original_idss     rg   	<setcomp>zFWorker.get_accelerator_ids_for_accelerator_resource.<locals>.<setcomp>`  s%    GGGQCQ00GGGri   )rF  resource_idssetreitemsmatchaddr  getr  
LOCAL_MODEr(  GPUr  get_resource_and_label_specnum_gpus	resourceslist)ra   r4  r5  r<  assigned_idsr>  resource
assignmentresource_idr  max_acceleratorsr:  s              @rg   ,get_accelerator_ids_for_accelerator_resourcez3Worker.get_accelerator_ids_for_accelerator_resource?  sf    '4466uu 				$0$6$6$8$8 	2 	2 Hj=((BHH^X,N,N(&0 2 2NK $$[1111 044]DIIU@OLGGGG,GGGLyJ&& M$555'+y'L'L'N'N'W$$ 	==??IMM)4  %
 $ C#/0A1A0A#BLL!!!ri   c                 J    | j         r| j                                          d S d S rd   )r  shutdownrv   s    rg   shutdown_gpu_object_managerz"Worker.shutdown_gpu_object_managero  s2    # 	0$--/////	0 	0ri   )rb   r4  )rb   Nrd   )NFFN)Hrj   rk   rl   __doc__rr   propertyr8  r;  boolr>  rA  rC  rJ  rM  rR  rU  rY  r\  r_  rb  rf  ri  rm  ro  rt  rw  r|  r  r  r  r  r   r  r  r  r   r   r  r  r4   r  r  r  r8  r  r  r  r  r  r@  r  r  r%   r  r  r   r  r  r  rH  floatr   r   r  SerializedRayObjectbytesr  r  r3  r   rN  rQ  re   ri   rg   r  r    s       
 
:) :) :)x ( ( ( X( " " X"*T * * * * ) ) X) * * X*   X   X
   X
 6 6 X6 8 8 X8 A A XA 6 6 X6 1 1 X1 ? ? X? 9 9 X9 9 9 X9 P P XP 9 9 X9 : : X: F F XF
   X % % X%H H H% % % 
 
 ^
 	O 	O ^	O7$ 74 7 7 7 7 )1(8 * * * * * )1(8 * * * * *
V 
S 
 
 
 
0
6 
3 
 
 
 
0L3 L L L LL3 L L L L        ' ' ',	 	 	  (: : : (,).!4!A!FY YY  }Y #'	Y
 Y Y Y Yv( ( ( 04	" "  (}	" " " "N $("'%*+/V+ V+V+ %V+  	V+
 #V+ $C=V+ 
tM56=	>V+ V+ V+ V+p  3 3 3j." ."25."	tCy$s)#	$." ." ." ."`0 0 0 0 0ri   r  funcrb   c                 <     t                      fd            }|S )Nc                  V    t           5   | i |cd d d            S # 1 swxY w Y   d S rd   )_connect_or_shutdown_lock)argskwargsrX  s     rg   wrapperz.with_connect_or_shutdown_lock.<locals>.wrapperx  s    & 	) 	)4(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	)s   ""r   )rX  r^  s   ` rg   with_connect_or_shutdown_lockr_  w  s3    
4[[) ) ) ) [) Nri   c                      t           } |                                  |                     t          j        dt          j         d          S )a  Get the IDs of the GPUs that are available to the worker.

    This method should only be called inside of a task or actor, and not a driver.

    If the CUDA_VISIBLE_DEVICES environment variable was set when the worker
    started up, then the IDs returned by this method will be a subset of the
    IDs in CUDA_VISIBLE_DEVICES. If not, the IDs will fall in the range
    [0, NUM_GPUS - 1], where NUM_GPUS is the number of GPUs that the node has.

    Returns:
        A list of GPU IDs.
    ^z_group_[0-9A-Za-z]+$)r  r@  rN  r(  rD  workers    rg   get_gpu_idsrd    sJ     F
>>F}0FFF  ri   z?Use ray.get_runtime_context().get_assigned_resources() instead.T)messager+  c                      t           } |                                  t                      t          k    rt	          d          t           j                                        S )a  Get the IDs of the resources that are available to the worker.

    Returns:
        A dictionary mapping the name of a resource to a list of pairs, where
        each pair consists of the ID of a resource and the fraction of that
        resource reserved for this worker.
    zCray._private.worker.get_resource_ids() does not work in local_mode.)r  r@  _moderC  RuntimeErrorrF  r<  rb  s    rg   get_resource_idsri    sV     F
ww*Q
 
 	
 $11333ri   z1Use ray.init().address_info['webui_url'] instead.)re  c                      t           j        t          j        v r6t	          t          j                            t           j                            S t          } |                                  t          j	        S )zGet the URL to access the Ray dashboard.

    Note that the URL does not specify which node the dashboard is on.

    Returns:
        The URL of the dashboard as a string.
    )
r(  RAY_OVERRIDE_DASHBOARD_URLr  r  _remove_protocol_from_urlrB  r  r@  _global_node	webui_urlrb  s    rg   get_dashboard_urlro    s\     /2:==(JNN=CDD
 
 	
    %%ri   urlc                     | s| S t           j                            |           }|j        r3|j         d}|                                                    |dd          S | S )zC
    Helper function to remove protocol from URL if it exists.
    ://r  rK   )urllibparser   schemegeturlreplace)rp  
parsed_urlru  s      rg   rl  rl    sk      
&&s++J :%***  ""**62q999Jri   c                       e Zd ZU dZee         ed<   eed<   eed<   ed             Zed             Z	ed             Z
d Zd	 Z ed
dg          deeef         fd            Zd ZdS )BaseContextz5
    Base class for RayContext and ClientContext
    dashboard_urlpython_versionray_versionc                     dS )z
        If this context is for directly attaching to a cluster, disconnect
        will call ray.shutdown(). Otherwise, if the context is for a ray
        client connection, the client will be disconnected.
        Nre   rv   s    rg   
disconnectzBaseContext.disconnect  s	     	ri   c                     d S rd   re   rv   s    rg   	__enter__zBaseContext.__enter__      ri   c                     d S rd   re   rv   s    rg   __exit__zBaseContext.__exit__  r  ri   c                     | j         r,t          d                              d| j         z             }nd }t          d                              | j        | j        |          S )Nzcontext_dashrow.html.j2http://)r{  zcontext_table.html.j2)r|  r}  dashboard_row)r{  rI   renderr|  r}  )ra   r  s     rg   _context_table_templatez#BaseContext._context_table_template  sz     	!$%>??FF'$*<< G  MM !M/0077.(' 8 
 
 	
ri   c                     t          d                              t          d                                          |                                           S )Nzcontext.html.j2context_logo.html.j2)context_logocontext_table)rI   r  r  rv   s    rg   _repr_html_zBaseContext._repr_html_  sO    )**11!"899@@BB6688 2 
 
 	
ri   
ipywidgets8rb   c           	          ddl }|                    dddd|                    d          	          } fd
}|                    |           |                    |                    t          d                                                    |g|                                          }|                                                               }|	                    ||g|                    d                    } |j
        di |S )a  Get the mimebundle for the widget representation of the context.

        Args:
            **kwargs: Passed to the _repr_mimebundle_() function for the widget

        Returns:
            Dictionary ("mimebundle") of the widget representation of the context.
        r   N
DisconnectFr  zDisconnect from the Ray clusterzauto 0px 0px 0px)margin)descriptiondisabledbutton_styletooltiplayoutc                 Z    d| _         d| _                                         d| _        d S )NTzDisconnecting...Disconnected)r  r  r  )buttonra   s    rg   disconnect_callbackz;BaseContext._get_widget_bundle.<locals>.disconnect_callback  s0    "FO!3FOO!/Fri   r  )r  z100%)widthre   )r  ButtonLayouton_clickVBoxHTMLrI   r  r  HBox_repr_mimebundle_)ra   r]  r  disconnect_buttonr  left_contentright_contentwidgets   `       rg   _get_widget_bundlezBaseContext._get_widget_bundle  s?    	&--$5$$,>$?? . 
 
	0 	0 	0 	0 	0 	""#6777!)? @ @ G G I IJJ! $$&& ' 
 
 #(D(D(F(FGG=)*2C2C&2C2Q2Q ! 
 
 (v'11&111ri   c                      | j         di |}|                    |                                 t          |           d           |S )N)z	text/htmlz
text/plainre   )r  updater  r  )ra   r]  bundles      rg   r  zBaseContext._repr_mimebundle_$  sM    ((22622 	D$4$4$6$6d4jjQQRRRri   N)rj   rk   rl   rR  r   r8  __annotations__r   r  r  r  r  r  rJ   r   r   r  r  re   ri   rg   rz  rz    s          C=     ^   ^   ^
 
 

 
 
 s+,,%2d38n %2 %2 %2 -,%2N    ri   rz  )	metaclassc                        e Zd ZU dZee         ed<   eed<   eed<   eed<   deeee         f         f fdZd Z	d	 Z
d
 ZddZd Zd Z xZS )
RayContextz/
    Context manager for attached drivers.
    r{  r|  r}  
ray_commitaddress_infoc                    t                                                       t                      | _         dj        t
          j        d d          | _        t          j	        | _
        t          j        | _        || _        d S )Nz{}.{}.{}rM   )superrr   ro  r{  formatr  version_infor|  r  __version__r}  
__commit__r  r  )ra   r  	__class__s     rg   rr   zRayContext.__init__7  sf    .00/j/1A"1"1EF?.(ri   c                     t          d          r#t          j        d| d| dt          d           | j        |         S )Nray_context_getitemzAccessing values through ctx["z("] is deprecated. Use ctx.address_info["z"] instead.rL   
stacklevel)rF   warningswarnDeprecationWarningr  )ra   keys     rg   __getitem__zRayContext.__getitem__?  se    )** 	M: : :),: : :"	     %%ri   c                 p    t          d          rt          j        d           t          | j                  S )Nray_context_lenz:len(ctx) is deprecated. Use len(ctx.address_info) instead.)rF   r  r  r  r  rv   s    rg   __len__zRayContext.__len__I  s6    %&& 	XMVWWW4$%%%ri   c                 p    t          d          rt          j        d           t          | j                  S )Nr  z<iter(ctx) is deprecated. Use iter(ctx.address_info) instead.)rF   r  r  iterr  rv   s    rg   __iter__zRayContext.__iter__N  s=    %&& 	MN   D%&&&ri   rb   c                     | S rd   re   rv   s    rg   r  zRayContext.__enter__U  s    ri   c                 ,    t          j                     d S rd   r  rP  )ra   excs     rg   r  zRayContext.__exit__X  s    ri   c                 ,    t          j                     d S rd   r  rv   s    rg   r  zRayContext.disconnect[  s    ri   )rb   r  )rj   rk   rl   rR  r   r8  r  r   rr   r  r  r  r  r  r  __classcell__)r  s   @rg   r  r  ,  s           C=   OOO)T#x}*<%= ) ) ) ) ) )& & && & &
' ' '           ri   r  rw  _skip_env_hookc                     t           j        rddlm}m}  |            }|r ||           S t           j        t          j        v r4|s2 t          t          j        t           j                           |           S | S )a4  
    If you set RAY_ENABLE_UV_RUN_RUNTIME_ENV, which is the default, and run the driver with `uv run`,
    this function sets up a runtime environment that replicates the driver's environment to the
    workers. Otherwise, if a runtime environment hook is present it will modify the runtime environment.
    r   )_get_uv_run_cmdlinehook)	r(  RAY_ENABLE_UV_RUN_RUNTIME_ENV,ray._private.runtime_env.uv_runtime_env_hookr  r  RAY_RUNTIME_ENV_HOOKr  r  r"   )rw  r  r  r  cmdlines        rg   _maybe_modify_runtime_envr  k  s     2 
%	
 	
 	
 	
 	
 	
 	
 	

 &%'' 	% 4$$$)RZ777Iz"*]%GHII+VVVri   F)num_cpusrF  rG  labelsobject_store_memory
local_modeignore_reinit_errorinclude_dashboarddashboard_hostdashboard_portr  configure_logginglogging_levellogging_formatr  log_to_driverrf  rw  enable_resource_isolationsystem_reserved_cpusystem_reserved_memoryaddressr  rF  rG  r  r  r  r  r  r  r  r  zray.job_config.JobConfigr  r  r  r  r  rf  
RuntimeEnvr  r  r  c                T   |t           j        }|rt          ||pt           j                   n+t	          j        d          j                                         |st           j        r0|pt          t           j                  }|
                                 |                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    d	d          }|                    d
d          }|                    dd          }|                    dt           j                  }|                    dt           j                  }|                    dd          } |                    dd          }!|                    dd          }"|                    dd          }#|                    dd          }$|                    dd          }%t          ||||          }&d }'t          j                    t          j                    u r%t$          j        j                            |'           nt,                              d           t0          j                            t           j                  }(|(r4| | dk    r,|(} t,                              d|( dt           j                    | -d| v r(t%          j        | d          })t=          j        t@                    }*i }+|*j!        "                                D ]0\  },}-|,dv r
|-j#        }.tI                      |,         }/|/|.k    r|/|+|,<   1|+%                    |            |)j&        dUi |+ |)'                                }0ddl(m)}1 |+                    d          d u r/|05  |1*                                 ddd           n# 1 swxY w Y   n|1*                                 |1+                    d!           |0S |                    d          rtY          d"t[                       d#          |                    d$          rtY          d%          |r'd&.                    |          }2tY          d'|2           	 ddl/}3|30                    |3j1                  \  }4}5|4|5k     rte          |4tg          |5d(                    }4t,          4                    d)|5            	 |35                    |3j1        |4|5f           n*# tl          $ r t,          4                    d*           Y nw xY w|30                    |3j1                  \  }4}6|4d+k     r-t,                              d,7                    |4                     n*# tp          $ r t,          4                    d-           Y nw xY w|t$          j9        :                                }tv          t0          j        v rty          j=        t0          j                            tv                              }7t$          j9        j:        >                    |7          }8|}9t          |8j@        |9t1          jA        d.          d/k    0          }|tm          d1|8j@         d2|9 d3          t          ||%          }|C                    |           |8jD        "                                D ]\  }:};|E                    |:|;           n't          ||%          }|r|C                    |           ||F                    |           d4\  }<}=t          jH        | |           }>|>|>}=t,                              d5|=           |r$t          }?t          jK        d6t          d78           nt          }?t          jO        r|rrt,                              d9           t          jP        Q                                }@t          t          t          jU        |@V                                :                    S tY          d;          |"pi }"t          |"t                    st          d<          |>t          |"d =           ddl(m)}1 |1Z                                r|1[                    d>           n|1\                    d           t$          j        j]        ^                    ||?d||||dd||||d||	|
||d| |"||!|#|$|&?          }At$          j        j_        `                    |Ad dd d @          aTnW||tm          dA          |tm          dB          |tm          dC          |tm          dD          |""t          |"          dk    rtm          dE          |rtm          dF          |$tm          dG          t          |"d=           t$          j        j]        ^                    ||=|<||| |"||!H	  	        }A	 t$          j        j_        `                    |Adddd I          aTnb# t          tX          f$ rN |=t$          j        j        c                    |           k    rt,                              dJ|= dK           t          w xY wt           jd        t0          j        v r*t0          j                            t           jd                  }Bnt          je        }B|Brt          |B          jg        sdL|Bz   }BdM}C|=dN}C|Br_t,                              |CdOz   t          ji        jj        t          jk        jl        |Bt          jk        jm        t          ji        jn                   nt,                              |C           tO          t          t          jo        |?|t          |d||t$          j        j        p                                P
  
         |r"|jq        rt          r                    d            nt          r                    d           t          D ]}D |D             ddQltmu}E  |Et$          j        jv        jw        d           }F|Fr)t          dR          rt          jK        dSt                     t          jP        Q                                }@t          jU        y                                }Gt          |B          |GdT<   t          t          |G|@V                                :                    S )Va&  
    Connect to an existing Ray cluster or start one and connect to it.

    This method handles two cases; either a Ray cluster already exists and we
    just attach this driver to it or we start all of the processes associated
    with a Ray cluster and attach to the newly started cluster.
    Note: This method overwrite sigterm handler of the driver process.

    In most cases, it is enough to just call this method with no arguments.
    This will autodetect an existing Ray cluster or start a new Ray instance if
    no existing cluster is found:

    .. testcode::

        ray.init()

    To explicitly connect to an existing local cluster, use this as follows. A
    ConnectionError will be thrown if no existing local cluster is found.

    .. testcode::
        :skipif: True

        ray.init(address="auto")

    To connect to an existing remote cluster, use this as follows (substituting
    in the appropriate address). Note the addition of "ray://" at the beginning
    of the address. This requires `ray[client]`.

    .. testcode::
        :skipif: True

        ray.init(address="ray://123.45.67.89:10001")

    More details for starting and connecting to a remote cluster can be found
    here: https://docs.ray.io/en/master/cluster/getting-started.html

    You can also define an environment variable called `RAY_ADDRESS` in
    the same format as the `address` parameter to connect to an existing
    cluster with ray.init() or ray.init(address="auto").

    Args:
        address: The address of the Ray cluster to connect to. The provided
            address is resolved as follows:
            1. If a concrete address (e.g., localhost:<port>) is provided, try to
            connect to it. Concrete addresses can be prefixed with "ray://" to
            connect to a remote cluster. For example, passing in the address
            "ray://123.45.67.89:50005" will connect to the cluster at the given
            address.
            2. If no address is provided, try to find an existing Ray instance
            to connect to. This is done by first checking the environment
            variable `RAY_ADDRESS`. If this is not defined, check the address
            of the latest cluster started (found in
            /tmp/ray/ray_current_cluster) if available. If this is also empty,
            then start a new local Ray instance.
            3. If the provided address is "auto", then follow the same process
            as above. However, if there is no existing cluster found, this will
            throw a ConnectionError instead of starting a new local Ray
            instance.
            4. If the provided address is "local", start a new local Ray
            instance, even if there is already an existing local Ray instance.
        num_cpus: Number of CPUs the user wishes to assign to each
            raylet. By default, this is set based on virtual cores.
        num_gpus: Number of GPUs the user wishes to assign to each
            raylet. By default, this is set based on detected GPUs.
        resources: A dictionary mapping the names of custom resources to the
            quantities for them available.
        labels: [Experimental] The key-value labels of the node.
        object_store_memory: The amount of memory (in bytes) to start the
            object store with.
            By default, this is 30% of available system memory capped by
            the shm size and 200G but can be set higher.
        local_mode: Deprecated: consider using the Ray Distributed Debugger instead.
        ignore_reinit_error: If true, Ray suppresses errors from calling
            ray.init() a second time. Ray won't be restarted.
        include_dashboard: Boolean flag indicating whether or not to start the
            Ray dashboard, which displays the status of the Ray
            cluster. If this argument is None, then the UI will be started if
            the relevant dependencies are present.
        dashboard_host: The host to bind the dashboard server to. Can either be
            localhost (127.0.0.1) or 0.0.0.0 (available from all interfaces).
            By default, this is set to localhost to prevent access from
            external machines.
        dashboard_port(int, None): The port to bind the dashboard server to.
            Defaults to 8265 and Ray will automatically find a free port if
            8265 is not available.
        job_config (ray.job_config.JobConfig): The job configuration.
        configure_logging: True (default) if configuration of logging is
            allowed here. Otherwise, the user may want to configure it
            separately.
        logging_level: Logging level for the "ray" logger of the driver process,
            defaults to logging.INFO. Ignored unless "configure_logging" is true.
        logging_format: Logging format for the "ray" logger of the driver process,
            defaults to a string containing a timestamp, filename, line number, and
            message. See the source file ray_constants.py for details. Ignored unless
            "configure_logging" is true.
        logging_config: [Experimental] Logging configuration will be applied to the
            root loggers for both the driver process and all worker processes belonging
            to the current job. See :class:`~ray.LoggingConfig` for details.
        log_to_driver: If true, the output from all of the worker
            processes on all nodes will be directed to the driver.
        namespace: A namespace is a logical grouping of jobs and named actors.
        runtime_env: The runtime environment to use
            for this job (see :ref:`runtime-environments` for details).
        object_spilling_directory: The path to spill objects to. The same path will
            be used as the object store fallback directory as well.
        enable_resource_isolation: Enable resource isolation through cgroupv2 by reserving
            memory and cpu resources for ray system processes. To use, only cgroupv2 (not cgroupv1)
            must be enabled with read and write permissions for the raylet. Cgroup memory and
            cpu controllers must also be enabled.
        system_reserved_cpu: The number of cpu cores to reserve for ray system processes.
            Cores can be fractional i.e. 1.5 means one and a half a cpu core.
            By default, the value will be atleast 1 core, and at maximum 3 cores. The default value
            is calculated using the formula min(3.0, max(1.0, 0.05 * num_cores_on_the_system))
            This option only works if enable_resource_isolation is True.
        system_reserved_memory: The amount of memory (in bytes) to reserve for ray system processes.
            By default, the value will be atleast 500MB, and at most 10GB. The default value is
            calculated using the formula min(10GB, max(500MB, 0.10 * memory_available_on_the_system))
            This option only works if enable_resource_isolation is True.
        _cgroup_path: The path for the cgroup the raylet should use to enforce resource isolation.
            By default, the cgroup used for resource isolation will be /sys/fs/cgroup.
            The process starting ray must have read/write permissions to this path.
            Cgroup memory and cpu controllers be enabled for this cgroup.
            This option only works if enable_resource_isolation is True.
        _enable_object_reconstruction: If True, when an object stored in
            the distributed plasma store is lost due to node failure, Ray will
            attempt to reconstruct the object by re-executing the task that
            created the object. Arguments to the task will be recursively
            reconstructed. If False, then ray.ObjectLostError will be
            thrown.
        _plasma_directory: Override the plasma mmap file directory.
        _node_ip_address: The IP address of the node that we are on.
        _driver_object_store_memory: Deprecated.
        _memory: Amount of reservable memory resource in bytes rounded
            down to the nearest integer.
        _redis_username: Prevents external clients without the username
            from connecting to Redis if provided.
        _redis_password: Prevents external clients without the password
            from connecting to Redis if provided.
        _temp_dir: If provided, specifies the root temporary
            directory for the Ray process. Must be an absolute path. Defaults to an
            OS-specific conventional location, e.g., "/tmp/ray".
        _metrics_export_port: Port number Ray exposes system metrics
            through a Prometheus endpoint. It is currently under active
            development, and the API is subject to change.
        _system_config: Configuration for overriding
            RayConfig defaults. For testing purposes ONLY.
        _tracing_startup_hook: If provided, turns on and sets up tracing
            for Ray. Must be the name of a function that takes no arguments and
            sets up a Tracer Provider, Remote Span Processors, and
            (optional) additional instruments. See more at
            docs.ray.io/tracing.html. It is currently under active development,
            and the API is subject to change.
        _node_name: User-provided node name or identifier. Defaults to
            the node IP address.

    Returns:
        If the provided address includes a protocol, for example by prepending
        "ray://" to the address to get "ray://1.2.3.4:10001", then a
        ClientContext is returned with information such as settings, server
        versions for ray and python, and the dashboard_url. Otherwise,
        a RayContext is returned with ray and python versions, and address
        information about the started processes.

    Raises:
        Exception: An exception is raised if an inappropriate combination of
            arguments is passed in.
    Nr  )encoding_cgroup_path_enable_object_reconstructionF_plasma_directoryobject_spilling_directory_node_ip_address_driver_object_store_memory_memory_redis_username_redis_password	_temp_dir_metrics_export_port_system_config_tracing_startup_hook
_node_namer  )r  cgroup_pathr  r  c                 .    t          j        |            d S rd   )r  r  r  s     rg   r  zinit.<locals>.sigterm_handler  s    ri   zISIGTERM handler is not set because current thread is not the main thread.autozUsing address z! set in the environment variable rr  )_deprecation_warn_enabled>   r]  r  r   )	usage_liballow_multipleTclientzj`allow_multiple` argument is passed to `ray.init` when the ray client is not used (https://docs.ray.io/en/z/cluster/running-applications/job-submission/ray-client.html#connect-to-multiple-ray-clusters-experimental). Do not pass the `allow_multiple` to `ray.init` to fix the issue.storagez{Cluster-wide storage configuration has been removed. The last Ray version supporting the `storage` argument is `ray==2.47`.z, zUnknown keyword argument(s): i   z7Automatically increasing RLIMIT_NOFILE to max value of zFailed to raise limit.i   zFile descriptor limit {} is too low for production servers and may result in connection errors. At least 8192 is recommended. --- Fix with 'ulimit -n 8192'z-Could not import resource module (on Windows)RAY_OVERRIDE_JOB_RUNTIME_ENV1)overridez&Failed to merge the Job's runtime env z with a ray.init's runtime env z because of a conflict. Specifying the same runtime_env fields or the same environment variable keys is not allowed. Use RAY_OVERRIDE_JOB_RUNTIME_ENV=1 to instruct Ray to combine Job and Driver's runtime environment in the event of a conflict.)NNz4Connecting to existing Ray cluster at address: %s...z`local_mode` is an experimental feature that is no longer maintained and will be removed in the near future. For debugging consider using the Ray distributed debugger.rL   r  z:Calling ray.init() again after it has already been called.)node_idzMaybe you called ray.init twice by accident? This error can be suppressed by passing in 'ignore_reinit_error=True' or by calling 'ray.shutdown()' prior to 'ray.init()'.z"The _system_config must be a dict.)create_token_if_missing)cli)rA  driver_moderedirect_outputr  rF  rG  r  num_redis_shardsredis_max_clientsredis_usernameredis_passwordplasma_directoryr  
huge_pagesr  r  r  memoryr  plasma_store_socket_nametemp_dirr  enable_object_reconstructionmetrics_export_porttracing_startup_hook	node_nameresource_isolation_config)
ray_paramsheadshutdown_at_exitspawn_reaperray_init_clusterzSWhen connecting to an existing cluster, num_cpus and num_gpus must not be provided.zGWhen connecting to an existing cluster, resources must not be provided.zDWhen connecting to an existing cluster, labels must not be provided.zQWhen connecting to an existing cluster, object_store_memory must not be provided.zLWhen connecting to an existing cluster, _system_config must not be provided.z[When connecting to an existing cluster, _enable_object_reconstruction must not be provided.zG_node_name cannot be configured when connecting to an existing cluster.)	rA  gcs_addressredis_addressr  r  r  r  r  r  )r  r  r  connect_onlyz8Failed to connect to the default Ray cluster address at z. This is most likely due to a previous Ray instance that has since crashed. To reset the default address to connect to, run `ray stop` or restart Ray with `ray start`.r  zConnected to Ray cluster.zStarted a local Ray instance.z" View the dashboard at %s%s%s %s%s)r  r  rc  driver_object_store_memoryr  rf  r  
entrypoint)env_bool"ray_accel_env_var_override_on_zerozTip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0rn  re   ){r(  RAY_LOG_TO_DRIVERr)   LOGGER_FORMATlogging	getLoggerhandlersclearRAY_LOGGING_CONFIG_ENCODINGr,   _applyr  REDIS_DEFAULT_USERNAMEREDIS_DEFAULT_PASSWORDr-   r!  current_threadmain_threadr  r  r  r  r  r+  r  r  rB   RAY_ADDRESS_ENVIRONMENT_VARIABLEinfor  inspect	signatureinit
parametersr?  defaultlocalsr  
_init_argsconnectray._common.usager  put_pre_init_usage_statsrecord_library_usagerh  r2   joinrJ  	getrlimitRLIMIT_NOFILEmaxmindebug	setrlimit
ValueErrorr  ImportErrorr  	JobConfigr.   jsonr  	from_jsonrB   rw  getenvr  set_runtime_envr  set_metadataset_py_logging_configr$  canonicalize_bootstrap_addressrC  r  r  FutureWarningSCRIPT_MODEr  r;  rF  r^  r  dictrm  r  r  rq  r  r#   is_nightly_wheelshow_usage_stats_prompt#set_usage_stats_enabled_via_env_var	parameter	RayParamsr  Noder  ConnectionErrorread_ray_addressrk  rn  r   ru  coloramaStyleBRIGHTForeGREENRESETNORMALsession_nameget_entrypoint_namecode_search_pathr  _post_init_hooksray._private.ray_constantsr!  accelerators*RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO_ENV_VARrF   copyrl  )Hr  r  rF  rG  r  r  r  r  r  r  r  r  r  r  r  r  r  rf  rw  r  r  r  r]  r  r  r  _object_spilling_directoryr  r  r  r  r  r  r  r  r  r  r  r  r  address_env_varbuilderinit_sigpassed_kwargsargument_name	param_objdefault_valuepassed_valuectxr  unknownrJ  softhardr  injected_job_config_jsoninjected_job_configdriver_runtime_envr  r  r  r  bootstrap_addressr  r  r  r{  info_strr  r!  override_on_zeroglobal_node_address_infosH                                                                           rg   r3  r3    s|   F %7  2]N$Qm6QRRRR%  )//111   B  ' 
=">,
 ,
 ,
 	 

>488L*0**'+ +! (.zz2Et'L'L06

#T1 1 #JJ'94@@17%t2 2 $ZZ	488G!::=? O "::=? O  &zz+t<<I*0**5KT*R*R/5zz:JD/Q/QN06

1 1 jjt44J!::&6>>N 7"; /5	! ! !   !!Y%:%<%<<<..????&	
 	
 	
 jnn]%STTO 
GOw&/@/@!I_ I I%FI I	
 	
 	

 u//*WFFF $T**(0(;(A(A(C(C 		< 		<$M9 555%-M!88M2L},, 0<m,V$$$++]+++oo//////-..$66 5 5224445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 ..000&&x000
zz"## 
O&9&;&;O O O
 
 	
 zz) 
U
 
 	

  F ))F##D7DDEEE''(>??
d$;;tSu--..DLLP$PP  7""8#9D$<HHHH 7 7 75666667$$X%;<<a$;;NN, -3F4LL	      DEEE ^--//
"bj00#':JNN677$
 $
  N$../GHH 	 )(+Y=>>#E
 
 

 &2 ,>  	 	 	 0^LL"";///-6<<>> 	0 	0JC##C////	0 0^LL 	4&&{333 !((888!+M; ?SS$'JKXXX 
" I 	
 	
 	
 	
 	
 "   
	KKTUUU#/CCEEGd<#<gkkmmTTTUUU:   $)rNnd++ ><===  	%^TRRRR 	0/////%%'' 	A--%-888899%@@@ \+55,# !"**.&@/)) 3%)))F 4!6 &?7 6 
 

B |(--!"! . 
 
 8#75    2   /   *<   %#n*=*=*B*B7   ) 	F   !'   	%^USSSS \+55,#'**))F 4 6 

 


	"<,11!&"! 2  LL  . 		" 		" 		"cl0AA)LLLL#"# # #   "!		" /2:==
}'OPP$.  2Xm44; 2!M1 +H2 
;;N!MMN!	
 	
 	
 	
 	H!##><%99;;     6j1 6..t4444 	..u555    433333x!L   
H%IJJ 
f 		
 	
 	
 ';;==G+8==??,Em,T,T[)d3W[[]]KKKLLLsW   'OOOA"U= $T U= $T)&U= (T))AU= =$V$#V$)g/ /Ai_exiting_interpreterc                    ddl m}  |             t                                           | r)t          j        t
          k    rt          j        d           t          |            t          t          d          rt          `
t                       t          t          d          rOt          j        t
          k    st          j        t          k    rt          j                                         t          `t          t                    t          _        t"          j        j        j                                         t(          Pt(                                          rt(                                           t(                              dd	           dat                              d           t                              d           dS )
aM  Disconnect the worker, and terminate processes started by ray.init().

    This will automatically run at the end when a Python process that uses Ray
    exits. It is ok to run this twice in a row. The primary use case for this
    function is to cleanup state between tests.

    Note that this will clear any remote function definitions, actor
    definitions, and existing actors, so if you wish to use any previously
    defined remote functions or actors after calling ray.shutdown(), then you
    need to redefine them. If they were defined in an imported module, then you
    will need to reload the module.

    Args:
        _exiting_interpreter: True if this is called by the atexit hook
            and false otherwise. If we are exiting the interpreter, we will
            wait a little while to print any extra error messages.
    r   )_shutdown_all_compiled_dagsg      ?
gcs_clientrF  NFT)check_aliveallow_graceful)ray.dag.compiled_dag_noder}  r  rQ  r  rN  timesleepr  rG  r~  r@   rC  rF  shutdown_driverr&   r   r  r  ry  rm  is_headdestroy_external_storagekill_all_processesr  r  )r{  r}  s     rg   rP  rP    s   , FEEEEE!!!--///  2k A A 	
3#$$$ }l++ %$
 }m,, &,,0Bj0P0P%55777%+?+N+NM(L''))) !! 	411333''E$'OOO 4   ##D)))))ri   c                    dd l mc mc m} t          j        t          k    rt          t          d          rd                    t          j
        |                    }t          j        }|j        }d|i}t          j        j        j                                         t          j        j        j                            |||           t%          | ||           d S )Nr   rm  r  	exception)ray.core.generated.common_pb2core	generated
common_pb2r  r  rN  rG  r<  	traceback	format_tbrm  DRIVERr  r  ry  _connect_and_get_accessor
add_workernormal_excepthook)typer  tbr  error_messagerm  worker_typeworker_infos           rg   custom_excepthookr  V  s    666666666666 [((W]K-P-P(	 3B 7 788!+	 '"M2 ::<<< ++I{KPPPdE2&&&&&ri   ignore_prefixc                 
   |                      d          dv}| d         r'|rt          j        |           }n| g}t          j        }n&|rt          j        |           }n| g}t          j        }|D ]}t          |||           d S )Npid)
autoscaleris_err)rB  r*   deduplicater  stderrr+   stdoutprint_worker_logs)r1  r  should_dedupbatchessinkbatchs         rg   print_to_stdstreamr  i  s    88E??.8LH~  	)5d;;GGfGz 	)5d;;GGfGz 6 6%}55556 6ri   linesc              #     K   t           j        sdS d}dt          fd}ddlm}  |            rRddlm}m} | D ]E} |            r|V   ||          }|r|j        s% ||t           j	                  r<|j        V  FdS | D ]E}t           j
        |v r5 |            r|V  |                    t           j
                  d         V  FdS )	a9  Given raw log lines from the monitor, return only autoscaler events.

    For Autoscaler V1:
        Autoscaler events are denoted by the ":event_summary:" magic token.
    For Autoscaler V2:
        Autoscaler events are published from log_monitor.py which read
        them from the `event_AUTOSCALER.log`.
    NzmTip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.rb   c                      t           sda dS dS )NTF)autoscaler_log_fyi_printedre   ri   rg   autoscaler_log_fyi_neededz;filter_autoscaler_events.<locals>.autoscaler_log_fyi_needed  s    ) 	)-&4uri   r   )is_autoscaler_v2)filter_event_by_levelparse_eventrK   )r(  AUTOSCALER_EVENTSrT  ray.autoscaler.v2.utilsr  ray._private.event.event_loggerr  r  re  RAY_LOG_TO_DRIVER_EVENT_LEVELLOG_PREFIX_EVENT_SUMMARYr   )	r  AUTOSCALER_LOG_FYIr  r  r  r  
event_lineeventlines	            rg   filter_autoscaler_eventsr    st      * 	0 t     988888 LVVVVVVVV 	  	 J((** )((((K
++E  $$}B   -	  	    	L 	LD5==,,.. -,,,, jj!GHHKKKK	L 	Lri   c                      t          j                     t          z
  } d}d}| dk    r|dz  }| dz  } | dk    | dk    r|dz  }| dz  } | dk    d}|r|| dz  }|r|| dz  }|t          |            dz  }|S )	zSReturn the relative time from the start of this job.

    For example, 15m30s.
    r   i  rK   <   r  hms)r  t0r  )deltahoursminutesoutputs       rg   time_stringr    s    
 IKK"EEG
$,,
 $,, "**1 "** F U+++  W---
UFMri   r1  r  c                    t           sd S dt          t          t          f         dt          fd}dt          t          t          f         dt          dt          fd}dt          t          t          f         dt          dt          fd}|                     d          dk    rEd	                    t                                }t          |                     d
g                     }n+|                     d          }|                     d
g           }|                     d          }||                     d          k    rdnd| }	|D ]}
t          |
v rt          |
           t                       t          j        d          dk    rd}d}n || |
          }t          j        j        }|rt           || |
           |           t          | d ||            | |	 d| d || |
           	|           t!                       d S )Nr1  rb   c                     |                      d          dv rdS d}|                      d          r| d          d| }n"|                      d          r| d          d| }|S )z!The PID prefix for this log line.r  )r  rayletr  zpid=rR   	task_namerB  )r1  ress     rg   
prefix_forz%print_worker_logs.<locals>.prefix_for  s    88E??6662Cxx%% 3l+33c33+&& 3k*22S22Jri   r  c                 l    t           j        |v r%|                    t           j                  d         S |S )z%The printed message of this log line.rK   )r(  LOG_PREFIX_INFO_MESSAGEr   )r1  r  s     rg   message_forz&print_worker_logs.<locals>.message_for  s/    0D88::mCDDQGGri   c                    |                      d          dk    rt          j        |vrt          j        j        S |                      d          dk    r*d|v sd|v rt          j        j        S t          j        j        S t          j        d          dk    rt          j        j	        t          j        j        t          j        j
        t          j        j        t          j        j        t          j        j        g}|                      dd          }	 t          |          }n# t          $ r d}Y nw xY w||t!          |          z           S t          j        j        S )	zThe color for this log line.r  r  r  zError:zWarning:RAY_COLOR_PREFIXr  r   )rB  r(  r  rX  r[  YELLOWCYANr  rH  MAGENTAr\  LIGHTBLACK_EXLIGHTBLUE_EXLIGHTMAGENTA_EXr  rC  r  )r1  r  colorsr  r  s        rg   	color_forz$print_worker_logs.<locals>.color_for  s0    HHUOOx''5TAA=''XXe__,,4:#5#5}++}))Y)**c11 %"# +* -F ((5!$$CHH   !c&kk/**=%%s   D D,+D,r  r  zautoscaler +{}r  ipr  r  z, ip=r  0)file()r  )_worker_logs_enabledr   r8  rB  r  r  r  rA   process_tqdm	hide_tqdmr  rH  rX  rY  	RESET_ALLprintrestore_tqdm)r1  r  r  r  r  r  r  r  r  	ip_prefixr  	color_pre
color_posts                rg   r  r    so      
c3h 
C 
 
 
 
$sCx.      #&S#X #&c #&c #& #& #& #&J xx,&&%%kmm44('2)>)>??hhuoo"%%	$BDHH[1111|r||I  T!!KKKy+,,33	

%IdD11	%^5
 
"{4..0#    
   1 1::d#3#3 1S 1) 1 1j 1 1"{4..1 1#     NNNNNri   c                     	 t          j        |           }t          j                                        |           dS # t
          $ r4 t          d          r!t                              d|  d           Y dS Y dS w xY w)z=Experimental distributed tqdm: see ray.experimental.tqdm_ray.tqdm_corruptionz[tqdm_ray] Failed to decode zN, this may be due to logging too fast. This warning will not be printed again.N)	rF  r  r;   instanceprocess_state_update	ExceptionrF   r  r+  )r  r1  s     rg   r  r  8	  s    z$0066666   %&& 	NNLt L L L     	 	 	s   :> 7A<;A<c                  P    t          j                                                     dS )zJHide distributed tqdm bars temporarily to avoid conflicts with other logs.N)r;   r  	hide_barsre   ri   rg   r  r  E	  s#    !!#####ri   c                  P    t          j                                                     dS )zUndo hide_tqdm().N)r;   r  unhide_barsre   ri   rg   r  r  J	  s#    ##%%%%%ri   c                    | j                                          	 t                      rGt          t          j                  }|,t                              |                                           	 |	                                rdS | j         
                                \  }}|6|d         G|d         | j                                        t          j                                                    fvr|d         }t          |gdddd           # t           t"          f$ r(}t                              d	|            Y d}~dS d}~ww xY w)
ai  Listen to error messages in the background on the driver.

    This runs in a separate thread on the driver and pushes (error, time)
    tuples to be published.

    Args:
        worker: The worker class that this thread belongs to.
        threads_stopped (threading.Event): A threading event used to signal to
            the thread that it should exit.
    NTr  r  r  F)r  r  r  r  zlisten_error_messages: )gcs_error_subscriberr"  r?   r>   r(  DEBUG_AUTOSCALING_ERRORr  r+  rQ  r&  r(  rJ  rl  r   rI  r  r,  rV  r  )rc  r#  r  r  
error_datar  s         rg   listen_error_messagesr  O	  s    ))+++4#%% 	7 -]-RSSM(}3355666	%%'' "7<<>>MAz!(#/Jx4H%,,..	""$$Q 5 5 &7M+_## 
 $   	. _% 4 4 42q223333333334s   A*D BD E%EEc                  8    t           j        j        j        j        S )zCheck if ray.init has been called yet.

    Returns:
        True if ray.init has already been called and false otherwise.
    )r  r  rc  r  r;  re   ri   rg   is_initializedr  	  s     <,66ri   r  r  r_  r  runtime_env_hashstartup_tokenr&  r   worker_launch_time_msworker_launched_time_msdebug_sourcec                    d}|j         r
J |            	 t          j                    st          j        d           n# t          j        $ r Y nw xY w|                                 |_        |j        J t          |j                   t          j
        j        j                            t          j        j                            | j        | j                                        dd                     |t&          t(          t*          fv r|J t-          j                    }n%|#t          j
        j                                        }|t2          ur_|t4          urVt6          j        }|t*          u rt6          j        }n|t(          u rt6          j        }t          j                            |           tA          |t,                    stC          d          | |_"        |#                    |           	 | $                                 nx# tJ          $ rk}|t2          k    r||t&          k    rItM          j'                    }t          j
        j(        )                    t6          j*        ||j                   Y d}~nd}~ww xY wd}d}|t2          k    r!d	dl+}tY          |d
          r|j-        }nd}d}nt4          st]          d          t          j        j                            | j        | j                                        dd          }|t          j/        0                                }|9t          j
        j(        1                    |           |2                    |           |t&          k    rdtf          j4        d<   ndtf          j4        d<   ||_5        |t2          k    r|j6        s|j7        rd	dl8m9} |j"        :                                }|j7        pi }tf          j4        ;                    |d          dk    }ty          |||tz                    }t}          |||tz                    }t          ||          }|@                    dd           |A                    |d           |t2          k    rg }|s|r|t          j        jB        k    sotf          jC        D                    tf          jC        E                    t          jG        d	                             }|t          jC        v r|H                    |           |j6        sW|I                                sCtf          jC        J                    tf          jC        jK                  }|H                    |           t          |          d	k    r|jM        N                    |           |O                                }| P                                sd} n| Q                                } t          j        R                    || jS        | jT        ||| | jU        | jV        |t4          k    ||| jW        |	|
|| j                                        |t2          k    rdn||||          |_X        |t2          k    rg|jY        }!t          j        Z                    |!|j        j[                  |_\        t          j        ]                    |!|j        j[                  |_^        |tz          _                    d           |t2          k    rt          ja        t          d||jc        f          |_d        d|jd        _e        |jd        f                                 t          jh        du}"|rst          jj        dt          jl        t          |"                     t          ja        |jn        d          |_o        d|jo        _e        |jo        f                                 |j        p                    dt6          jq                  }#|#vt          jr        js        jt        u                                 t          t          dd          s8t          |#x                    d                     }$ |$             dt          _y        |z                    d           dS )!ap  Connect this worker to the raylet, to Plasma, and to GCS.

    Args:
        node (ray._private.node.Node): The node to connect.
        session_name: The current Ray session name.
        mode: The mode of the worker. One of SCRIPT_MODE, WORKER_MODE, and LOCAL_MODE.
        log_to_driver: If true, then output from all of the worker
            processes on all nodes will be directed to the driver.
        worker: The ray.Worker instance.
        driver_object_store_memory: Deprecated.
        job_id: The ID of job. If it's None, then we will generate one.
        namespace: Namespace to use.
        job_config (ray.job_config.JobConfig): The job configuration.
        runtime_env_hash: The hash of the runtime env for this worker.
        startup_token: The startup token of the process assigned to
            it during startup as a command line argument.
        ray_debugger_external: If True, make the debugger external to the
            node this worker is running on.
        entrypoint: The name of the entrypoint script. Ignored if the
            mode != SCRIPT_MODE
        worker_launch_time_ms: The time when the worker process for this worker
            is launched. If the worker is not launched by raylet (e.g.,
            driver), this must be -1 (default value).
        worker_launched_time_ms: The time when the worker process for this worker
            finshes launching. If the worker is not launched by raylet (e.g.,
            driver), this must be -1 (default value).
        debug_source: Source information for `CoreWorker`, used for debugging and informational purpose, rather than functional purpose.
    z.Perhaps you called ray.init twice by accident?F)all_threadsN)allow_cluster_id_nilfetch_cluster_id_if_nilz'The type of given job id must be JobID.)r~  r  r   __file__TzINTERACTIVE MODEz6Invalid worker mode. Expected DRIVER, WORKER or LOCAL.zray.util.rpdb.set_tracePYTHONBREAKPOINTzray.util.rpdb._driver_set_trace) RAY_RUNTIME_ENV_IGNORE_GITIGNOREr  r  )include_gitignorescratch_dirr  excludes)validate)rm  r  zM`driver_object_store_memory` is deprecated and will be removed in the future.ray_listen_error_messages)targetr  r\  ray_print_logsr  )r
  r  s   tracing_startup_hook
__traced__rO  ){r;  faulthandler
is_enabledenabler  UnsupportedOperationget_gcs_clientr~  r=   r  r  ry  _initialize_global_state_rayletGcsClientOptionscreater  rr  r  WORKER_MODERESTORE_WORKER_MODESPILL_WORKER_MODEr   rI  next_job_idrN  rC  r(  WORKER_PROCESS_TYPE_IDLE_WORKER%WORKER_PROCESS_TYPE_SPILL_WORKER_IDLE'WORKER_PROCESS_TYPE_RESTORE_WORKER_IDLEsetproctitlerq  r  r  r  check_version_infor  r  
format_excr  publish_error_to_driverVERSION_MISMATCH_PUSH_ERROR__main__rG  r  rC  r  rE  validate_namespaceset_ray_namespacer  r  r&  _client_jobrw  rc  r  get_runtime_env_dir_pathrB  r/   r  r1   r0   r  rI   RAY_INTERNAL_DASHBOARD_NAMESPACEr  dirnamerealpathr  argvappend_runtime_env_has_working_dirabspathcurdirr  _py_driver_sys_pathextend
_serializeshould_redirect_logsget_logs_dir_path
CoreWorkerr  raylet_socket_namerA  node_manager_portmetrics_agent_portrF  rm  GcsErrorSubscriberr  r  GcsLogSubscriberr!  r+  r!  Threadr  r#  listener_threaddaemonstartr  r  r(   add_handler	functoolspartialr  r3  logger_threadinternal_kv_getKV_NAMESPACE_TRACINGr  tracingtracing_helper_enable_tracinggetattrrH   rQ  r  r>  )%r  r_  r  r  rc  r  r  rf  r  r  r  r&  r   r  r  r  r  process_namer  traceback_strdriver_nameinteractive_modemaingcs_optionsr  r  rw  r  
code_pathsscript_directorycurrent_directoryserialized_job_configlogs_dirrm  r  tracing_hook_val_setup_tracings%                                        rg   r8  r8  	  sj   ` EM....&(( 	3E2222"    ++--F(((F-...L55$++O!!!&$)	 	, 	
 	
   02CDDD~~~ >\'3355F;4z#9#9$D$$$(NLL((((PL  ...fe$$ CABBB FK
OOD!!!! 	 	 	;G[  %022ML669!, 7   	 K{4$$ 	--KK#,KK SQRRR+.55" %	 6  K ^--//
--i888 	$$Y///
 {)B
%&& *K
%&#8F 
 {:#9j>TOOOOOO!;??AA ,2 JNN+KSQQUXX1/#	
 
 
 3/#	
 
 
 A
 

 	
D)))"";">>>{ 
 		4		4#s{'SSS!wrw/?/?/L/LMM
  38++!!"2333 % 	1j.U.U.W.W 	1 " ? ?/000z??a*11*===&1133$$&& ,))++//%		k!!z) F. {$	&)k&D&D):)B 'E '
 '
# %(K$@$@):)B %A %
 %
! "-2	
 	
 	
 {!*!1(,&01"
 "
 "

 )-%$$&&& &8D 		).: !"4MRRR   $-#3(/?$ $ $F  +/F ' &&((( (88!C  #'77999sL%00 	"01A1H1H1Q1QRRNN!CN D!!!!!s(   (> AA0H 
I:A!I55I:c                    t           }|j        rU|j                                         t	          |d          r|j                                         t	          |d          r|j                                         t	          |d          r|j        	                                 t	          |d          r|j
        	                                 |j                                         |j        du}t          j                    D ]}t          |t           j        |           t%          j                    D ]}t          |t           j        |           t)          j        d           d|_        |j                                         	 t0          j        }n# t4          $ r d}Y nw xY w||j                                         |                    d           dS )z8Disconnect this worker from the raylet and object store.r  r!  r;  rA  Nr  F)r  r;  r#  r=  rG  r  r'  r!  r;  r<  rA  r(  r  r+   flushr  r  r  r*   r  r(   remove_handlerr  r  r  actorAttributeError_ActorClassMethodMetadatareset_cacher>  )exiting_interpreterrc  r  leftover	ray_actors        rg   r  r  
  s    F L 	""$$$6122 	0'--///6/00 	.%++---6,-- 	*"'')))6?++ 	( %%'''$$&&& 1=+133 	C 	CHh
MBBBB+133 	C 	CHh
MBBBB*9:JKKKFK
$**,,,I		   			+77999 E"""""s   F F"!F"c              #   \  K   t                      t          urt          j                            |            	 d V  t                      t          ur!t          j                            |           d S d S # t                      t          ur t          j                            |           w w xY wrd   )rg  rC  r  r  r  )title
next_titles     rg   _changeproctitlerb  
  s      wwj    '''177*$$K$$Z00000 %$577*$$K$$Z0000 %s   A4 47B+textre  r  dtypec                    t           }|                                 ddh}||v sJ d|             | |d}t          j        |                                          }|j                            |                                |           dS )a  Display message in dashboard.

    Display message for the current task or actor in the dashboard.
    For example, this can be used to display the status of a long-running
    computation.

    Args:
        message: Message to be displayed.
        key: The key name for the message. Multiple message under
            different keys will be displayed at the same time. Messages
            under the same key will be overridden.
        dtype: The type of message for rendering. One of the
            following: text, html.
    rc  htmlzdtype accepts only: )re  rd  N)r  r@  rF  dumpsencoderF  set_webui_display)re  r  rd  rc  acceptable_dtypesmessage_wrappedmessage_encodeds          rg   show_in_dashboardrm  
  s      F
(%%%%'Q>O'Q'Q%%%")E::Oj1188::O
((GGGGGri   r  r  zSequence[ObjectRef[R]]r  c                    d S rd   re   r  r  s     rg   rB  rB  	  	     Cri   zSequence[ObjectRef[Any]]c                    d S rd   re   rp  s     rg   rB  rB    rq  ri   rt   c                    d S rd   re   rp  s     rg   rB  rB        Cri   c                    d S rd   re   rp  s     rg   rB  rB    rq  ri   c                    d S rd   re   rp  s     rg   rB  rB  #  rt  ri   r  r  zObjectRef[Any]r  c          	      &   t           }|                                 t          |d          rV|j                                        r=t
          s6t          j        t          d          rt          
                    d           dat          j        d          5  t          | t                    r| cddd           S t          | t                    r"|                               cddd           S t          | t"                    rSd}d}| D ]}t          |t                    }|o|}|p|} |rfd| D             cddd           S |rt%          d	          t          | t&          j                  }|r| g} t          | t"                    s t%          d
t+          |            d          |                    | |          \  }	}
t/          |	          D ]\  }}t          |t0                    rt          |t&          j        j                  r8t          |t&          j        j                  s|j                                         t          |t:                    r|                                ||r|	d         }	|
dk    rst?          j                     j!        }t&          j"        j#        $                    dddd|
r|
%                                nd|j&                  }|'                    |           |	cddd           S # 1 swxY w Y   dS )a  Get a remote object or a list of remote objects from the object store.

    This method blocks until the object corresponding to the object ref is
    available in the local object store. If this object is not in the local
    object store, it will be shipped from an object store that has it (once the
    object has been created). If object_refs is a list, then the objects
    corresponding to each object in the list will be returned.

    Ordering for an input list of object refs is preserved for each object
    returned. That is, if an object ref to A precedes an object ref to B in the
    input list, then A will precede B in the returned list.

    This method will issue a warning if it's running inside async context,
    you can use ``await object_ref`` instead of ``ray.get(object_ref)``. For
    a list of object refs, you can use ``await asyncio.gather(*object_refs)``.

    Passing :class:`~ObjectRefGenerator` is not allowed.

    Related patterns and anti-patterns:

    - :doc:`/ray-core/patterns/ray-get-loop`
    - :doc:`/ray-core/patterns/unnecessary-ray-get`
    - :doc:`/ray-core/patterns/ray-get-submission-order`
    - :doc:`/ray-core/patterns/ray-get-too-many-objects`


    Args:
        object_refs: Object ref of the object to get or a list of object refs
            to get.
        timeout (Optional[float]): The maximum amount of time in seconds to
            wait before returning. Set this to None will block until the
            corresponding object becomes available. Setting ``timeout=0`` will
            return the object immediately if it's available, else raise
            GetTimeoutError in accordance with the above docstring.
        _tensor_transport: [Alpha] The tensor transport to use to fetch `torch.Tensors` found in the Ray Direct Transport object. Currently, this supports "object_store" and "nixl".

    Returns:
        A Python object or a list of Python objects.

    Raises:
        GetTimeoutError: A GetTimeoutError is raised if a timeout is set and
            the get takes longer than timeout to return.
        Exception: An exception is raised immediately if any task that created
            the object or that created one of the objects raised an exception,
            without waiting for the remaining ones to finish.
    rF  TzUsing blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.zray.getNrn  Fc                 <    g | ]}|                                S )rn  r  )r9  r  r  s     rg   
<listcomp>zget.<locals>.<listcomp>  s'    VVVJ
w77VVVri   zwInvalid type of object refs. 'object_refs' must be a list of CompiledDAGRefs if there is any CompiledDAGRef within it. zInvalid type of object refs, zO, is given. 'object_refs' must either be an ObjectRef or a list of ObjectRefs. rw  r   ri   )hostr  patch_stdstreamsquietbreakpoint_uuiddebugger_external)r  )(r  r@  rG  rF  current_actor_is_asyncio blocking_get_inside_async_warnedr(  r!  r!   r  r+  	profilingprofilerq  r3   r<   rB  rH  rC  r  r   r  r  r  r8   r  r  r  r	  r:   r
  r  	_getframef_backr  pdb_connect_ray_pdbrQ  r&  	set_trace)r  r  r  rc  all_compiled_dag_refsany_compiled_dag_refsr  
is_dag_refis_individual_idr  r$  r  r  r  rdbs    `             rg   rB  rB  (  sK   v F
v}%% 8&*<*U*U*W*W 8/ 	8%:  
8 A   480		9	%	% C C k#566 		C C C C C C C C k>22 	4??7?33C C C C C C C C k4(( 	$(!$)!) L L
'
NCC
(=(L*%(=(K%%$ VVVV+VVV#C C C C C C C C$ '  Q  
 &k3=AA 	(&-K+t,, 	V[0A0A V V V  
 '-&8&8<M '9 '
 '
## "&)) 	  	 HAu%** 
  eS^%CDD :Z3>8N N : &77999e\22  33555K
   	AYF#%%MOO*E(,//!&4GQ'..000T"("> 0 	 	C MMM&&&GC C C C C C C C C C C C C C C C C Cs&   L;*L2A
L	F0LL
L
object_store)_ownerr  r  r  zray.actor.ActorHandlezray.ObjectRefc                   t           }|                                 |d}nt          |t          j        j                  rt          j        j        j                                         t          j	        
                    t          j        j        j                            |j                            }|st          | d          nt          dt          |                     t!          j        d          5  	 |                    | ||          }n)# t&          $ r t(                              d            w xY w|cddd           S # 1 swxY w Y   dS )a!  Store an object in the object store.

    The object may not be evicted while a reference to the returned ID exists.

    Related patterns and anti-patterns:

    - :doc:`/ray-core/patterns/return-ray-put`
    - :doc:`/ray-core/patterns/pass-large-arg-by-value`
    - :doc:`/ray-core/patterns/closure-capture-large-objects`

    Args:
        value: The Python object to be stored.
        _owner [Experimental]: The actor that should own this object. This
            allows creating objects with lifetimes decoupled from that of the
            creating process. The owner actor must be passed a reference to the
            object prior to the object creator exiting, otherwise the reference
            will still be lost. *Note that this argument is an experimental API
            and should be avoided if possible.*
        _tensor_transport: [Alpha] The tensor transport to use for the GPU object. Currently, this supports "object_store" and "nixl" for tensor transport in ray.put().

    Returns:
        The object ref assigned to this value.
    Nz' is not alive, it's worker_id is empty!z,Expect an `ray.actor.ActorHandle`, but got: r  )r  r  zXPut failed since the value was either too large or the store was full of pinned objects.)r  r@  rq  r  rX  ActorHandler  ry  r  r  +_get_actor_serialized_owner_address_or_noneget_actor_info	_actor_idrh  r  r  r  r  r  r7   r  r0  )r  r  r  rc  serialize_owner_addressr  s         rg   putr    s   > F
~"&	FCI1	2	2 W ::<<<KCC"(778HII  	 
 ' 	S&QQQRRR	S UtF||UUVVV		9	%	%  	**5"3 +  JJ
 $ 	 	 	KK4   	                  s*   ;E=DE&D<<EEE)num_returnsr  fetch_localray_waitablesr  r  c                   t           }|                                 t          |d          rB|j                                        r)|dk    r#t
          st                              d           dat          | t                    st          | t                    rt          d|            t          | t                    st          dt          |                      ||dk     rt          d|           | D ]K}t          |t                    s4t          |t                    st          d	t          |                     L|                                 t          j        d
          5  t#          |           dk    rg g fcddd           S t#          |           t#          t%          |                     k    rt          d          |dk    rt          d|z            |t#          |           k    rt          d          ||nd}t'          |dz            }|j                            | |||          \  }}||fcddd           S # 1 swxY w Y   dS )a	  Return a list of IDs that are ready and a list of IDs that are not.

    If timeout is set, the function returns either when the requested number of
    IDs are ready or when the timeout is reached, whichever occurs first. If it
    is not set, the function simply waits until that number of objects is ready
    and returns that exact number of object refs.

    `ray_waitables` is a list of :class:`~ray.ObjectRef` and
    :class:`~ray.ObjectRefGenerator`.

    The method returns two lists, ready and unready `ray_waitables`.

    ObjectRef:
        object refs that correspond to objects that are available
        in the object store are in the first list.
        The rest of the object refs are in the second list.

    ObjectRefGenerator:
            Generators whose next reference (that will be obtained
            via `next(generator)`) has a corresponding object available
            in the object store are in the first list.
            All other generators are placed in the second list.

    Ordering of the input list of ray_waitables is preserved. That is, if A
    precedes B in the input list, and both are in the ready list, then A will
    precede B in the ready list. This also holds true if A and B are both in
    the remaining list.

    This method will issue a warning if it's running inside an async context.
    Instead of ``ray.wait(ray_waitables)``, you can use
    ``await asyncio.wait(ray_waitables)``.

    Related patterns and anti-patterns:

    - :doc:`/ray-core/patterns/limit-pending-tasks`
    - :doc:`/ray-core/patterns/ray-get-submission-order`

    Args:
        ray_waitables: List of :class:`~ObjectRef` or
            :class:`~ObjectRefGenerator` for objects that may or may
            not be ready. Note that these must be unique.
        num_returns: The number of ray_waitables that should be returned.
        timeout: The maximum amount of time in seconds to wait before
            returning.
        fetch_local: If True, wait for the object to be downloaded onto
            the local node before returning it as ready. If the `ray_waitable`
            is a generator, it will wait until the next object in the generator
            is downloaed. If False, ray.wait() will not trigger fetching of
            objects to the local node and will return immediately once the
            object is available anywhere in the cluster.

    Returns:
        A list of object refs that are ready and a list of the remaining object
        IDs.
    rF  r   z}Using blocking ray.wait inside async method. This blocks the event loop. Please use `await` on object ref with asyncio.wait. Tzxwait() expected a list of ray.ObjectRef or ray.ObjectRefGenerator, got a single ray.ObjectRef or ray.ObjectRefGenerator zGwait() expected a list of ray.ObjectRef or ray.ObjectRefGenerator, got Nz5The 'timeout' argument must be nonnegative. Received zWwait() expected a list of ray.ObjectRef or ray.ObjectRefGenerator, got list containing zray.waitz-Wait requires a list of unique ray_waitables.z'Invalid number of objects to return %d.zTnum_returns cannot be greater than the number of ray_waitables provided to ray.wait.i@B r  )r  r@  rG  rF  r  !blocking_wait_inside_async_warnedr  rA  rq  r   r3   r  rH  r  rC  r  r  r  r=  r  wait)	r  r  r  r  rc  ray_waitabletimeout_milliseconds	ready_idsremaining_idss	            rg   r  r     sE   F F
 	&&577995 qLL 1 	5LL4  
 15--++ 
z)0 0 
  
 
 	
 mT** 
)&&) )
 
 	
 w{{PwPP
 
 	
 &  ,	22 	:,<
 <
 	 <'+L'9'9< <  
  
	:	&	& ( ( }""r6( ( ( ( ( ( ( ( }S%7%7!8!888LMMM!FTUUU]++++9  
 %0''e"7T>22#)#5#:#: 	$
 $
 	= -'3( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s   .IB1IIIr  c                     | st          d          |$t          j        j                            |           t
          }|                                 |j                            | |pd          S )a  Get a handle to a named actor.

    Gets a handle to an actor with the given name. The actor must
    have been created with Actor.options(name="name").remote(). This
    works for both detached & non-detached actors.

    This method is a sync call and it'll timeout after 60s. This can be modified
    by setting OS env RAY_gcs_server_request_timeout_seconds before starting
    the cluster.

    Args:
        name: The name of the actor.
        namespace: The namespace of the actor, or None to specify the current
            namespace.

    Returns:
        ActorHandle to the actor.

    Raises:
        ValueError: if the named actor does not exist.
    z,Please supply a non-empty value to get_actorNr  )	rC  r  r  r  r#  r  r@  rF  get_named_actor_handle)r  rf  rc  s      rg   	get_actorr    so    0  IGHHH--i888F
44T9?KKKri   )
no_restartrX  r  c                   t           }|                                 t          | t          j        j                  s/t          d                    t          |                               |j	        
                    | j        |           dS )a  Kill an actor forcefully.

    This will interrupt any running tasks on the actor, causing them to fail
    immediately. ``atexit`` handlers installed in the actor will not be run.

    If you want to kill the actor but let pending tasks finish,
    you can call ``actor.__ray_terminate__.remote()`` instead to queue a
    termination task. Any ``atexit`` handlers installed in the actor *will*
    be run in this case.

    If the actor is a detached actor, subsequent calls to get its handle via
    ray.get_actor will fail.

    Args:
        actor: Handle to the actor to kill.
        no_restart: Whether or not this actor should be restarted if
            it's a restartable actor.
    zKray.kill() only supported for actors. For tasks, try ray.cancel(). Got: {}.N)r  r@  rq  r  rX  r  rC  r  r  rF  
kill_actor_ray_actor_id)rX  r  rc  s      rg   killr    s    * F
eSY233 
vd5kk**
 
 	
 !!%"5zBBBBBri   )force	recursiver  )rt   zObjectRefGenerator[R]r  r  c                r   t           j        j        j        }|                                 t          | t           j        j                  rt          | d          sJ | j	        } t          | t           j
                  s t          dt          |            d          |j                            | ||          S )a	  Cancels a task.

    Cancel API has a different behavior depending on if it is a remote function
    (Task) or a remote Actor method (Actor Task).

    Task:
        If the specified Task is pending execution, it is cancelled and not
        executed. If the Task is currently executing, the behavior depends
        on the `force` flag. When `force=False`, a KeyboardInterrupt is
        raised in Python and when `force=True`, the executing Task
        immediately exits. If the Task is already finished, nothing happens.

        Cancelled Tasks aren't retried. `max_task_retries` aren't respected.

        Calling ray.get on a cancelled Task raises a TaskCancelledError
        if the Task has been scheduled or interrupted.
        It raises a WorkerCrashedError if `force=True`.

        If `recursive=True`, all the child Tasks and Actor Tasks
        are cancelled. If `force=True` and `recursive=True`, `force=True`
        is ignored for child Actor Tasks.

    Actor Task:
        If the specified Task is pending execution, it is cancelled and not
        executed. If the Task is currently executing, the behavior depends
        on the execution model of an Actor. If it is a regular Actor
        or a threaded Actor, the execution isn't cancelled.
        Actor Tasks cannot be interrupted because Actors have
        states. If it is an async Actor, Ray cancels a `asyncio.Task`.
        The semantic of cancellation is equivalent to asyncio's cancellation.
        https://docs.python.org/3/library/asyncio-task.html#task-cancellation
        If the Task has finished, nothing happens.

        Only `force=False` is allowed for an Actor Task. Otherwise, it raises
        `ValueError`. Use `ray.kill(actor)` instead to kill an Actor.

        Cancelled Tasks aren't retried. `max_task_retries` aren't respected.

        Calling ray.get on a cancelled Task raises a TaskCancelledError
        if the Task has been scheduled or interrupted. Also note that
        only async actor tasks can be interrupted.

        If `recursive=True`, all the child Tasks and actor Tasks
        are cancelled.

    Args:
        ray_waitable: :class:`~ObjectRef` and
            :class:`~ObjectRefGenerator`
            returned by the task that should be canceled.
        force: Whether to force-kill a running task by killing
            the worker that is running the task.
        recursive: Whether to try to cancel tasks submitted by the
            task specified.
    _generator_refzNray.cancel() only supported for object refs. For actors, try ray.kill(). Got: .)r  r  rc  r  r@  rq  r  r3   rG  r  r   r  r  rF  cancel_task)r  r  r  rc  s       rg   cancelr    s    | \ .F
, >?? 3|%566666#2lCM22 
F04\0B0BF F F
 
 	
 )),yIIIri   c                     | j         S )a/  This is a wrapper around worker.mode.

    We use this wrapper so that in the remote decorator, we can call _mode()
    instead of worker.mode. The difference is that when we attempt to
    serialize remote functions, we don't attempt to serialize the worker
    object, which cannot be serialized.
    r  rb  s    rg   rg  rg     s     ;ri   c                    | j         sd| _         t          j        |           st          |           rBt	          j        |d           t          j                            t          j
        | d |          S t          j        |           r6t	          j        |d           t          j                            | |          S t          d          )NglobalF)
in_optionszJThe @ray.remote decorator must be applied to either a function or a class.)rk   r1  
isfunctionr'   r    validate_task_optionsr  remote_functionRemoteFunctionr   PYTHONisclassvalidate_actor_optionsrX  _make_actorr  )function_or_classrh   s     rg   _make_remoter  +  s    ' 0'/$+,, 
	:K0L0L 
.w5IIII"11O	
 
 	
 ()) A/EJJJJy$$%6@@@
T  ri   c                      e Zd Zedee         dee         fd            Zedeg e	f         de
e	         fd            Zedeege	f         dee	ef         fd            Zedeeege	f         dee	eef         fd            Zedeeeege	f         dee	eeef         fd            Zedeeeeege	f         dee	eeeef         fd	            Zedeeeeeege	f         dee	eeeeef         fd
            Zedeeeeeeege	f         dee	eeeeeef         fd            Zedeeeeeeeege	f         dee	eeeeeeef         fd            Zedeeeeeeeeege	f         dee	eeeeeeeef	         fd            Zedeeeeeeeeeeg	e	f         dee	eeeeeeeeef
         fd            Zedeeeeeeeeeeeg
e	f         dee	eeeeeeeeeef         fd            ZdS )RemoteDecorator_RemoteDecorator__trb   c                     d S rd   re   )ra   r  s     rg   __call__zRemoteDecorator.__call__B      ri   _RemoteDecorator__functionc                     d S rd   re   ra   r  s     rg   r  zRemoteDecorator.__call__F  r  ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__J  r  ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__N  r  ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__R  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__X  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__^  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__d  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__j  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__p  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__v  r   ri   c                     d S rd   re   r  s     rg   r  zRemoteDecorator.__call__|  r   ri   N)rj   rk   rl   r   r   rO   r6   r  r   rZ   rn   rP   r}   rQ   r   rR   r   rS   r   rT   r   rU   r   rV   r   rW   r   rX   r   rY   r   re   ri   rg   r  r  A  s       DG 
1    X 8BE? 7KA7N    X 8RD!G#4 B9O    X 8RHaK#8 _QPRTVY=W    X "BB<?3	BB	'   X
 "BB#3Q#67	BB*	+   X
 "BBB#7#:;	BBB.	/   X
 "BBB#;Q#>?	BBB2	3   X
 "BBBB#?#BC	BBBB6	7   X
 "BBBB#CQ#FG	BBBB:	;   X
 "BBBBB#G#JK	BBBBB>	?   X
 "BBBBB#KQ#NO	BBBBBB	C   X  ri   r  __tc                     d S rd   re   )r  s    rg   rw   rw     rt  ri   
__functionc                     d S rd   re   r  s    rg   rw   rw     rt  ri   c                     d S rd   re   r  s    rg   rw   rw     rt  ri   c                     d S rd   re   r  s    rg   rw   rw     rt  ri   c                     d S rd   re   r  s    rg   rw   rw     rt  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   c                     d S rd   re   r  s    rg   rw   rw     rq  ri   r  r  rF  rG  accelerator_typer  	max_callsmax_restartsmax_task_retriesmax_retriesrw  retry_exceptionsscheduling_strategylabel_selectorfallback_strategy	streamingr  r  r  r  r  r  r  r  DEFAULTSPREADr  r  c                     d S rd   re   r  s                  rg   rw   rw     s	    * Cri   c                  N   t          |           dk    r>t          |          dk    r+t          | d                   rt          | d         i           S t          |           dk    rt          |          dk    sJ t          j                    t          j        t          |          S )aw(  Defines a remote function or an actor class.

    This function can be used as a decorator with no arguments
    to define a remote function or actor as follows:

    .. testcode::

        import ray

        @ray.remote
        def f(a, b, c):
            return a + b + c

        object_ref = f.remote(1, 2, 3)
        result = ray.get(object_ref)
        assert result == (1 + 2 + 3)

        @ray.remote
        class Foo:
            def __init__(self, arg):
                self.x = arg

            def method(self, a):
                return self.x + a

        actor_handle = Foo.remote(123)
        object_ref = actor_handle.method.remote(321)
        result = ray.get(object_ref)
        assert result == (123 + 321)

    Equivalently, use a function call to create a remote function or actor.

    .. testcode::

        def g(a, b, c):
            return a + b + c

        remote_g = ray.remote(g)
        object_ref = remote_g.remote(1, 2, 3)
        assert ray.get(object_ref) == (1 + 2 + 3)

        class Bar:
            def __init__(self, arg):
                self.x = arg

            def method(self, a):
                return self.x + a

        RemoteBar = ray.remote(Bar)
        actor_handle = RemoteBar.remote(123)
        object_ref = actor_handle.method.remote(321)
        result = ray.get(object_ref)
        assert result == (123 + 321)


    It can also be used with specific keyword arguments as follows:

    .. testcode::

        @ray.remote(num_gpus=1, max_calls=1, num_returns=2)
        def f():
            return 1, 2

        @ray.remote(num_cpus=2, resources={"CustomResource": 1})
        class Foo:
            def method(self):
                return 1

    Remote task and actor objects returned by @ray.remote can also be
    dynamically modified with the same arguments as above using
    ``.options()`` as follows:

    .. testcode::
        :hide:

        ray.shutdown()

        ray.init(num_cpus=5, num_gpus=5)

    .. testcode::

        @ray.remote(num_gpus=1, max_calls=1, num_returns=2)
        def f():
            return 1, 2

        f_with_2_gpus = f.options(num_gpus=2)
        object_refs = f_with_2_gpus.remote()
        assert ray.get(object_refs) == [1, 2]

        @ray.remote(num_cpus=2, resources={"CustomResource": 1})
        class Foo:
            def method(self):
                return 1

        Foo_with_no_resources = Foo.options(num_cpus=1, resources=None)
        foo_actor = Foo_with_no_resources.remote()
        assert ray.get(foo_actor.method.remote()) == 1


    A remote actor will be terminated when all actor handle to it
    in Python is deleted, which will cause them to complete any outstanding
    work and then shut down. If you only have 1 reference to an actor handle,
    calling ``del actor`` *could* trigger actor deletion. Note that your program
    may have multiple references to the same ActorHandle, and actor termination
    will not occur until the reference count goes to 0. See the Python
    documentation for more context about object deletion.
    https://docs.python.org/3.9/reference/datamodel.html#object.__del__

    If you want to kill actors immediately, you can also call ``ray.kill(actor)``.

    .. tip::
        Avoid repeatedly passing in large arguments to remote task or method calls.

        Instead, use ray.put to create a copy of the object in the object store.

        See :ref:`more info here <ray-pass-large-arg-by-value>`.

    Args:
        num_returns: This is only for *remote functions*. It specifies
            the number of object refs returned by the remote function
            invocation. The default value is 1.
            Pass "dynamic" to allow the task to decide how many
            return values to return during execution, and the caller will
            receive an ObjectRef[DynamicObjectRefGenerator].
            See :ref:`dynamic generators <dynamic-generators>` for more details.
        num_cpus: The quantity of CPU resources to reserve
            for this task or for the lifetime of the actor.
            By default, tasks use 1 CPU resource and actors use 1 CPU
            for scheduling and 0 CPU for running
            (This means, by default, actors cannot get scheduled on a zero-cpu node,
            but an infinite number of them can run on any non-zero cpu node.
            The default value for actors was chosen for historical reasons.
            It's recommended to always explicitly set num_cpus for actors
            to avoid any surprises.
            If resources are specified explicitly,
            they are required for both scheduling and running.)
            See :ref:`specifying resource requirements <resource-requirements>`
            for more details.
        num_gpus: The quantity of GPU resources to reserve
            for this task or for the lifetime of the actor.
            The default value is 0.
            See :ref:`Ray GPU support <gpu-support>` for more details.
        resources (Dict[str, float]): The quantity of various
            :ref:`custom resources <custom-resources>`
            to reserve for this task or for the lifetime of the actor.
            This is a dictionary mapping strings (resource names) to floats.
            By default it is empty.
        label_selector: [Experimental] If specified, the labels required for the node on
                which this actor can be scheduled on. The label selector consist of key-value pairs,
                where the keys are label names and the value are expressions consisting of an operator
                with label values or just a value to indicate equality.
        fallback_strategy: [Experimental] If specified, expresses soft constraints for scheduling
                through a list of dicts of decorator options to fall back on when scheduling on a node.
                Decorator options are evaluated together during scheduling. The first satisfied
                dict of options is used. Currently only `label_selector` is a supported option.
        accelerator_type: If specified, requires that the task or actor run
            on a node with the specified type of accelerator.
            See :ref:`accelerator types <accelerator_types>`.
        memory: The heap memory request in bytes for this task/actor,
            rounded down to the nearest integer.
        max_calls: Only for *remote functions*. This specifies the
            maximum number of times that a given worker can execute
            the given remote function before it must exit
            (this can be used to address :ref:`memory leaks <gpu-leak>` in third-party
            libraries or to reclaim resources that cannot easily be
            released, e.g., GPU memory that was acquired by TensorFlow).
            By default this is infinite for CPU tasks and 1 for GPU tasks
            (to force GPU tasks to release resources after finishing).
        max_restarts: Only for *actors*. This specifies the maximum
            number of times that the actor should be restarted when it dies
            unexpectedly. The minimum valid value is 0 (default),
            which indicates that the actor doesn't need to be restarted.
            A value of -1 indicates that an actor should be restarted
            indefinitely.
            See :ref:`actor fault tolerance <fault-tolerance-actors>` for more details.
        max_task_retries: Only for *actors*. How many times to
            retry an actor task if the task fails due to a system error,
            e.g., the actor has died. If set to -1, the system will
            retry the failed task until the task succeeds, or the actor
            has reached its max_restarts limit. If set to `n > 0`, the
            system will retry the failed task up to n times, after which the
            task will throw a `RayActorError` exception upon :obj:`ray.get`.
            Note that Python exceptions are not considered system errors
            and will not trigger retries.
            The default value is 0.
            See :ref:`actor fault tolerance <fault-tolerance-actors>` for more details.
        max_retries: Only for *remote functions*. This specifies
            the maximum number of times that the remote function
            should be rerun when the worker process executing it
            crashes unexpectedly. The minimum valid value is 0,
            the default value is 3, and a value of -1 indicates
            infinite retries.
            See :ref:`task fault tolerance <fault-tolerance-tasks>` for more details.
        allow_out_of_order_execution: Only for *actors*. Whether Ray executes actor
            tasks out of order. If you're using multi-threaded (``max_concurrency > 1``)
            or async actors, you can't set this to False. Defaults to True if you're
            using multi-threaded or async actors, and False otherwise. Actor task
            retries are always executed out of order.
        runtime_env (Dict[str, Any]): Specifies the runtime environment for
            this actor or task and its children. See
            :ref:`runtime-environments` for detailed documentation.
        retry_exceptions: Only for *remote functions*. This specifies whether
            application-level errors should be retried up to max_retries times.
            This can be a boolean or a list of exceptions that should be retried.
            See :ref:`task fault tolerance <fault-tolerance-tasks>` for more details.
        scheduling_strategy: Strategy about how to
            schedule a remote function or actor. Possible values are
            None: ray will figure out the scheduling strategy to use, it
            will either be the PlacementGroupSchedulingStrategy using parent's
            placement group if parent has one and has
            placement_group_capture_child_tasks set to true,
            or "DEFAULT";
            "DEFAULT": default hybrid scheduling;
            "SPREAD": best effort spread scheduling;
            `PlacementGroupSchedulingStrategy`:
            placement group based scheduling;
            `NodeAffinitySchedulingStrategy`:
            node id based affinity scheduling.
            See :ref:`Ray scheduling strategies <ray-scheduling-strategies>`
            for more details.
        _labels: The key-value labels of a task or actor.
    rK   r   )rh   )r  callabler  r    remote_args_error_stringr?  r@  )r\  r]  s     rg   rw   rw     s    F 4yyA~~#f++**xQ/@/@* DGR(((t99>>c&kkAooo/?/Xoo-\6::::ri   rd   )F)r  rc  )atexitr  r?  r1  r  rF  r%  r  r  r!  r  r  rs  r  abcr   r   collections.abcr   
contextlibr   dataclassesr   r	   typingr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   urllib.parser   torchrX  r  ray._private.noderay._private.parameterray._private.profilingr  r  rc  r(  ray._private.serializationr  ray._private.servicesr$  ray._private.stateray._private.worker	ray.actorray.cloudpicklecloudpickler  ray.job_configray.remote_functionr   r   r   r   ray._commonr    ray._common.constantsr!   ray._common.utilsr"   6ray._private.authentication.authentication_token_setupr#   ray._private.client_mode_hookr$   ray._private.custom_typesr%   ray._private.function_managerr&   ray._private.inspect_utilr'   ray._private.ray_loggingr(   r)   r*   r+   'ray._private.ray_logging.logging_configr,   &ray._private.resource_isolation_configr-   "ray._private.runtime_env.constantsr.   #ray._private.runtime_env.py_modulesr/   #ray._private.runtime_env.setup_hookr0   $ray._private.runtime_env.working_dirr1   ray._private.utilsr2   ray._rayletr3   r4   r5   r6   ray.exceptionsr7   r8   r9   r:   r7  r;   !ray.experimental.compiled_dag_refr<   ray.experimental.internal_kvr=   r>   r?   r@   ray.experimental.tqdm_rayrA   ray.runtime_env.runtime_envrB   ray.util.annotationsrC   rD   rE   ray.util.debugrF   ray.util.scheduling_strategiesrG   ray.util.tracing.tracing_helperrH   ray.widgetsrI   ray.widgets.utilrJ   rN  r  rC  r  r  r&  rj   r  rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   objectr\   r  r]   r^   rn   r}   r   r   r   r   r   r   r   r   r   r  r  r  r.  r[  r_  r  r8  rd  ri  ro  rl  rz  r  r  rm  rT  r  DEFAULT_DASHBOARD_IPLOGGER_LEVELrU  r3  rb  rP  register
excepthookr  r  r  r  r  r  r  r  r  r  r  r  r  r  r8  r  rb  rm  r  rB  r  r  r  r  r  r  rg  r  r  rw   r  r  rX  re   ri   rg   <module>r&     s             				   				 



            ' ' ' ' ' ' ' ' # # # # # # % % % % % % ! ! ! ! ! !                                           & " ! ! ! ! ! LLL  



         * * * * * * * * * 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 ( ( ( ( ( ( ( ( (                                 3 3 3 3 3 3 3 3 3 3 3 3 ( ( ( ( ( ( L L L L L L ( ( ( ( ( (      ; : : : : : 9 9 9 9 9 9 > > > > > > / / / / / /            B A A A A A J J J J J J J J J J J J K K K K K K      N M M M M M 2 2 2 2 2 2         
 !           W W W W W W W W W W W W % % % % % % < < < < < <            5 4 4 4 4 4 : : : : : : D D D D D D D D D D # # # # # # K K K K K K ? ? ? ? ? ?             / / / / / /
  
 
	8	$	$ GCLLWT]]WT]]WT]]WT]]WT]]WT]]WT]]WT]]WT]]WT]]GCLL
')

 	3    WT&&&       
    :wqz       j'!R%.   "    j'!R)"4   &    j'!RR-"8   *    j'!RR*;"<   .    j'!RRR*?"@   2    j'!RRR*C"D   6    j'!RRRR*G"H   :    j'!RRRR*K"L   >         j'!RRRRR*O"P      F" " " " "j'!RRRRRQS*S"T " " "L	    {
0 {
0 {
0 {
0 {
0 {
0 {
0 {
0| ,IO--  X     U49d3i/0     ( M  4 4	 4& GHHH& & IH&$8C= S    [ [ [ [ [G [ [ [ [| 0 0 0 0 0g 0 0 0f   S$sCx.);?	#s(^   6 !@
M #",0'+)- %(,'<$(-1"&3$(.2$(#AE&++/,0/@
M @
M @
Mc]@
M sm@
M sm	@
M
 S%Z()@
M T#s(^$@
M "#@
M @
M @
M  ~@
M @
M SM@
M +@
M @
M @
M  SM!@
M" ]+#@
M$ D>%@
M& }'@
M( %S#X <=>)@
M*  $+@
M, "%-@
M. %SM/@
M2 3@
M @
M @
M  @
MH   :* :*4 :* :* :*   :*z $    N ' ' '  #6D 6 6 6 6* TY[[" 5LDI 5L(3- 5L 5L 5L 5LpS    0   BG` `
sCx.`&)`:>` ` ` `F
 
 
$ $ $
& & &
.4 .4 .4b 7 7 7 7  7  
04#"'!##%!d" d"d" 	d" !)d" }d" d" d"  d" d" d" !d"  !d" d" d" d"N	(# (# (# (#V 1 1 1 H Hs H H# H H H H8 $)   
IM  )7?	!W   
 
KO  +9A%	#Y   
 
CG   ^ % A    
 
IM  .)7?	#Y   
 
CG   ^ % C    
   $'+N N N!" 	"N e_N  }N 3S	>N N N  Nb  15+	< < << ,-< 	<
 < < <  <@ %* !  #L( L( L(i);;<=L( L( e_	L(
 L( y,,	-.y,,	-.0L( L( L(  L(^ L LC LHSM L=T L L L  LB ?C C C C' C C C C  C:  	HJ HJ HJ?@HJ HJ 	HJ
 
HJ HJ HJ  HJV       ,? ? ? ? ?h ? ? ?D 
Q JqM    
 
xA +?+B    
 
xa( _QU-C    
 
xR!, B1K    
 
xRa0 _QBPR]5S    
 
"b"b)1,-QBB&'   
 
"b"b"-q01QBB*+   
 
"b"b"b1145QBBB./   
 
"b"b"b"5q89QBBB23   
 
"b"b"b"b91<=QBBBB67   
 
"b"b"b"b"=q@AQBBBB:;   
 
"b"b"b"b"bA1DEQBBBBB>?   
 
 5>"+"+"+% )!% "+& 	%..7%  sGK001 CJ CJ	
 CJ  #u*     c3h  gi '("35UU" cN#$ DcN+%& '   
. g;
3-sy/CCDg; g; g; g; g; g;ri   