
    &`idg                     J   d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
c mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/m0Z0  ej1        e2          Z3dZ4dZ5 G d d          Z6dS )    N)ThreadPoolExecutor)islice)ListOptional)NodeID)get_or_create_event_loop)chrome_tracing_dump)env_integer)	do_filter)compose_state_message)
RuntimeEnv)RAY_MAX_LIMIT_FROM_API_SERVER
ActorStateActorSummariesJobStateListApiOptionsListApiResponse	NodeStateObjectStateObjectSummariesPlacementGroupStateRuntimeEnvStateStateSummarySummaryApiOptionsSummaryApiResponse	TaskStateTaskSummariesWorkerStateprotobuf_message_to_dictprotobuf_to_task_state_dict)DataSourceUnavailableStateDataSourceClientzFailed to query data from GCS. It is due to (1) GCS is unexpectedly failed. (2) GCS is overloaded. (3) There's an unexpected network issue. Please check the gcs_server.out log to find the root cause.a  Failed to query data from {type}. Queried {total} {type} and {network_failures} {type} failed to reply. It is due to (1) {type} is unexpectedly failed. (2) {type} is overloaded. (3) There's an unexpected network issue. Please check the {log_command} to find the root cause.c                      e Zd ZdZdedefdZed             Zde	de
fdZde	de
fd	Zde	de
fd
Zde	de
fdZde	de
fdZde	de
fdZde	de
fdZde	de
fdZdedefdZdedefdZdedefdZdee         dee         fdZdS )StateAPIManagerzZA class to query states from data source, caches, and post-processes
    the entries.
    state_data_source_clientthread_pool_executorc                 "    || _         || _        d S N)_client_thread_pool_executor)selfr%   r&   s      r/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/dashboard/state_aggregator.py__init__zStateAPIManager.__init__B   s    
 0%9"""    c                     | j         S r(   )r)   )r+   s    r,   data_source_clientz"StateAPIManager.data_source_clientJ   s
    |r.   optionreturnc                  K   	 | j                             j        j                   d{V }n"# t          $ r t	          t
                    w xY wdt          ffd}t                                          | j	        ||           d{V S )zList all actor information from the cluster.

        Returns:
            {actor_id -> actor_data_in_dict}
            actor_data_in_dict's schema is in ActorState

        timeoutfiltersNr2   c                    g }| j         D ]*}t          |g d          }|                    |           +t          |          | j        z   }t          |j        t          j                  }t          |          }|	                    d            t          t          |j                            }t          || j        ||          S )N)actor_idowner_idjob_idnode_idplacement_group_idmessagefields_to_decodec                     | d         S )Nr8    entrys    r,   <lambda>z@StateAPIManager.list_actors.<locals>.transform.<locals>.<lambda>w   s    %
*; r.   keyresulttotalnum_after_truncationnum_filtered)actor_table_datar   appendlenrK   r   r6   r   detailsortlistr   limitr   rI   replyrH   r>   datarJ   rK   r1   s         r,   	transformz.StateAPIManager.list_actors.<locals>.transform]   s    F 1 $ $ 0#& & &	 	 	 d#####&v;;1C#C vv~z6=QQFv;;L KK;;K<<<&6677F"k%9)	   r.   )
r)   get_all_actor_infor5   r6   r!   GCS_QUERY_FAILURE_WARNINGr   r   run_in_executorr*   r+   r1   rT   rV   s    `  r,   list_actorszStateAPIManager.list_actorsN   s      	C,99 :        EE % 	C 	C 	C'(ABBB	C!	 !	 !	 !	 !	 !	 !	F .//??&	5
 
 
 
 
 
 
 
 	
	   ,2 Ac                  K   	 | j                             j                   d{V }n"# t          $ r t          t                    w xY wdt
          ffd}t                                          | j        ||           d{V S )zList all placement group information from the cluster.

        Returns:
            {pg_id -> pg_data_in_dict}
            pg_data_in_dict's schema is in PlacementGroupState
        r5   Nr2   c                    g }| j         D ]*}t          |g d          }|                    |           +t          |          }t	          |j        t          j                  }t          |          }|                    d            t          t          t          |j                            | j        ||          S )N)r<   creator_job_idr;   r=   c                     | d         S )Nr<   rA   rB   s    r,   rD   zJStateAPIManager.list_placement_groups.<locals>.transform.<locals>.<lambda>   s    %0D*E r.   rE   rG   )placement_group_table_datar   rM   rN   r   r6   r   rO   rP   r   rQ   r   rR   rI   rS   s         r,   rV   z8StateAPIManager.list_placement_groups.<locals>.transform   s    F ; 	$ 	$/#& & &   d#####&v;; (;V] F v;;LKKEEKFFF"F66<8899k%9)	   r.   )	r)   get_all_placement_group_infor5   r!   rX   r   r   rY   r*   rZ   s    `  r,   list_placement_groupsz%StateAPIManager.list_placement_groups   s      	C,CC D        EE % 	C 	C 	C'(ABBB	C	 	 	 	 	 	 	6 .//??&	5
 
 
 
 
 
 
 
 	
	   &, Ac                  K   	 | j                             j        j                   d{V }n"# t          $ r t	          t
                    w xY wdt          ffd}t                                          | j	        ||           d{V S )zList all node information from the cluster.

        Returns:
            {node_id -> node_data_in_dict}
            node_data_in_dict's schema is in NodeState
        r4   Nr2   c                    g }| j         D ]}t          |dg          }|d         |d<   t          |d                   |d<   t          |d                   |d<   |                    di           }t	          |                    dd           |                    d	d                     |d
<   |                    |           t          |          | j        z   }t          |j	        t          j                  }t          |          }|                    d            t          t          |j                            }t!          || j        ||          S )Nr;   r=   node_manager_addressnode_ipstart_time_msend_time_ms
death_inforeasonreason_messagestate_messagec                     | d         S )Nr;   rA   rB   s    r,   rD   z?StateAPIManager.list_nodes.<locals>.transform.<locals>.<lambda>       %	*: r.   rE   rG   )node_info_listr   intgetr   rM   rN   rK   r   r6   r   rO   rP   rQ   r   rR   r   rI   )rT   rH   r>   rU   rl   rJ   rK   r1   s          r,   rV   z-StateAPIManager.list_nodes.<locals>.transform   sc   F / $ $/#yk   #''=">Y(+D,A(B(B_%&)$}*=&>&>]#!XXlB77
(=NN8T22NN#3T::) )_%
 d#####&v;;1C#C vv~y&-PPFv;;L KK::K;;;&6677F"k%9)	   r.   )
r)   get_all_node_infor5   r6   r!   rX   r   r   rY   r*   rZ   s    `  r,   
list_nodeszStateAPIManager.list_nodes   s      	C,88 9        EE % 	C 	C 	C'(ABBB	C	 	 	 	 	 	 	> .//??&	5
 
 
 
 
 
 
 
 	
r\   c                  K   	 | j                             j        j                   d{V }n"# t          $ r t	          t
                    w xY wdt          ffd}t                                          | j	        ||           d{V S )zList all worker information from the cluster.

        Returns:
            {worker_id -> worker_data_in_dict}
            worker_data_in_dict's schema is in WorkerState
        r4   Nr2   c                    g }| j         D ]}t          |ddg          }|d         d         |d<   |d         d         |d<   |d         d         |d<   t          |d                   |d<   t          |d                   |d<   t          |d	                   |d	<   t          |d
                   |d
<   |                    |           t	          |          | j        z   }t          |j        t          j	                  }t	          |          }|
                    d            t          t          |j                            }t          || j        ||          S )N	worker_idr;   r=   worker_address
ip_addressiprj   rk   worker_launch_time_msworker_launched_time_msc                     | d         S )Nry   rA   rB   s    r,   rD   zAStateAPIManager.list_workers.<locals>.transform.<locals>.<lambda>      %*< r.   rE   rG   )worker_table_datar   rs   rM   rN   rK   r   r6   r   rO   rP   rQ   r   rR   r   rI   rS   s         r,   rV   z/StateAPIManager.list_workers.<locals>.transform   sx   F 2 $ $/#{I6N   %))9$:;$G[!"&'7"8"CY!"23LAT
(+D,A(B(B_%&)$}*=&>&>]#03D9P4Q0R0R,-25d;T6U2V2V./d#####&v;;1C#C vv~{FMRRFv;;LKK<<K===&6677F"k%9)	   r.   )
r)   get_all_worker_infor5   r6   r!   rX   r   r   rY   r*   rZ   s    `  r,   list_workerszStateAPIManager.list_workers   s      	C,:: ;        EE % 	C 	C 	C'(ABBB	C	 	 	 	 	 	 	: .//??&	5
 
 
 
 
 
 
 
 	
r\   c                  K   	 | j                             j                   d {V }n"# t          $ r t          t                    w xY wdt
          ffd}t                                          | j        ||           d {V S )Nr^   r2   c                 2   d | D             }t          |          }t          |j        t          j                  }t          |          }|                    d            t          t          |j                            }t          ||||          S )Nc                 6    g | ]}|                                 S rA   )dict).0jobs     r,   
<listcomp>z@StateAPIManager.list_jobs.<locals>.transform.<locals>.<listcomp>  s     222Schhjj222r.   c                     | d         pdS )Nr:    rA   rB   s    r,   rD   z>StateAPIManager.list_jobs.<locals>.transform.<locals>.<lambda>  s    %/*?R r.   rE   rG   )
rN   r   r6   r   rO   rP   rQ   r   rR   r   )rT   rH   rI   rK   r1   s       r,   rV   z,StateAPIManager.list_jobs.<locals>.transform  s    22E222FKKEvv~xOOFv;;LKK??K@@@&6677F"%*)	   r.   )	r)   get_job_infor5   r!   rX   r   r   rY   r*   rZ   s    `  r,   	list_jobszStateAPIManager.list_jobs  s      	C,33FN3KKKKKKKKEE$ 	C 	C 	C'(ABBB	C	 	 	 	 	 	 	 .//??&	5
 
 
 
 
 
 
 
 	
re   c                  K   	 | j                             j        j        j                   d{V }n"# t
          $ r t          t                    w xY wdt          ffd}|j        j	        dk    rt          g ddd|j        j
        g          S t                                          | j        ||           d{V S )zList all task information from the cluster.

        Returns:
            {task_id -> task_data_in_dict}
            task_data_in_dict's schema is in TaskState
        )r5   r6   exclude_driverNr2   c                 j   d | j         D             }t          |          }t          |          | j        z   }t          |j        t
          j                  }t          |          }|                    d            t          t          |j
                            }t          ||||          S )z
            Transforms from proto to dict, applies filters, sorts, and truncates.
            This function is executed in a separate thread.
            c                 ,    g | ]}t          |          S rA   )r    )r   r>   s     r,   r   zAStateAPIManager.list_tasks.<locals>.transform.<locals>.<listcomp>?  s.       9@+G44  r.   c                     | d         S )Ntask_idrA   rB   s    r,   rD   z?StateAPIManager.list_tasks.<locals>.transform.<locals>.<lambda>M  rq   r.   rE   rG   )events_by_taskrN   num_status_task_events_droppedr   r6   r   rO   rP   rQ   r   rR   r   )rT   rH   rJ   	num_totalrK   r1   s        r,   rV   z-StateAPIManager.list_tasks.<locals>.transform:  s    
 DIDX  F $'v;; Fe&JJI vv~y&-PPFv;;LKK::K;;;&6677F #%9)	   r.   r   )rH   rI   rJ   rK   warnings)r)   get_all_task_infor5   r6   r   r!   rX   r   statuscoder>   r   rY   r*   rZ   s    `  r,   
list_taskszStateAPIManager.list_tasks*  s6     	C,88%4 9        EE
 % 	C 	C 	C'(ABBB	C	 	 	 	 	 	 	> <!!"%&,./    .//??&	5
 
 
 
 
 
 
 
 	
s	   28 Ac                ,   K    j                             j        ddg           d{V } fd|j        D             t	          j        ddi d{V }dt          ffd}t                                           j	        ||           d{V S )	zList all object information from the cluster.

        Returns:
            {object_id -> object_data_in_dict}
            object_data_in_dict's schema is in ObjectState
        Nstate=ALIVEr5   rR   r6   c                 h    g | ].}j                             |j        |j        j                   /S r^   )r)   get_object_inforh   node_manager_portr5   r   	node_infor1   r+   s     r,   r   z0StateAPIManager.list_objects.<locals>.<listcomp>r  sS     
 
 
  L((.+ )  
 
 
r.   return_exceptionsTr2   c           	      n   d}g }d}| D ]n}t          |t                    r|dz  }t          |t                    r|||j        z  }|j        D ](}|                    t          |dgd                     )od }t                    dk    rX|dk    rRt          	                    dt                    |d          }|t                    k    rt          |          d	| }g }t          j        |          }	|	j        D ]}}
|
                                }|d
         |d<   |d
= |d         |d<   |d= |d                                         |d<   |d         dk    rdn|d         |d<   |                    |           ~g }t          dd          }|s|                    d           t          |          }t!          |j        t$          j                  }t          |          }|                    d            t+          t-          |j                            }t1          ||||||          S )Nr      	object_idF)r>   r?   preserving_proto_field_namerayletz
raylet.outtyperI   network_failureslog_command1The returned data may contain incomplete result. 
object_refnode_ip_addressr|   r   task_status-NILRAY_record_ref_creation_siteszCallsite is not being recorded. To record callsite information for each ObjectRef created, set env variable RAY_record_ref_creation_sites=1 during `ray start` and `ray.init`.c                     | d         S )Nr   rA   rB   s    r,   rD   zAStateAPIManager.list_objects.<locals>.transform.<locals>.<lambda>  r   r.   rE   )rH   partial_failure_warningrI   rJ   rK   r   )
isinstancer!   	ExceptionrI   core_workers_statsrM   r   rN   NODE_QUERY_FAILURE_WARNINGformatmemory_utilsconstruct_memory_tabletableas_dictupperr
   r   r6   r   rO   rP   rQ   r   rR   r   )repliesunresponsive_nodesworker_statstotal_objectsrT   core_worker_statr   warning_msgrH   memory_tablerC   rU   callsite_warningcallsite_enabledrJ   rK   r1   taskss                   r,   rV   z/StateAPIManager.list_objects.<locals>.transform  s   !"LM   e%:;;  &!+&y11  K,(-(@  $
 !''0$4.9]8=      '+#5zzA~~"4q"8"88??!e**%7 ,	 @   &U33/<<<UUU ( F'>|LLL%+ $ $}} %)$6[!&!"34T
*+#F|1133V!-0C77EET-=P ]# d####  "*+JANN#  ''&   $'v;; vv~{FMRRFv;;LKK<<K===&6677F"(?#%9))   r.   
r)   ru   r5   rr   asynciogatherr   r   rY   r*   )r+   r1   all_node_info_replyr   rV   r   s   ``   @r,   list_objectszStateAPIManager.list_objectsf  s>      %)L$B$BN,- %C %
 %
 
 
 
 
 
 


 
 
 
 
 1?
 
 
  
"
 
 
 
 
 
 
 

P	/ P	 P	 P	 P	 P	 P	 P	d .//??&	7
 
 
 
 
 
 
 
 	
r.   c                H   K    j                             j        ddg           d{V }d |j        D              fdD             t	          j        ddi d{V }dt          ffd	}t                                           j	        ||           d{V S )
ao  List all runtime env information from the cluster.

        Returns:
            A list of runtime env information in the cluster.
            The schema of returned "dict" is equivalent to the
            `RuntimeEnvState` protobuf message.
            We don't have id -> data mapping like other API because runtime env
            doesn't have unique ids.
        Nr   r   c                      g | ]}|j         	|S r(   )runtime_env_agent_port)r   r   s     r,   r   z5StateAPIManager.list_runtime_envs.<locals>.<listcomp>  s+     
 
 
/; ;;;r.   c                 h    g | ].}j                             |j        |j        j                   /S r   )r)   get_runtime_envs_inforh   r   r5   r   s     r,   r   z5StateAPIManager.list_runtime_envs.<locals>.<listcomp>  sS     
 
 
  L...0 /  
 
 
r.   r   Tr2   c                    g }d}d}t          |           D ]\  }}t          |t                    r|dz  } t          |t                    r|||j        z  }|j        }|D ]}t          |g           }t          j        |d                   	                                |d<   t          |j                                                  |d<   |                    |           d }	t                    dk    rX|dk    rRt                              dt                    |d          }
|t                    k    rt          |
          d	|
 }	t          |          }t#          |j        t&          j                  }t          |          }d
 }|                    |d           t-          t/          |j                            }t3          ||	|||          S )Nr   r   r=   runtime_envr;   agentzdashboard_agent.logr   r   c                     d| vrt          d          S | d         t          d          S t          | d                   S )Ncreation_time_msinf)floatrB   s    r,   	sort_funczGStateAPIManager.list_runtime_envs.<locals>.transform.<locals>.sort_func   sG     &U22 <<'-.6 <<' '9!:;;;r.   T)rF   reverse)rH   r   rI   rJ   rK   )zipr   r!   r   rI   runtime_env_statesr   r   deserializeto_dictr   r;   hexrM   rN   r   r   r   r6   r   rO   rP   rQ   r   rR   r   )r   rH   r   total_runtime_envsr   rT   statesr   rU   r   r   rJ   rK   r   
node_infosr1   r   s                 r,   rV   z4StateAPIManager.list_runtime_envs.<locals>.transform  s   F!"!"$'
G$<$< ( ( 	5e%:;;  &!+&y11  K"ek1"1# ( (E3ETVWWWD*4*@]++ +gii ' '-Y->&?&?&C&C&E&EDOMM$''''( '+#5zzA~~"4q"8"88?? e**%7 5	 @   &U33/<<<UUU ( $'v;; vv~VVFv;;L	< 	< 	< KKItK444&6677F"(?(%9)   r.   r   )r+   r1   live_node_info_replyr   rV   r   r   s   ``   @@r,   list_runtime_envsz!StateAPIManager.list_runtime_envs  sZ      &*\%C%CN,- &D &
 &
  
  
  
  
  
  


 
1@
 
 



 
 
 
 
 (
 
 
  
"
 
 
 
 
 
 
 

;	/ ;	 ;	 ;	 ;	 ;	 ;	 ;	 ;	z .//??&	7
 
 
 
 
 
 
 
 	
r.   c           	        K   |j         pd}|dvrt          d          |                     t          |j        t
          |j        |dk                         d {V }|dk    rt          j        |j	                  }nV| 
                    t          |j        t
          d	                     d {V }t          j        |j	        |j	        
          }t          d|i          }|j        }|j        |j        z   |j        z   |j        k     r|pg }|                    d           t'          |j        ||j        ||j        |j                  S )N	func_name)r   lineagez3summary_by must be one of "func_name" or "lineage".r   )r5   rR   r6   rO   r1   )r   T)r5   rR   rO   )r   actorsclusternode_id_to_summaryzfThere is missing data in this aggregation. Possibly due to task data being evicted to preserve memory.rI   rH   r   r   rJ   rK   )
summary_by
ValueErrorr   r   r5   r   r6   r   to_summary_by_func_namerH   r[   to_summary_by_lineager   r   total_actor_scheduledtotal_actor_taskstotal_tasksrK   rM   r   rI   r   rJ   )r+   r1   r   rH   summary_resultsr   summaryr   s           r,   summarize_taskszStateAPIManager.summarize_tasks9  s     &5+
555RSSS !3!Y.	   ' 
 
 
 
 
 
 
 
 $$+C&-XXXOO  ++%"N7   ,        F ,AmFM  O 9o2NOOO?1/0)* !" "
  ~2HOON   ",$*$B!'!<,
 
 
 	
r.   c                 ,  K   |                      t          |j        t          |j                             d {V }t          dt          j        |j                  i          }t          |j
        ||j        |j        |j        |j                  S )Nr   r   r   )r   r   r   )r[   r   r5   r   r6   r   r   
to_summaryrH   r   rI   r   r   rJ   rK   r+   r1   rH   r   s       r,   summarize_actorsz StateAPIManager.summarize_actorsl  s      ''!3   ( 
 
 
 
 
 
 
 
 >4FMJJJ 
 
 

 ",$*$B_!'!<,
 
 
 	
r.   c                 ,  K   |                      t          |j        t          |j                             d {V }t          dt          j        |j                  i          }t          |j
        ||j        |j        |j        |j                  S )Nr   r   r   )objectsr   r   )r   r   r5   r   r6   r   r   r  rH   r   rI   r   r   rJ   rK   r  s       r,   summarize_objectsz!StateAPIManager.summarize_objects  s      ((!3   ) 
 
 
 
 
 
 
 
 ?5fmLLL 
 
 

 ",$*$B_!'!<,
 
 
 	
r.   r:   c                    K   |rdd|fgnd }|                      t          d|d                     d {V }t          |j                  S )Nr:   r   Ti'  )rO   r6   rR   r   )r   r   r	   rH   )r+   r:   r6   rH   s       r,   generate_task_timelinez&StateAPIManager.generate_task_timeline  sy      /5?Hc6*++4!weLLL ' 
 
 
 
 
 
 
 
 #6=111r.   N)__name__
__module____qualname____doc__r"   r   r-   propertyr0   r   r   r[   rd   rv   r   r   r   r   r   r   r   r   r  r  r   strr   r   r  rA   r.   r,   r$   r$   =   s        :"7: 1: : : :   X4
> 4
o 4
 4
 4
 4
l+
^ +
 +
 +
 +
 +
Z/
. /
_ /
 /
 /
 /
b.
N .
 .
 .
 .
 .
`
 
O 
 
 
 
0:
. :
_ :
 :
 :
 :
xn
N n
 n
 n
 n
 n
`a
 a
O a
 a
 a
 a
F1
,= 1
BT 1
 1
 1
 1
f
-> 
CU 
 
 
 
.
.? 
DV 
 
 
 
.28C= 2T$Z 2 2 2 2 2 2r.   r$   )7r   loggingconcurrent.futuresr   	itertoolsr   typingr   r   ray.dashboard.memory_utils	dashboardr   rayr   ray._common.utilsr   ray._private.profilingr	   ray._private.ray_constantsr
   ray.dashboard.state_api_utilsr   ray.dashboard.utilsr   ray.runtime_envr   ray.util.state.commonr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    ray.util.state.state_managerr!   r"   	getLoggerr	  loggerrX   r   r$   rA   r.   r,   <module>r      s<     1 1 1 1 1 1       ! ! ! ! ! ! ! ! 1 1 1 1 1 1 1 1 1       6 6 6 6 6 6 6 6 6 6 6 6 2 2 2 2 2 2 3 3 3 3 3 3 5 5 5 5 5 5 & & & & & &                                         * V U U U U U U U		8	$	$B , b	2 b	2 b	2 b	2 b	2 b	2 b	2 b	2 b	2 b	2r.   