
    &`i
y                     d   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZmZ d dlZd dlZd dlZd dlmc mZ d dlmc mZ d dlm Z  d dl!m"Z" d dl#m$Z$ d d	l%m&Z&m'Z'm(Z( d d
l)m*Z* d dl+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1m2Z2 d dl3m4Z4m5Z5m6Z6 d dl7m8Z8m9Z9m:Z: d dl;m<Z<m=Z= d dl>m?Z?m@Z@ d dlAmBZB d dlCmDZD d dlEmFZG d dlmHZH  ejI        eJ          ZK e.dd          ZL eMd ejN        O                                          ZPdZQdZRde4jS        deTfdZUd ZV G d deD          ZWdeXdeTfdZYdS )     N)defaultdictdeque)ThreadPoolExecutor)chain)AnyAsyncGeneratorDictIterableListOptionalSet)get_or_create_event_loop)ray_constants)split)GcsAioActorSubscriberGcsAioNodeInfoSubscriberGcsAioResourceUsageSubscriber)init_grpc_channel)DEBUG_AUTOSCALING_ERRORDEBUG_AUTOSCALING_STATUSenv_integer)LoadMetricsSummaryget_per_node_breakdown_as_dictparse_usage)gcs_pb2node_manager_pb2node_manager_pb2_grpc)DASHBOARD_AGENT_ADDR_IP_PREFIX#DASHBOARD_AGENT_ADDR_NODE_ID_PREFIXGCS_RPC_TIMEOUT_SECONDS)actor_constsnode_consts)DataOrganizer
DataSource)StatsPayload)SubprocessModule)SubprocessRouteTable)async_loop_forever'RAY_DASHBOARD_NODE_HEAD_TPE_MAX_WORKERS   )	stateaddressnumRestarts	timestamppid
exitDetail	startTimeendTimereprNamemessagereturnc                 2    t          j        | dhd          S )NnodeIdT$always_print_fields_with_no_presence)dashboard_utilsmessage_to_dict)r4   s    x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/dashboard/modules/node/node_head.py_gcs_node_info_to_dictr=   N   s%    *($       c                    t          j        | h dd          }h dfd|                                D             }|d         |d<   d}d	|v rS|d	         }d
|v r|d
         d         }n8d|v r|d         d         }n%d|v r|d         d         }nd|v r|d         d         }||d<   t          |d                   |d<   t          |d                   |d<   t	          | j                  |d<   t	          | j                  |d<   |S )N>
   jobIdr7   taskIdactorIdcallerIdparentIdworkerIdparentTaskIdsourceActorIdplacementGroupIdTr8   >   r/   namer@   r+   rB   r,   r2   callSiter3   	classNamer1   r.   r-   labelSelectorrH   c                 $    i | ]\  }}|v 	||S  rN   ).0kvfieldss      r<   
<dictcomp>z-_actor_table_data_to_dict.<locals>.<dictcomp>z   s$    LLLfq!VQr>   rK   
actorClass-
deathCauseactorDiedErrorContexterrorMessageruntimeEnvFailedContextactorUnschedulableContextcreationTaskFailureContextformattedExceptionStringr0   r1   r2   requiredResourcesrL   )r:   r;   itemsintdictrequired_resourceslabel_selector)r4   orig_messagelight_messageexit_detailcontextrR   s        @r<   _actor_table_data_to_dictrg   T   ss   "2	
 	
 	
 .2  L(  F" MLLL(:(:(<(<LLLM".{";M,K|##|,"g--!"9:>JKK&'11!";<^LKK(G33!"=>~NKK)W44!">?*K #.M,!$];%?!@!@M+"=#;<<M))-g.H)I)IM%&%)'*@%A%AM/"r>   c                       e Zd Z fdZd Zdeedf         fdZdefdZd Z	defd	Z
 ej        d
          ej        dej        j        fd                        Z ej        d          ej        dej        j        fd                        Z eej                  d             Zd Zd Zdedeeeeef         f         fdZdedeeef         fdZdeeef         fdZ d Z! ej        d          ej        dej        j        fd                        Z" ej        d          ej        dej        j        fd                        Z# ej        d          dej        j        fd            Z$ fdZ% xZ&S )NodeHeadc                     t                      j        |i | i | _        d| _        t	          j                    | _        d | _        d | _        t                      | _	        t          t          d          | _        d | _        t                      | _        t                      | _        t          dd          | _        t%                      | _        d S )NFnode_head_node_executor)max_workersthread_name_prefixr*   node_head_actor_executor)super__init___stubs_collect_memory_infotime_module_start_time_head_node_registration_time_s_registered_head_node_idr   _dead_node_queuer   r)   _node_executor_gcs_actor_channel_subscriber_destroyed_actors_queuer   _loop_actor_executorset_background_tasks)selfargskwargs	__class__s      r<   rp   zNodeHead.__init__   s    $)&)))$)! #')++ /3+(,% %0?8
 
 

 .2*',ww$ .//
  2.H 
  
  
 58EEr>   c                 x    | j         t          t          j                  t	          j                    | j        z
  dS )N)head_node_registration_time_sregistered_nodesmodule_lifetime_s)ru   lenr$   nodesrs   rt   )r   s    r<   get_internal_stateszNodeHead.get_internal_states   s6    -1-P #J$4 5 5!%t/F!F
 
 	
r>   r5   Nc                  K   t          | j                  }|                                 d{V  | j                            d           d{V }dt
          t          j                 dt          t                   fd}| j
                            | j        ||                                           d{V }|D ]}|W V  	 	 |                    t          j                   d{V }|rt#          | \  }}ng }| j
                            | j        ||           d{V }	|	D ]}|W V  n*# t$          $ r t&                              d	           Y nw xY w)
z
        Yields the initial state of all nodes, then yields the updated state of nodes.

        It makes GetAllNodeInfo call only once after the subscription is done, to get
        the initial state of the nodes.
        r,   Ntimeoutmessagesr5   c                     d | D             S )Nc                 ,    g | ]}t          |          S rN   )r=   )rO   ms     r<   
<listcomp>zRNodeHead._subscribe_for_node_updates.<locals>._convert_to_dict.<locals>.<listcomp>   s!    @@@!*1--@@@r>   rN   )r   s    r<   _convert_to_dictz>NodeHead._subscribe_for_node_updates.<locals>._convert_to_dict   s    @@x@@@@r>   T
batch_sizezFailed handling updated nodes.)r   gcs_address	subscribe
gcs_clientasync_get_all_node_infor
   r   GcsNodeInfor   r`   r{   run_in_executorrx   valuespollr"   'RAY_DASHBOARD_NODE_SUBSCRIBER_POLL_SIZEzip	Exceptionlogger	exception)
r   
subscriberall_node_infor   all_node_infosnodenode_id_updated_info_tuples_updated_infos_protoupdated_infoss
             r<   _subscribe_for_node_updatesz$NodeHead._subscribe_for_node_updates   s      .d6FGGG
""$$$$$$$$$ #oEEdESSSSSSSS	Ax0C'D 	Ad 	A 	A 	A 	A  $z99  "" 
  
 
 
 
 
 
 
 # 	 	DJJJJJ	CC4>OO*R 5D 5 5 / / / / / /+ / --02M-N*A***,'&*j&@&@'$'' ' ! ! ! ! ! ! *  DJJJJJ C C C  !ABBBBBC'	Cs   A(D. .$EEr   c                    K   |d         }|d         r|d         dk    r j         |k    r j         4t                              d| j                                                     | _         t	          j                     j        z
   _         j                            t          j
        |                                dt          j        t                     d {V  |d         dv sJ |d         dk    }|st           | t           |d	          g} fd
|D             }t!          j        |  d {V   j                            |           t)           j                  t*          j        k    rT j                                        }t0          j                            |d             j                            |d            |t0          j        |<   d                    |d	         t;          |d                             }t          j        }t?          ||d          }tA          j!        |          }	|	 j        |<   d S )Nr7   
isHeadNoder+   ALIVEzaA new head node has become ALIVE. New head node ID: %s, old head node ID: %s, internal states: %sT)	overwrite	namespacer   )r   DEADnodeManagerAddressc                 j    g | ]/}j                             |d t          j        t                    0S )F)del_by_prefixr   r   )r   async_internal_kv_delr   KV_NAMESPACE_DASHBOARDr    rO   keyr   s     r<   r   z)NodeHead._update_node.<locals>.<listcomp>  sQ         55"'+B3	 6    r>   z{}:{}nodeManagerPort)asynchronous)"rv   r   warningr   rs   rt   ru   r   async_internal_kv_putr   KV_HEAD_NODE_ID_KEYencodeKV_NAMESPACE_JOBr    r   r   asynciogatherrw   appendr   r"   MAX_DEAD_NODES_TO_CACHEpopleftr$   r   poprq   formatr_   GLOBAL_GRPC_OPTIONSr   r   NodeManagerServiceStub)
r   r   node_idis_alivekeystasksr,   optionschannelstubs
   `         r<   _update_nodezNodeHead._update_node   s     x.	W((-88,8w1,,..	   -4D)26)++@W2WD/ /771  '8/ 8          G} 11111=G+ 	/ 7AAA1O48L3MOOD       E .%((((((((!((1114())K,OOO/7799 $$Wd333...$(
!..%&D1B,C(D(D
 
  3#GW4HHH$;GDD#Gr>   c                 ^  K   d}|                                  2 3 d{V }|                     |           d{V  | j        si|sgt          j                    | j        z
  t
          j        k    r>t                              dt
          j         d| 	                                            d}6 dS )z
        Subscribe to node updates and update the internal states. If the head node is
        not registered after RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUT, it logs a
        warning only once.
        FNz'Head node is not registered even after zb seconds. The API server might not work correctly. Please report a Github issue. Internal states :T)
r   r   ru   rs   rt   r"   ,RAY_DASHBOARD_HEAD_NODE_REGISTRATION_TIMEOUTr   r   r   )r   warning_shownr   s      r<   _update_nodeszNodeHead._update_nodes*  s       ::<< 	) 	) 	) 	) 	) 	) 	)$##D)))))))))6 ) &)t'>>!NO O NN8&S8 8  3355	8 8   %)M! =<<s   B,c                 F   K   ddl m}  |            rddlm} ddlm} 	 t          j                    } j                                         d {V }t          j                    }|	                    | |||z
  |                    }n,# t          $ r t                              d           i cY S w xY wi }t          |j        |j                  D ]L}|j        s
d |j        j        D             }	d	                    t'          |	d
                    ||j        <   M|S t+          j         fdt.          t0          fD               d {V \  }
}|
si S t3          j        |
          }|                    d          }|rt9          di |}t;          |          }||ni S )Nr   )is_autoscaler_v2)Stats)ClusterStatusParser)gcs_request_time_srequest_ts_s)statszError getting cluster statusc                 6    i | ]}|j         |j        |j        fS rN   )resource_nameusedtotal)rO   rs     r<   rS   z8NodeHead.get_nodes_logical_resources.<locals>.<dictcomp>`  s5        Oafag%6  r>   
T)verbosec                 x    g | ]6}j                             |                                d t                    7S )N)r   r   )r   async_internal_kv_getr   r    r   s     r<   r   z8NodeHead.get_nodes_logical_resources.<locals>.<listcomp>l  sR         55JJLLD:Q 6    r>   load_metrics_reportrN   )ray.autoscaler.v2.utilsr   ray.autoscaler.v2.schemar   ray.autoscaler.v2.sdkr   rs   r   async_get_cluster_statusfrom_get_cluster_status_replyr   r   r   r   active_nodes
idle_nodesresource_usageusagejoinr   r   r   r   r   r   jsonloadsgetr   r   )r   r   r   r   req_timecluster_status
reply_timeper_node_resourcesr   
usage_dictstatus_stringerrorstatus_dictlm_summary_dict
lm_summarynode_logical_resourcess   `               r<   get_nodes_logical_resourcesz$NodeHead.get_nodes_logical_resourcesC  sb     <<<<<< !	&666666AAAAAA9;;'+'O'O'Q'Q!Q!Q!Q!Q!Q!Q!Y[[
!4!R!R"%+5+@x   "S " "      !?@@@			 "$n9>;TUU 
 
*  !06  
 4899
D9994 4"4<00 &% (/~   
 -+	  
(
 
"
 
"
 
"
 
"
 
"
 
"
  	Ij//%//*?@@ 	?+>>o>>J!?
!K!K).%%B>s   A*B &B65B6z/nodesc                   K   |j                             d          }|dk    rlt          j                    }|                                 }t          j        ||           d {V \  }}t          j        t          j
        j        d||          S ||                                d                                k    rt                      }t          j                                        D ])}|d         dk    r|                    |d                    *t          j        t          j
        j        d	t%          |          
          S t          j        t          j
        j        d|           S )NviewsummaryzNode summary fetched.)status_coder4   r  r   hostNameListr+   r   nodeManagerHostnamezNode hostname list fetched.)r  r4   host_name_listzUnknown view r  r4   )queryr   r#   get_all_node_summaryr   r   r   dashboard_optional_utilsrest_responser:   HTTPStatusCodeOKlowerr}   r$   r   r   addlistINTERNAL_ERROR)	r   reqr   all_node_summary_tasknodes_logical_resource_taskall_node_summarynodes_logical_resourcesalive_hostnamesr   s	            r<   get_all_nodeszNodeHead.get_all_nodes  s      y}}V$$9$1$F$H$H!*.*J*J*L*L'>En%'B? ? 9 9 9 9 9 955 ,9+:=/('>	    $**,,.2F2F2H2H"H"H!eeO"(//11 E E=G++#''-B(CDDD+9+:=5#O44    ,9+:I...   r>   z/nodes/{node_id}c                    K   |j                             d          }t          j        |           d {V }t	          j        t          j        j        d|          S )Nr   zNode details fetched.r  r4   detail)	
match_infor   r#   get_node_infor	  r
  r:   r  r  )r   r  r   	node_infos       r<   get_nodezNodeHead.get_node  sh       .$$Y//'5g>>>>>>>>	'5'69+
 
 
 	
r>   c           	        K   t          dt          j        dz
            t          | j                                                  }g }g }t          |          D ]\  }\  }}t          j        	                    |          }|d         dk    r4|
                    |           |
                    |                    t          j        | j                                       g }t          |d          D ]H}	t!          j        |	dd	i d {V }
|                    |
           t!          j        d
           d {V  Ifd}t)          |          t)          |          k    s0J dt)          |           d| dt)          |           d|             | j                            | j        |t1          ||                     d {V }|                                D ]\  }}|t          j        |<   d S )N   r*   r+   r   )include_memory_infor   d   return_exceptionsTg?c                 (   i }| D ]
\  }}t          |t          j                  r!t          |t          j                  r|                                t          j        j        k    r
d| d d}n4|                                t          j        j        k    rd| d}nd| d}t          
                    ||           t          |t                    r!t          
                    d| d|           t          j        |          ||<   |S )z:Pure function reorganizing the data into {node_id: stats}.zCannot reach the node, z, after timeout  zI. This node may have been overloaded, terminated, or the network is slow.z$. The node may have been terminated.zError updating node stats of .exc_info)
isinstancer   CancelledErrorgrpcRpcErrorcode
StatusCodeDEADLINE_EXCEEDEDUNAVAILABLEr   r   r   r:   node_stats_to_dict)node_id_response_tuplesnew_node_statsr   responser4   r   s        r<   postprocessz0NodeHead._update_node_stats.<locals>.postprocess  s`   N%<  !h(>?? $-88 }}$/*KKKBg B B 'B B B  
 "DO,GGGAg A A A  
 #M'"L"L"LLL8L<<<<)44 LLBBBBX !     />.P / /N7++ "!r>   z	node_ids(z): z, responses()maxr"   "NODE_STATS_UPDATE_INTERVAL_SECONDSr  rq   r^   	enumerater$   r   r   r   GetNodeStatsr   GetNodeStatsRequestrr   r   r   r   extendsleepr   r{   r   rx   r   
node_stats)r   current_stub_node_id_tuplesnode_idsget_node_stats_tasksr   r   r   r  	responsesget_node_stats_tasks_chunkcurrent_chunk_responsesr4  r2  new_statr   s                 @r<   _update_node_statszNodeHead._update_node_stats  s     aG!KLL
 '+4;+<+<+>+>&?&?#!"+,G"H"H 	 	A"(,,W55I!W,,OOG$$$ ''!!$8,0,E   $	 "      	 +00Dc*J*J 	' 	'&,3N+-"&- - ' ' ' ' ' '#
 4555
 -&&&&&&&&&& 	"  	"  	"  	"  	"H 8}}!
 !
 
 
 
]s8}}]]]]s9~~]]R[]]
 
 
  $z99c(I.F.F 
  
 
 
 
 
 
 
 "0!5!5!7!7 	6 	6GX-5J!'**	6 	6r>   c                   K   t          | j                  }|                                 d{V  	 	 |                                 d{V \  }}|"| j                            | j        t          |           d{V }|                    d          d         }|t          j
        |<   n*# t          $ r t                              d           Y nw xY w)za
        Update DataSource.node_physical_stats by subscribing to the GCS resource usage.
        r   NT:zEError receiving node physical stats from _update_node_physical_stats.)r   r   r   r   r{   r   rx   _parse_node_statsr   r$   node_physical_statsr   r   r   )r   r   r   dataparsed_datar   s         r<   _update_node_physical_statsz$NodeHead._update_node_physical_stats  s%      34;KLLL
""$$$$$$$$$	 #-//"3"3333333	T; %)J$>$>'):D% %       ))C..,:E
.w77     [    !	s   B+ AB+ +$CCc                   K   | j         }t          |          }|                                 d{V  	 	 t                              d           |                                  d{V }|t          _        t          t                    }|
                                D ].\  }}|d         d         }|t          j        k    r|||         |<   /|t          _        t                              dt          |                     nY# t          $ rJ}t                              d|	           t#          j        t          j                   d{V  Y d}~nd}~ww xY w&	 	 |                     |           d{V }	|	
                                D ]\  }
}|                     |
|           t                              d
t          |	           d|j                    n3# t          $ r&}t                              d|	           Y d}~nd}~ww xY w)z
        Processes actor info. First gets all actors from GCS, then subscribes to
        actor updates. For each actor update, updates DataSource.node_actors and
        DataSource.actors.
        r   NTz Getting all actor info from GCS.r,   r7   z Received %d actor info from GCS.z%Error Getting all actor info from GCSr&  zTotal events processed: z, queue size: z%Error processing actor info from GCS.)r   r   r   r   info_get_all_actorsr$   actorsr   r`   r^   r!   NIL_NODE_IDnode_actorsr   r   r   r   r;  )RETRY_GET_ALL_ACTOR_INFO_INTERVAL_SECONDS_poll_updated_actor_table_data_process_updated_actor_tabledebug
queue_size)r   gcs_addractor_channel_subscriberactor_dictsrR  actor_id_bytesupdated_actor_tabler   eupdated_actor_table_entriesactor_ids              r<   _update_actorszNodeHead._update_actors'  s      ##8#J#J#J &00222222222	>???$($8$8$:$:::::::$/
! *$//;F;L;L;N;N S S7N$71)<XFG,":::?RG,^< *5
&>K@P@PQQQ      !HST UUUm J             /	<	VV484W4W,5 5 / / / / / /+ 16688U U '55h@STTTT Is3N/O/O I I#;#FI I   
  V V V  !HST UUUUUUUUV%	Vs2   CD 
EA EEA=G 
H%HHrY  c                    K   |                     d           d {V fd}| j                            | j        |           d {V S )N   r   c                      d  D             S )Nc                 \    i | ])\  }}||                                 t          |          *S Nhexrg   )rO   r[  actor_table_data_messages      r<   rS   zUNodeHead._poll_updated_actor_table_data.<locals>._convert_to_dict.<locals>.<dictcomp>u  sM        =N$<!-	 ""$$&?,' ' .--r>   rN   )batchs   r<   r   zANodeHead._poll_updated_actor_table_data.<locals>._convert_to_dictt  s(      AF	   r>   )r   r{   r   r|   )r   rY  r   ri  s      @r<   rT  z'NodeHead._poll_updated_actor_table_datak  sz       /33s3CCCCCCCC
	 	 	 	 	 Z//0DFVWWWWWWWWWr>   r_  actor_table_datac                    t           j                            |          }|r'|d         dk    rt          D ]}||v r||         ||<   |}|d         }|d         d         }|d         dk    r| j                            |           |t           j        |<   |t          j        k    r6t           j                            |i           }|||<   |t           j        |<   dS dS )zNOTE: This method has to be executed on the event-loop, provided that it
        accesses DataSource data structures (to follow its thread-safety model)r+   DEPENDENCIES_UNREADYrB   r,   r7   r   N)	r$   rP  r   ACTOR_TABLE_STATE_COLUMNSrz   r   r!   rQ  rR  )r   r_  rj  actorrP   r   rR  s          r<   rU  z%NodeHead._process_updated_actor_table  s     !%%h// 	%%g.2HHH. 3 3(((/2E!H$#I."9-h7G$..(//999 '7
(#l...$044WbAAK$4K!.9J"7+++ /.r>   c                    K   | j                             t                     d {V fd}| j                            | j        |           d {V S )Nr   c                  @    d                                   D             S )Nc                 X    i | ]'\  }}|                                 t          |          (S rN   rf  )rO   r_  rj  s      r<   rS   zFNodeHead._get_all_actors.<locals>._convert_to_dict.<locals>.<dictcomp>  sA       .H.  9:J K K  r>   )r^   )rP  s   r<   r   z2NodeHead._get_all_actors.<locals>._convert_to_dict  s,     28,,..   r>   )r   async_get_all_actor_infor    r{   r   r|   )r   r   rP  s     @r<   rO  zNodeHead._get_all_actors  s      ??+ @ 
 
 
 
 
 
 
 
	 	 	 	 	 Z//0DFVWWWWWWWWWr>   c                 "  K   	 	 t          | j                  t          k    r| j                                        }|t          j        v r_t          j                            |          }|d                             d          }|r#|t          j	        k    rt          j
        |         |= t          | j                  t          k    t          j        t                     d {V  n*# t          $ r t                              d           Y nw xY w)NTr,   r7   z&Error cleaning up actor info from GCS.)r   rz   MAX_DESTROYED_ACTORS_TO_CACHEr   r$   rP  r   r   r!   rQ  rR  r   r;  ACTOR_CLEANUP_FREQUENCYr   r   r   )r   r_  rn  r   s       r<   _cleanup_actorszNodeHead._cleanup_actors  s      	K
K$677:WWW#;CCEEH:#444 * 1 5 5h ? ?"'	"2"6"6x"@"@" Jw,2J'J'J * 6w ? I $677:WWW m$;<<<<<<<<<< K K K  !IJJJJJK	Ks   CC% %$DDz/logical/actorsc                    K   d }d|j         v r |j         d                             d          }t          j        |           d {V }t	          j        t          j        j        d|d          S )Nids,	actor_idszAll actors fetched.F)r  r4   rP  convert_google_style)	r  r   r#   get_actor_infosr	  r
  r:   r  r  )r   r  r{  rP  s       r<   get_all_actorszNodeHead.get_all_actors  s       *.	CI	%(..s33I$4yIIIIIIIII'5'69) "'
 
 
 	
r>   z/logical/actors/{actor_id}c                    K   |j                             d          }t          j        |g           d {V }t	          j        t          j        j        d||                   S )Nr_  rz  zActor details fetched.r  )	r  r   r#   r}  r	  r
  r:   r  r  )r   r  r_  rP  s       r<   	get_actorzNodeHead.get_actor  sr       >%%j11$4zJJJJJJJJJ'5'69,(#
 
 
 	
r>   z
/test/dumpc                 ~  K   |j                             d          }|Kd t          j                                        D             }t          j        dt          j        j	        dd|S t          t          j                            |                    }t          j        dt          j        j	        d| dd||iS )	zW
        Dump all data from datacenter. This is used for testing purpose only.
        r   Nc                 ^    i | ]*\  }}|                     d           |t          |          +S )r   )
startswithr`   )rO   rP   rQ   s      r<   rS   z!NodeHead.dump.<locals>.<dictcomp>  sF       Aq||C((477  r>   z'Fetch all data from datacenter success.r  zFetch z from datacenter success.rN   )r  r   r$   __dict__r^   r	  r
  r:   r  r  r`   )r   r  r   all_datarJ  s        r<   dumpzNodeHead.dump  s      
 immE""; &/5577  H
 ,9 +:=A     
+//4455D+9 +:=????  +  r>   c           	        K   t                                                       d {V  |                                 |                                 |                                 |                                 |                                 t          j                    t          j	        | j
                  g}|D ]U}| j                            |          }| j                            |           |                    | j        j                   Vd S re  )ro   runr   rD  rL  r`  rv  r#   purgeorganizerx   r{   create_taskr~   r  add_done_callbackdiscard)r   coroscorotaskr   s       r<   r  zNodeHead.run  s     ggkkmm  ##%%,,..!!  ""!!"4#677
  	C 	CD:))$//D"&&t,,,""4#9#ABBBB	C 	Cr>   )'__name__
__module____qualname__rp   r   r   r`   r   r   r   r   routesr   r	  aiohttp_cacheaiohttpwebResponser  r  r(   r"   r6  rD  rL  r`  r   r	   strr   rT  rU  rO  rv  r~  r  r  r  __classcell__)r   s   @r<   ri   ri      s*       !: !: !: !: !:F
 
 
0C>$*3M 0C 0C 0C 0Cd:$t :$ :$ :$ :$x) ) )2<?4 <? <? <? <?| VZ+'+*>    ,+ @ VZ"##+
W[%9 
 
 
 ,+ $#
 FGGZ6 Z6 HGZ6x  8BV BV BVHX(=X	c4S>!	"X X X X(::/3CH~: : : :<XtCI X X X XK K K VZ!""+
7;+? 
 
 
 ,+ #"
 VZ,--+
gk&: 
 
 
 ,+ .-
 VZ!5    0C C C C C C C C Cr>   ri   node_stats_strc                 h    t          j        |           }t          t          j        |           |S |S re  )r   r   r%   	parse_obj)r  
stats_dicts     r<   rH  rH     s4    N++Jz***r>   )Zr   r   loggingrs   collectionsr   r   concurrent.futuresr   	itertoolsr   typingr   r   r	   r
   r   r   r   aiohttp.webr  r*  ray._private.utilsrayray.dashboard.optional_utils	dashboardoptional_utilsr	  ray.dashboard.utilsutilsr:   ray._common.utilsr   ray._privater   ray._private.collections_utilsr   ray._private.gcs_pubsubr   r   r   ray._private.grpc_utilsr   ray._private.ray_constantsr   r   r   ray.autoscaler._private.utilr   r   r   ray.core.generatedr   r   r   ray.dashboard.constsr   r   r    ray.dashboard.modules.noder!   r"   %ray.dashboard.modules.node.datacenterr#   r$   .ray.dashboard.modules.reporter.reporter_modelsr%   !ray.dashboard.subprocesses.moduler&   !ray.dashboard.subprocesses.routesr'   r  r(   	getLoggerr  r   r)   r5  _config(maximum_gcs_destroyed_actor_cached_countrt  ru  rm  r   r`   r=   rg   ri   r  rH  rN   r>   r<   <module>r     s       * * * * * * * * 1 1 1 1 1 1       K K K K K K K K K K K K K K K K K K          ? ? ? ? ? ? ? ? ? - - - - - - - - - 6 6 6 6 6 6 & & & & & & 0 0 0 0 0 0         
 6 5 5 5 5 5         
         
 P O O O O O O O O O         
 A @ @ @ @ @ @ @ K K K K K K K K G G G G G G > > > > > > L L L L L L 2 2 2 2 2 2		8	$	$ +6+-q+ + ' !$s{;;==! !   
 G$7 D    : : :zl	C l	C l	C l	C l	C l	C l	C l	C^c d      r>   