
    &`i%J                     ,   d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	 d dl
Z
d dlZd dlmZ d dlmc mZ d dlmc mc mc mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZm Z m!Z! d d	l"m#Z# d d
l$m%Z%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9m:Z:m;Z; d dl<m=Z= d dl>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZE d dlFmGZG d dlHmIZImJZJmKZK d dlLmMZM  ejN        eO          ZPg ejQ        dejR        fdejR        fZSd ZT G d d          ZUdS )    N)wraps)ListOptionalTuple)UnaryStreamCall)build_address)hex_to_binary)ray_constants)ActorID	GcsClientJobIDNodeIDTaskID)gcs_service_pb2_grpc)ActorTableDataGcsNodeInfo)FilterPredicateGetAllActorInfoReplyGetAllActorInfoRequestGetAllNodeInfoReplyGetAllNodeInfoRequestGetAllPlacementGroupReplyGetAllPlacementGroupRequestGetAllWorkerInfoReplyGetAllWorkerInfoRequestGetTaskEventsReplyGetTaskEventsRequest)GetObjectsInfoReplyGetObjectsInfoRequest)NodeManagerServiceStub)ListLogsReplyListLogsRequestStreamLogRequest)LogServiceStub)GetRuntimeEnvsInfoReplyGetRuntimeEnvsInfoRequest)JobInfoStorageClient)
JobDetailsJobType)get_driver_jobs)RAY_MAX_LIMIT_FROM_DATA_SOURCEPredicateTypeSupportedFilterType)DataSourceUnavailablezgrpc.max_send_message_lengthzgrpc.max_receive_message_lengthc                 h     t          j                   sJ t                      fd            }|S )zDecorator to add a network handling logic.

    It is a helper method for `StateDataSourceClient`.
    The method can only be used for async methods.
    c                  R  K   	  | i | d{V S # t           j        j        $ r}|                                t           j        j        k    s'|                                t           j        j        k    rt          d          |t          	                    |           |d}~ww xY w)a  Apply the network error handling logic to each APIs,
        such as retry or exception policies.

        Returns:
            If RPC succeeds, it returns what the original function returns.
            If RPC fails, it raises exceptions.

        Raises:
            DataSourceUnavailable: if the source is unavailable because it is down
                or there's a slow network issue causing timeout.

            Exception: Otherwise, the raw network exceptions (e.g., gRPC) will be
                raised.
        Nz]Failed to query the data source. It is either there's a network issue, or the source is down.)
grpcaioAioRpcErrorcode
StatusCodeDEADLINE_EXCEEDEDUNAVAILABLEr.   logger	exception)argskwargsefuncs      p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/util/state/state_manager.pyapi_with_network_error_handlerzBhandle_grpc_network_errors.<locals>.api_with_network_error_handlerK   s       	t.v.........x# 	 	 	DO===6688t:::+S  
   ###	s    B&A:B!!B&)inspectiscoroutinefunctionr   )r=   r?   s   ` r>   handle_grpc_network_errorsrB   C   sM     &t,,,,,
4[[    [< *)    c                      e Zd ZdZdej        j        defdZdej        j        fdZ	de
defdZd	ed
efdZdee
         d
ee
         fdZededfdededeeee
eef                           d
ee         fd            Zededdfdededeeee
eef                           ded
ee         f
d            Zedefdeded
ee         fd            Zededfdededeeee
eef                           d
ee         fd            Z ededfdededeeee
eef                           d
ee!         fd            Z"d(ded
ee#         fdZ$edefde
dededed
ee%         f
d            Z&defde
dededed
ee'         f
dZ(e	 d(d	e
de
ded
e)fd             Z*e	 	 d)d	e
d!e
d"ed#ed$ee+         ded%ee         d&ee         d
e,fd'            Z-dS )*StateDataSourceClienta  The client to query states from various data sources such as Raylet, GCS, Agents.

    Note that it doesn't directly query core workers. They are proxied through raylets.

    The module is not in charge of service discovery. The caller is responsible for
    finding services and register stubs through `register*` APIs.

    Non `register*` APIs
    - Return the protobuf directly if it succeeds to query the source.
    - Raises an exception if there's any network issue.
    - throw a ValueError if it cannot find the source.
    gcs_channel
gcs_clientc                     |                      |           t          |          | _        || _        t	          j                    | _        d S N)register_gcs_clientr'   _job_client_gcs_clientaiohttpClientSession_client_session)selfrF   rG   s      r>   __init__zStateDataSourceClient.__init__{   sE      ---/
;;%&466rC   c                     t          j        |          | _        t          j        |          | _        t          j        |          | _        t          j        |          | _        t          j	        |          | _
        d S rI   )r   ActorInfoGcsServiceStub_gcs_actor_info_stub PlacementGroupInfoGcsServiceStub_gcs_pg_info_stubNodeInfoGcsServiceStub_gcs_node_info_stubWorkerInfoGcsServiceStub_gcs_worker_info_stubTaskInfoGcsServiceStub_gcs_task_info_stub)rP   rF   s     r>   rJ   z)StateDataSourceClient.register_gcs_client   s    $8$P%
 %
! "6!V"
 "
 $8#N$
 $
  &:%R&
 &
" $8#N$
 $
   rC   ipportc                 r    ddl m} t          } |t          ||          |d          }t	          |          S )Nr   init_grpc_channelT)asynchronous)ray._private.grpc_utilsra   _STATE_MANAGER_GRPC_OPTIONSr   r    )rP   r]   r^   ra   optionschannels         r>   get_raylet_stubz%StateDataSourceClient.get_raylet_stub   sI    ======-##M"d$;$;WSWXXX%g...rC   node_idreturnc                   K   ddl m} | j                            t          j         |                                                                 t          j	        t          j
                   d{V }|sdS t          j        |          \  }}}t          j        } |t          ||          |d          }t          |          S )zGReturns None if the agent on the node is not registered in Internal KV.r   r`   	namespacetimeoutNT)re   rb   )rc   ra   rL   async_internal_kv_getdashboard_consts#DASHBOARD_AGENT_ADDR_NODE_ID_PREFIXhexencoder
   KV_NAMESPACE_DASHBOARDGCS_RPC_TIMEOUT_SECONDSjsonloadsGLOBAL_GRPC_OPTIONSr   r$   )	rP   rh   ra   
agent_addrr]   	http_port	grpc_portre   rf   s	            r>   get_log_service_stubz*StateDataSourceClient.get_log_service_stub   s      ======+AACTW[[]]TT[[]]#:$< B 
 
 
 
 
 
 
 


  	4#':j#9#9 Iy3##"i(('
 
 
 g&&&rC   c                    K   |sdS | j                             t          j         |                                 t
          j        t          j                   d{V }|sdS t          j	        |          \  }}}|S )a  Return the node id in hex that corresponds to the given ip.

        Args:
            ip: The ip address.

        Returns:
            None if the corresponding id doesn't exist.
            Node id otherwise. If None node_ip is given,
            it will also return None.
        Nrk   )
rL   rn   ro   DASHBOARD_AGENT_ADDR_IP_PREFIXrr   r
   rs   rt   ru   rv   )rP   r]   rx   rh   ry   rz   s         r>   ip_to_node_idz#StateDataSourceClient.ip_to_node_id   s        	4+AA>DDDKKMM#:$< B 
 
 
 
 
 
 
 


  	4(,
:(>(>%IrC   Nrm   limitfiltersc                   K   |g }t          j                    }|D ]}|\  }}}|dk    r|dk    r4t          t          |                                                    |_        I|dk    rk|                                }|t          j        	                                vrt          d|           t          j                            |          |_        |dk    r3t          t          |                                                    |_        t          ||          }	| j                            |	|           d {V }
|
S )N=actor_idstatez#Invalid actor state for filtering: job_idr   r   rm   )r   Filtersr   r	   binaryr   upperr   
ActorStatekeys
ValueErrorValuer   r   r   rT   GetAllActorInforP   rm   r   r   req_filtersfilterkey	predicatevaluerequestreplys              r>   get_all_actor_infoz(StateDataSourceClient.get_all_actor_info   sh      ?G,466 	J 	JF$*!CECj  '.}U/C/C'D'D'K'K'M'M$$ 9 > > @ @@@$%R5%R%RSSS$2$=$C$CE$J$J!!%*=+?+?%@%@%G%G%I%I"(ukJJJ/??W @ 
 
 
 
 
 
 
 
 rC   Fexclude_driverc                 B  K   |g }t          j                    }|D ]G}|\  }}}	d }
|dk    rt          j        }
n |dk    rt          j        }
nJ d|z               |dk    rst           j                                        }t          t          |	                                                    |_	        |
|_
        |j                            |           |dk    rtt           j                                        }t          t          |	                                                    |_        |
|_
        |j                            |           1|dk    rtt           j                                        }t%          t          |	                                                    |_        |
|_
        |j                            |           |dk    rHt           j                                        }|	|_        |
|_
        |j                            |           |d	k    rHt           j                                        }|	|_        |
|_
        |j                            |           GI||_        t          ||
          }| j                            ||           d {V }|S )Nr   !=FzInvalid predicate: r   r   task_idnamer   r   r   )r   r   r   EQUAL	NOT_EQUALActorIdFilterr   r	   r   r   r   actor_filtersappendJobIdFilterr   r   job_filtersTaskIdFilterr   r   task_filtersTaskNameFilter	task_nametask_name_filtersStateFilterr   state_filtersr   r\   GetTaskEvents)rP   rm   r   r   r   r   r   r   r   r   filter_predicateactor_filter
job_filtertask_filtertask_name_filterstate_filterr   r   s                     r>   get_all_task_infoz'StateDataSourceClient.get_all_task_info   s      ?G*244 ,	 ,	F$*!CE#C#2#8  d""#2#<  
 @3i???uj  3;IIKK(/e0D0D(E(E(L(L(N(N%)9&)00>>>>19EEGG
$)-*>*>$?$?$F$F$H$H
!'7
$'..z::::	!!2:GGII&,]5-A-A&B&B&I&I&K&K#(8%(//<<<<#7#?#N#N#P#P -2 *-= *-445EFFFF3;GGII%*")9&)00>>>> %3"&UKHHH.<<Wg<VVVVVVVVrC   c                 n   K   t          |          }| j                            ||           d {V }|S Nr   r   )r   rV   GetAllPlacementGroup)rP   rm   r   r   r   s        r>   get_all_placement_group_infoz2StateDataSourceClient.get_all_placement_group_info%  s`      
 .E:::,AAW B 
 
 
 
 
 
 
 
 rC   c                   K   |g }g }d }|D ]}|\  }}}	|dk    r|dk    r\t          j                    }
t          t          |	                                                    |
_        |                    |
           r|dk    rf|	                                }	|	t          j	        
                                vrt          d|	           t          j	                            |	          }|dk    r1t          j                    }
|	|
_        |                    |
           t          |||          }| j                            ||           d {V }|S )Nr   rh   r   z"Invalid node state for filtering: 	node_name)r   node_selectorsr   r   )r   NodeSelectorr   r	   r   rh   r   r   r   GcsNodeStater   r   r   r   rX   GetAllNodeInfo)rP   rm   r   r   r   r   r   r   r   r   node_selectorr   r   s                r>   get_all_node_infoz'StateDataSourceClient.get_all_node_info0  s      ?G 	 	F$*!CECi 5 B D D(.}U/C/C(D(D(K(K(M(M%%%m4444 8 = = ? ???$%Q%%Q%QRRR*7==eDD## 5 B D D*/'%%m4444'\
 
 
 .==gw=WWWWWWWWrC   c                   K   |g }t          j                    }|D ]:}|\  }}}|dk    r|dv r|dk    rd|_         |dk    r|dk    r|dk    rd|_        :;t          ||          }	| j                            |	|	           d {V }
|
S )
Nnum_paused_threads)r   >0Tis_aliver   Truer   r   )r   r   exist_paused_threadsr   rZ   GetAllWorkerInfor   s              r>   get_all_worker_infoz)StateDataSourceClient.get_all_worker_info[  s       ?G-577 	 	F$*!CE +++,,SLL370j  Y#%5%5%6//'+$){KKK0AAW B 
 
 
 
 
 
 
 
 rC   c                   K   t          | j        |           d {V \  }| j                            |           d {V }fd|                                D             }t          |                                          |z   S )Nr   c                     g | ]e\  }}t          di t          j        |          ||v r                    |          j        nd                     |          t
          j        dfS )N)submission_idr   driver_infotype )r(   dataclassesasdictgetidr)   
SUBMISSION).0r   jobsubmission_job_driverss      r>   
<listcomp>z6StateDataSourceClient.get_job_info.<locals>.<listcomp>  s     
 
 
 #s   $S))+ $::: .11-@@CC266}EE'   
 
 
rC   )r*   rL   rK   get_all_jobsitemslistvalues)rP   rm   driver_jobssubmission_jobsr   s       @r>   get_job_infoz"StateDataSourceClient.get_job_info  s       5Dg5
 5
 5
 /
 /
 /
 /
 /
 /
++ !% 0 = =g = N NNNNNNN
 
 
 
 '6&;&;&=&=
 
 
 K&&(())O;;rC   node_manager_ipnode_manager_portc                    K   |                      ||          }|                    t          |          |           d {V }|S r   )rg   GetObjectsInfor   )rP   r   r   rm   r   stubr   s          r>   get_object_infoz%StateDataSourceClient.get_object_info  sn       ##O5FGG))!... * 
 
 
 
 
 
 
 
 rC   node_ipruntime_env_agent_portc                 N  K   |r|st          d| d| d          t          j        |          }dt          ||           d}t	          |          }|                                }| j                            |||          4 d {V }|j        d	k    r\|j        d
k     rQ|	                                 d {V }	t                      }
|
                    |	           |
cd d d           d {V  S t          d|j         d|j                   # 1 d {V swxY w Y   d S )Nz;Expected non empty node ip and runtime env agent port, got z and .)totalzhttp://z/get_runtime_envs_infor   )datarm      i,  zFailed to query the runtime env agent for get_runtime_envs_info. Either there's a network issue, or the source is down. Response is z	, reason )r   rM   ClientTimeoutr   r&   SerializeToStringrO   poststatusreadr%   ParseFromStringr.   reason)rP   r   r   rm   r   urlr   r   respresponse_datar   s              r>   get_runtime_envs_infoz+StateDataSourceClient.get_runtime_envs_info  s       	4 	uguu\ruuu   'g666^g/EFF^^^+%888((**',,StW,MM 	 	 	 	 	 	 	QU{c!!dkC&7&7&*iikk 1 1 1 1 1 1/11%%m444	 	 	 	 	 	 	 	 	 	 	 	 	 	 ,G#';G G9=G G  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AD5D
D!Dglob_filterc                    K   |                      t          j        |                     d {V }|st          d| d          |                    t          |          |           d {V S )NAgent for node id:  doesn't exist.)r   r   )r{   r   from_hexr   ListLogsr"   )rP   rh   r   rm   r   s        r>   	list_logszStateDataSourceClient.list_logs  s       ..vw/G/GHHHHHHHH 	MK7KKKLLL]]444g # 
 
 
 
 
 
 
 
 	
rC   log_file_name
keep_alivelinesintervalstart_offset
end_offsetc	           
        K   |                      t          j        |                     d {V }	|	st          d| d          |	                    t          ||||||          |          }
|
                                 d {V }|                    t          j	                  ,t          |                    t          j	                            |
S )Nr   r   )r   r   r  r  r  r  r   )
r{   r   r   r   	StreamLogr#   initial_metadatar   
log_constsLOG_GRPC_ERROR)rP   rh   r   r   r  r  rm   r  r  r   streammetadatas               r>   
stream_logz StateDataSourceClient.stream_log  s       ..vw/G/GHHHHHHHH 	MK7KKKLLL%+!)%      

 

  0022222222<<
122>X\\**CDDEEErC   rI   )NN).__name__
__module____qualname____doc__r1   r2   Channelr   rQ   rJ   strintrg   r   r$   r{   r   r~   rB   r+   r   r   r,   r-   r   r   boolr   r   r   r   r   r   r   r   r(   r   r   r   r%   r   r!   r   floatr   r  r   rC   r>   rE   rE   m   s        7DH$4 7) 7 7 7 7
tx/? 
 
 
 
"/# /S / / / /'& '^ ' ' ' '$hsm     0   3RV	   $uS-9L%LMNO	
 
&	'    @   3RV$> >> > $uS-9L%LMNO	>
 > 
$	%> > >  >@  !0N *-	+	,       3RV	( (( ( $uS-9L%LMNO	(
 
%	&( ( (  (T   3RV	   $uS-9L%LMNO	
 
'	(    H< <# <j9I < < < <,  
 3   	
  
%	&    & 3  !$ 	
  
)	*   8  =A
 

),
7:
	
 
 
  
   '+$(   	
  5/  sm SM 
      rC   rE   )Vr   r@   ru   logging	functoolsr   typingr   r   r   rM   r1   grpc.aio._callr   ray.dashboard.consts	dashboardconstsro   $ray.dashboard.modules.log.log_constsmoduleslogr  ray._common.network_utilsr   ray._common.utilsr	   ray._privater
   ray._rayletr   r   r   r   r   ray.core.generatedr   ray.core.generated.gcs_pb2r   r   "ray.core.generated.gcs_service_pb2r   r   r   r   r   r   r   r   r   r   r   #ray.core.generated.node_manager_pb2r   r   (ray.core.generated.node_manager_pb2_grpcr    ray.core.generated.reporter_pb2r!   r"   r#   $ray.core.generated.reporter_pb2_grpcr$   (ray.core.generated.runtime_env_agent_pb2r%   r&    ray.dashboard.modules.job.commonr'   )ray.dashboard.modules.job.pydantic_modelsr(   r)   ray.dashboard.modules.job.utilsr*   ray.util.state.commonr+   r,   r-   ray.util.state.exceptionr.   	getLoggerr  r8   rw   GRPC_CPP_MAX_MESSAGE_SIZErd   rB   rE   r   rC   r>   <module>r3     s                ( ( ( ( ( ( ( ( ( (   * * * * * * / / / / / / / / / 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 3 3 3 3 3 3 + + + + + + & & & & & & A A A A A A A A A A A A A A 3 3 3 3 3 3 B B B B B B B B                                 L K K K K K         
 @ ? ? ? ? ?        B A A A A A I I I I I I I I ; ; ; ; ; ;         
 ; : : : : :		8	$	$&#]%LM '(OP '* '* '*T~ ~ ~ ~ ~ ~ ~ ~ ~ ~rC   