
    &`iL                     "   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZd dlm	c m
Z
 d dlmc mZ d dlmZ d dlmZ d dlmZ d dlZ ej        e          ZdZdZ	 e j        Zn# e$ r
 e j        ZY nw xY wd Zd	 Zd
edefdZ d
edefdZ!d Z"dS )    N)ThreadPoolExecutor)run_background_task)	GcsClient)_PARENT_DEATH_THREASHOLD   i   c                      t           j        dv rd S t          t          j        d                   } | dk    sJ t
                              d|            | S )Nwin32cygwinRAY_RAYLET_PIDr   zraylet pid is %s)sysplatformintosenvironloggerinfo)
raylet_pids    p/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/process_watcher.pyget_raylet_pidr   %   sT     |***tRZ 0122J>>>>
KK"J///    c                 @   t           j        dv rt          d          t                      }t          j        r-t                              d           t          | |||          }n,t                              d           t          || ||          }t          |          S )al  
    Creates an asyncio task to periodically check if the raylet process is still
    running. If raylet is dead for _PARENT_DEATH_THREASHOLD (5) times, prepare to exit
    as follows:

    - Write logs about whether the raylet exit is graceful, by looking into the raylet
    log and search for term "SIGTERM",
    - Flush the logs via GcsClient,
    - Exit.
    r	   z&can't check raylet process in Windows.check_parent_via_pipe_check_parent)r   r   RuntimeErrorr   dashboard_constsPARENT_HEALTH_CHECK_BY_PIPEr   r   _check_parent_via_piper   r   )log_dir
gcs_clientparent_dead_callbackloopr   check_parent_tasks         r   create_check_raylet_taskr$   1   s     |***CDDD!!J3 	
+,,,2Z';
 
 	O$$$)-A
 
 0111r   r   r    c                    t           j                            | d          }d}d}	 t          |dd          5 }|                    dt
          j                   t          d|                                t          z
            }|                    |t
          j
                   |                                }t          d |D                       r |d	z  }t                              |           n9|d
t           dz  }|dd                    |t           d                    z   z  }d}d d d            n# 1 swxY w Y   n?# t           $ r2}|d| d| dz  }t                              |           d}Y d }~nd }~ww xY w|rMt                              |           t&          j        j                            t.          j        ||           d S t                              |           d S )Nz
raylet.outFzRaylet is terminated. rzutf-8)encodingr   c              3      K   | ]}d |v V  	dS )zRaylet received SIGTERMN ).0lines     r   	<genexpr>z+report_raylet_error_logs.<locals>.<genexpr>]   s(      MM,4MMMMMMr   zTermination is graceful.zTermination is unexpected. Possible reasons include: (1) SIGKILL by the user or system OOM killer, (2) Invalid memory access from Raylet causing SIGSEGV or SIGBUS, (3) Other termination signals. Last z lines of the Raylet logs:
z    TzFailed to read Raylet logs at z: !)r    )r   pathjoinopenseekioSEEK_ENDmaxtell_RAYLET_LOG_MAX_TAIL_SIZESEEK_SET	readlinesanyr   r   _RAYLET_LOG_MAX_PUBLISH_LINES	Exception	exceptionerrorray_privateutilspublish_error_to_driverray_constantsRAYLET_DIED_ERROR)	r   r    log_pathr=   msgfposraylet_logses	            r   report_raylet_error_logsrJ   N   sV   w||G\22HE
"C(C'222 	a FF1bk"""a$==>>CFF3$$$++--K MMMMMMM 11C    ,
 :, , , v!> > ? ?@! !   5	 	 	 	 	 	 	 	 	 	 	 	 	 	 	6    @@@A@@@@  	S22+! 	3 	
 	
 	
 	
 	
 	Cs;   E C1D5)E 5D99E <D9=E 
E=(E88E=c                   K   	 	 t          d          5 }|                    |d            d {V }d d d            n# 1 swxY w Y   t          |          dk    r/ |d           t          | |           t	          j        d           n4# t          $ r'}t                              d|            Y d }~nd }~ww xY w)NT   )max_workersc                  >    t           j                                        S )N)r   stdinreadliner)   r   r   <lambda>z(_check_parent_via_pipe.<locals>.<lambda>   s    ci&8&8&:&: r   r   z+_check_parent_via_pipe: The parent is dead.zIraylet health checking is failed. The agent process may leak. Exception: )	r   run_in_executorlenrJ   r   exitr;   r   r<   )r   r    r"   r!   executor
input_datarI   s          r   r   r   ~   s[     	
 $222 h#'#7#7::$ $      
               :!##$$%RSSS(*=== 	 	 	>:;> >       	s8   B ?B AB AAB 
B>B99B>c                   K   	 t          j                    }d}	 |                                }|du }d}d}	|r|j        dk    }| |j        k    }	|s|s|	r|dz  }t                              d| dt           d| d	| d
| d|	 d           |t          k     r%t          j        t          j
                   d{V   |d           t          ||           t          j        d           nd}t          j        t          j
                   d{V  # t          $ r2 t                              d           t          j        d           Y dS w xY w)z0Check if raylet is dead and fate-share if it is.r   TNFrL   zRaylet is considered dead z X. If it reaches to z&, the agent will kill itself. Parent: z, parent_gone: z, init_assigned_for_parent: z, parent_changed: .z"_check_parent: The parent is dead.z$Failed to check parent PID, exiting.)psutilProcessparentpidr   warningr   asynciosleepr   'DASHBOARD_AGENT_CHECK_PARENT_INTERVAL_SrJ   r   rT   r;   r<   )
r   r   r    r!   	curr_procparent_death_cntr[   parent_goneinit_assigned_for_parentparent_changeds
             r   r   r      s	     ,N$$	&	%%''F D.K',$"N : ,2:?( ",vz!9 %6 %. % A% 91A 9 9(@9 9179 9 %09 9 2J	9 9
 (69 9 9   $&>>>!-(P         $$%IJJJ(*===#$ - H        I&	N    ?@@@s   DD 8EE)#r^   r2   loggingr   r   concurrent.futuresr   r>   ray._private.ray_constantsr?   rB   ray.dashboard.consts	dashboardconstsr   ray._common.utilsr   ray._rayletr   r   rY   	getLogger__name__r   r:   r6   create_taskAttributeErrorensure_futurer   r$   strrJ   r   r   r)   r   r   <module>rt      s    				  				 



 1 1 1 1 1 1 



 2 2 2 2 2 2 2 2 2 / / / / / / / / / 1 1 1 1 1 1 ! ! ! ! ! ! 9 9 9 9 9 9 		8	$	$ !#  ( (%KK ( ( ('KKK(	 	 	2 2 2:-c -y - - - -`'   2. . . . .s   A$ $A32A3