
    %`i_              	       T   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ d dlmc mZ d dlmc mZ d dlZd dlmZ d dlmZ d dlmZ  ej        e          Z  ej!        d          Z" ej!        d          Z# e$ ej%        dd	                    Z& e' ej%        d
d                    Z( e' ej%        dd                     Z) G d d          Z* G d d          Z+d Z,edk    r e j-        d          Z.e./                    dde0d           e./                    dde0ej1        ej2        ej3                   e./                    dde0ej4        ej5                   e./                    dde0ej6        dej6         d           e./                    d d!e0d"           e./                    d#d!e0d$           e./                    d%d!e'd&           e./                    d'd!e'd(           e./                    d)dd*e0d+,           e./                    d-dd*e0d.,           e.7                                Z8e	j        d/k    re8j9        nd Z:e	j        d/k    re8j;        nd0Z< e=e8j>        e8j?        e8j@        e8jA        e:e<1          ZB ed7i eBZ  ejC        e8jD        e8jE        e:e<            ejF        e8jG                  ZH ee8jI        2          ZJ e+eHe8j@        eJe,e8jI        3          ZK	 eKL                                 dS # eM$ rZNej        jO        P                     ejQ                              ZRd4 ejS                     d5eR ZTej        jO        U                    ejV        eTeJ6           e W                    eT           eNdZN[Nww xY wdS )8    N)CallableListOptionalSet)logging_utils)setup_component_logger)	GcsClientz.*worker.*-([0-9a-f]+)-(\d+)z.*runtime_env_setup-(\d+).logLOG_NAME_UPDATE_INTERVAL_Sg      ?$RAY_LOG_MONITOR_MANY_FILES_THRESHOLDi  %RAY_RUNTIME_ENV_LOG_TO_DRIVER_ENABLEDc                   0    e Zd Z	 	 	 	 	 	 	 ddZd Zd ZdS )LogFileInfoNFc                     |||J || _         || _        || _        || _        || _        || _        || _        d | _        d | _        d S )N	filenamesize_when_last_openedfile_positionfile_handleis_err_filejob_id
worker_pid
actor_name	task_name)selfr   r   r   r   r   r   r   s           l/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/log_monitor.py__init__zLogFileInfo.__init__.   sg      %1))* !%:"*&&$    c                    	 d}| j         r<| j         j        s0t          j        | j                                                   j        }t          j        | j                  }|j        |k    r^t          | j        d          | _         |j	        | j
        k     rd| _
        | j                             | j
                   |j	        | _        dS dS # t          $ r& t                              d| j                    Y dS w xY w)a  Check if the file's inode has changed and reopen it if necessary.

        There are a variety of reasons what we would logically consider a file
        would have different inodes, such as log rotation or file syncing
        semantics.

        If the file is smaller than our recorded file position, we assume it has been
        rotated and start reading it from the beginning.
        Nrbr   z*file no longer exists, skip re-opening of )r   closedosfstatfilenost_inostatr   openst_sizer   seekr   	Exceptionloggerdebug)r   
open_inodenew_statinfos      r   reopen_if_necessaryzLogFileInfo.reopen_if_necessaryG   s   	WJ H(8(? HXd&6&=&=&?&?@@G
74=11L"j00#'t#<#< 
  '$*<<<)*D& %%d&8999-9-A*** 10  	W 	W 	WLLUdmUUVVVVVV	Ws   CC ,C;:C;c                     d| j          d| j         d| j         d| j         d| j         d| j         d| j         d| j         d	| j         d
S )NzFileInfo(
	filename: z
	size_when_last_opened: z
	file_position: z
	file_handle: z
	is_err_file: z

	job_id: z
	worker_pid: z
	actor_name: z
	task_name: z
)r   )r   s    r   __repr__zLogFileInfo.__repr__e   s    
=
 
(,(B
 
 !% 2
 
 #.	
 

 #.
 
 
 
 "_
 
 "_
 
 !N
 
 
	
r   )NNNNFNN)__name__
__module____qualname__r   r.   r0    r   r   r   r   -   s`         "   2W W W<
 
 
 
 
r   r   c                       e Zd ZdZej        dfdedededee	ge
f         de	dee         fd	Zdee         d
e
fdZd Zd Zd Zd Zded
e
fdZd ZdS )
LogMonitora  A monitor process for monitoring Ray log files.

    This class maintains a list of open files and a list of closed log files. We
    can't simply leave all files open because we'll run out of file
    descriptors.

    The "run" method of this class will cycle between doing several things:
    1. First, it will check if any new files have appeared in the log
       directory. If so, they will be added to the list of closed files.
    2. Then, if we are unable to open any new files, we will close all of the
       files.
    3. Then, we will open as many closed files as we can that may have new
       lines (judged by an increase in file size since the last time the file
       was opened).
    4. Then we will loop through the open files and see if there are any new
       lines in the file. If so, we will publish them to Ray pubsub.

    Attributes:
        ip: The hostname of this machine, for grouping log messages.
        logs_dir: The directory that the log files are in.
        log_filenames: This is the set of filenames of all files in
            open_file_infos and closed_file_infos.
        open_file_infos (list[LogFileInfo]): Info for all of the open files.
        closed_file_infos (list[LogFileInfo]): Info for all of the closed
            files.
        can_open_more_files: True if we can still open more files and
            false otherwise.
        max_files_open: The maximum number of files that can be open.
    Nnode_ip_addresslogs_dir
gcs_clientis_proc_alive_fnmax_files_opengcs_addressc                    || _         || _        || _        t                      | _        g | _        g | _        d| _        || _        || _	        | 
                    |          | _        t                              d| d| j         d           dS )z"Initialize the log monitor object.Tz*Starting log monitor with [max open files=z], [is_autoscaler_v2=]N)ipr8   r9   setlog_filenamesopen_file_infosclosed_file_infoscan_open_more_filesr;   r:   get_is_autoscaler_v2is_autoscaler_v2r*   info)r   r7   r8   r9   r:   r;   r<   s          r   r   zLogMonitor.__init__   s     '%$'*uu2446)- #17G&*&?&?&L&L; ; ;"&"7; ; ;	
 	
 	
 	
 	
r   returnc                     |dS t           j        j                                        s)t           j        j                            | j                   ddlm}  |            S )z"Check if autoscaler v2 is enabled.NFr   )rF   )rayexperimentalinternal_kv_internal_kv_initialized_initialize_internal_kvr9   ray.autoscaler.v2.utilsrF   )r   r<   rF   s      r   rE   zLogMonitor.get_is_autoscaler_v2   se    5+DDFF 	R(@@QQQ<<<<<<!!!r   c                    t          | j                  dk    r| j                            d          }|j                                         d|_        d}|j        dk    r2|j        dk    r&|j        dk    r|j        dk    r|j        t          |j        t                    rJ d|j         d	            |                     |j                  }|st          j
                            | j        d
t          j
                            |j                            }	 t          j        |j        |           nY# t"          t$          f$ rE}|j        t&          j        k    r$t*                              d|j         d           n|Y d}~nd}~ww xY w|r| j                            |           t          | j                  dk    d| _        dS )z0Close all open files (so that we can open more).r   NTraylet
gcs_server
autoscalerruntime_envz&PID should be an int type. Given PID: .oldWarning: The file  was not found.)lenrB   popr   closer   
isinstancestrr:   r!   pathjoinr8   basenamer   shutilmoveIOErrorOSErrorerrnoENOENTr*   warningrC   appendrD   )r   	file_info
proc_alivetargetes        r   _close_all_fileszLogMonitor._close_all_files   s   $&''!++,0033I!'')))$(I!J
 $00(L88(L88(M99(4%i&:C@@  W	@TWWW @ "2293GHH
! $  W\\ubg.>.>y?Q.R.R F$I$6????#W- $ $ $7el22"NN XY5G X X X    #$G	    $  9&--i888M $&''!++P $(   s    D; ;F;FFc                    g }|t          j         | j         d          t          j         | j         d          z   z  }|t          j         | j         d          z  }| j        s |t          j         | j         d          z  }n|t          j         | j         d          z  }|t          j         | j         d          z  }| j         d}t          j                            |          r|t          j         | j         d          z  }t          r|t          j         | j         d	          z  }|D ]>}t          j                            |          r|| j        vrt          
                    |          }|r#t          |                    d
                    }nd}d}d|v r1t          
                    |          }|r|                    d          }|                    d          }| j                            |           | j                            t%          |ddd|||                     t          j                            |          }	t(                              d|	            @dS )z(Update the list of log files to monitor.z/worker*[.out|.err]z/java-worker*.logz/raylet*.errz/monitor.logz/events/event_AUTOSCALER.logz/gcs_server*.errz	/tpu_logsz/tpu_logs/**z/runtime_env*.log   NrT      errr   )r   r   r   r   r   r   r   zBeginning to track file )globr8   rF   r!   r^   isdirr   isfilerA   WORKER_LOG_PATTERNmatchintgroupRUNTIME_ENV_SETUP_PATTERNendswithaddrC   rh   r   r`   r*   rG   )
r   monitor_log_pathstpu_log_dir	file_pathworker_matchr   r   runtime_env_job_matchr   log_filenames
             r   update_log_filenameszLogMonitor.update_log_filenames   s   TY}111
 
I999::; 	; 	TY$-'E'E'EFFF $ 	dm+I+I+I!J!JJ =>>>" " 
 	TY$-'I'I'IJJJ 1117==%% 	Kdm+I+I+I!J!JJ 1 	Pdm+N+N+N!O!OO* 	G 	GIw~~i(( GYd>P-P-P177	BB &!$\%7%7%:%:!;!;JJ!%J !I--,E,K,KI,V,V), @!6!<!<Q!?!?'0077"&&y111&--!*./&'$($/%#-  
 
 
  "w//	::E|EEFFF?	G 	Gr   c                    | j         s|                                  g }t          | j                  dk    rt          | j                  | j        k    r	d| _         n| j                            d          }|j        J 	 t          j	        
                    |j                  }nw# t          t          f$ rc}|j        t          j        k    rGt                               d|j         d           | j                            |j                   Y d}~|d}~ww xY w||j        k    r	 t+          |j        d          }nx# t          t          f$ rd}|j        t          j        k    rHt                               d|j         d           | j                            |j                   Y d}~|d}~ww xY w|                    |j                   ||_        ||_        | j                            |           n|                    |           t          | j                  dk    t          | j                  | j        k    rd| _         | xj        |z  c_        dS )zOpen some closed files if they may have new lines.

        Opening more files may require us to close some of the already open
        files.
        r   FNrW   rX   r   )rD   rm   rY   rC   rB   r;   rZ   r   r!   r^   getsizer   rc   rd   re   rf   r*   rg   rA   remover   r&   r(   r   rh   )r   files_with_no_updatesri   	file_sizerl   fs         r   open_closed_fileszLogMonitor.open_closed_files%  s    ' 	$!!### "$())A--4'((D,???+0(.22155I(000
GOOI,>??		W%   7el**NNPY-?PPP   &--i.@AAAHHHH 9:::
 Y/66AA)      w%,..T1CTTT   *11)2DEEE   y.///2;	/()	%$++I6666%,,Y777S $())A--V t#$$(;;;',D$"77s>   $B& &D7ADDD)D? ?F4AF/-F//F4c           
          dg  fd} j         D ]j        j        rJ                                  t          j        }t          |          D ]}	 j                                        }|                    dd          }|dk    r n|	                    d          }|
                    t          j                  r= |             |                    t          j        d          d         _        d_        n|
                    t          j                  r6 |             |                    t          j        d          d         _        n|
                    t          j                  r,|                    t          j        d          d         _        nD|
                    d	          rj                                         n                    |           # t&          $ rD t(                              d
j         dj        j                                         d            w xY wj        dk    rNj                            dd          }d|v rd_        n'd|v rd_        nd|v sd|v rd_        nd|v rd_        j                                        _         |             S )zGets updates to the log files and publishes them.

        Returns:
            True if anything was published and false otherwise.
        Fc                     t                    dk    rwj        j        j        j        j        j        d} 	 j                            |            n-# t          $ r  t                              d|             Y nw xY wdg d S d S )Nr   )r?   pidjobis_errlinesr   r   zFailed to publish log messages T)rY   r?   r   r   r   r   r   r9   publish_logsr)   r*   	exception)dataanything_publishedri   lines_to_publishr   s    r   flushz=LogMonitor.check_log_files_and_publish_updates.<locals>.flushi  s     #$$q(('$/$+'3-"+"6!*!4 OO006666  O O O$$%Mt%M%MNNNNNO%)"#%    )(s   A 'BBzutf-8replace z
rp   Nz)Windows fatal exception: access violationzError: Reading file: z, position: z failed.r   \/z/rayletrQ   z/gcs_serverrR   z/monitorevent_AUTOSCALERrS   z/runtime_envrT   )rB   r   r    r.   ray_constantsLOG_MONITOR_NUM_LINES_TO_READrangereadlinedecoderstrip
startswithLOG_PREFIX_ACTOR_NAMEsplitr   r   LOG_PREFIX_TASK_NAMELOG_PREFIX_JOB_IDr   rh   r)   r*   errorr   ri   tellr   r   r   )	r   r   max_num_lines_to_read_	next_liner   r   ri   r   s	   `     @@@r   #check_log_files_and_publish_updatesz.LogMonitor.check_log_files_and_publish_updates`  s>    #	& 	& 	& 	& 	& 	& 	& 	&( - E	 E	I ,3333))+++$1$O!011 0 0/ ) 5 > > @ @I !* 0 0) D DI B ) 0 0 8 8I ++M,OPP ;/8)?0 00	, /3	++"--m.PQQ ;.7oo)>/ //	++ #--m.MNN ;+4??);Q, ,,	(( #--C  ; "-668888(//	:::    LL"	0B " "%.%8%D%I%I%K%K" " "  
  &!++$-55dC@@((+3I(("h..+7I((8++/AX/M/M+7I((#x//+8I( '0&;&@&@&B&BI#EGGGG!!s   5G+EG++AH9last_file_updated_timec                     t          t          j                    |z
            }t          | j                  t          k     p
|t
          k    S )as  Return true if filenames should be updated.

        This method is used to apply the backpressure on file updates because
        that requires heavy glob operations which use lots of CPUs.

        Args:
            last_file_updated_time: The last time filenames are updated.

        Returns:
            True if filenames should be updated. False otherwise.
        )floattimerY   rA   r   r
   )r   r   elapsed_secondss      r   should_update_filenamesz"LogMonitor.should_update_filenames  sD      	.D DEE"##&JJ <!;;	
r   c                     t          j                     }	 |                     |          r'|                                  t          j                     }|                                  |                                 }|st          j        d           {)zRun the log monitor.

        This will scan the file system once every LOG_NAME_UPDATE_INTERVAL_S to
        check if there are new log files to monitor. It will also publish new
        log lines.
        Tg?)r   r   r   r   r   sleep)r   last_updatedr   s      r   runzLogMonitor.run  s     y{{
	 ++L99 +))+++#y{{""$$$!%!I!I!K!K &  
3
	 r   )r1   r2   r3   __doc__r   LOG_MONITOR_MAX_OPEN_FILESr]   r	   r   rw   boolr   r   rE   rm   r   r   r   r   r   r   r4   r   r   r6   r6   u   s)        H ,F%)
 

 
 	

 #C5$;/
 
 c]
 
 
 
4	" 	"$ 	" 	" 	" 	"*( *( *(X>G >G >G@98 98 98vd" d" d"L
e 
 
 
 
 
$         r   r6   c                 |    dd l }	 |                    |                                           S # |j        $ r Y dS w xY w)Nr   F)psutilProcess
is_runningNoSuchProcess)r   r   s     r   is_proc_aliver     sS    MMM~~c""--///   uus   &- 
;;__main__z;Parse GCS server address for the log monitor to connect to.)descriptionz--gcs-addressFzThe address (ip:port) of GCS.)requiredtypehelpz--logging-level)r   r   defaultchoicesr   z--logging-format)r   r   r   r   z--logging-filenamezFSpecify the name of log file, log to stderr if set empty, default is ""z--session-dirTz@Specify the path of the session directory used by Ray processes.z
--logs-dirz<Specify the path of the log directory used by Ray processes.z--logging-rotate-bytesz,Specify the max bytes for rotating log file.z--logging-rotate-backup-countz-Specify the backup count of rotated log file.z--stdout-filepathr   z(The filepath to dump log monitor stdout.)r   r   r   r   z--stderr-filepathz(The filepath to dump log monitor stderr.win32rp   )logging_levellogging_formatlog_dirr   	max_bytesbackup_count)address)r<   zThe log monitor on node z" failed with the following error:
)r9   r4   )Xargparsere   rr   logginglogging.handlersr!   platformrera   sysr   	tracebacktypingr   r   r   r   ray._private.ray_constants_privater   ray._private.servicesservicesray._private.utilsrJ   ray._privater   ray._private.ray_loggingr   ray._rayletr	   	getLoggerr1   r*   compileru   ry   r   getenvr
   rw   r   r   r   r6   r   ArgumentParserparseradd_argumentr]   LOGGER_LEVELLOGGER_LEVEL_CHOICESLOGGER_LEVEL_HELPLOGGER_FORMATLOGGER_FORMAT_HELPLOG_MONITOR_LOG_FILE_NAME
parse_argsargslogging_rotate_byteslogging_rotation_byteslogging_rotate_backup_countlogging_rotation_backup_countdictr   r   r8   logging_filenamelogging_params redirect_stdout_stderr_if_neededstdout_filepathstderr_filepathget_cached_node_ip_addresssession_dirnode_ipr<   r9   log_monitorr   r)   rl   utilsformat_error_message
format_exctraceback_strnodemessagepublish_error_to_driverLOG_MONITOR_DIED_ERRORr   r4   r   r   <module>r     s7           				  				  



      0 0 0 0 0 0 0 0 0 0 0 0 2 2 2 2 2 2 2 2 2 ( ( ( ( ( ( ( ( (     & & & & & & ; ; ; ; ; ; ! ! ! ! ! !
 
	8	$	$  RZ ?@@ &BJ'GHH  #U929-I3#O#OPP  (+sBI4d;;( ( $ ),BI5q99) ) %
E
 E
 E
 E
 E
 E
 E
 E
Pu  u  u  u  u  u  u  u p   z$X$R  F %c8W     *2,     +-     7737 7 7     O	     K	      ;	     '<	     7     7     D ;>,':Q:QT66WX,/LG,C,C(( " T(*&(2  N $#55n55F 3M2%	   2h1$2BCCG4#3444J*$  K   *??@T	@T@V@VWWA}x} A A1>A A 	 	220! 	3 	
 	
 	

 	WW s   L N%!A?N  N%