&`itddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z m Z m Z ddlZddlmcmZddlmZmZddlmZddlmZddlmZddlmZddlmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,dd l-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9ddl:m;Z;ddlZ>ej?e@ZAdeBfdZCGddZDdS)N)Any AsyncIteratorDictOptionalUnion)Timerrun_background_task)'NOSET_ASCEND_RT_VISIBLE_DEVICES_ENV_VAR)"NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR)get_event_logger) GcsClient) ActorHandle)Event)!DEFAULT_JOB_START_TIMEOUT_SECONDS,RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR%RAY_JOB_START_TIMEOUT_SECONDS_ENV_VAR4RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR)JOB_ACTOR_NAME_TEMPLATESUPERVISOR_ACTOR_RAY_NAMESPACEJobInfoJobInfoStorageClient)JobLogStorageClient) JobSupervisor)get_head_node_id)close_logger_file_descriptor)ActorDiedErrorActorUnschedulableErrorRuntimeEnvSetupError) JobErrorType JobStatus)RuntimeEnvConfig)NodeAffinitySchedulingStrategySchedulingStrategyTreturnctj}tttjtjzhdz }d||d}d|S)zReturns a job_id of the form 'raysubmit_XYZ'. Prefixed with 'raysubmit' to avoid confusion with Ray JobID (driver ID). >0IOlo)k raysubmit_) random SystemRandomlistsetstring ascii_lettersdigitsjoinchoices)randpossible_charactersid_parts y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/dashboard/modules/job/job_manager.pygenerate_job_idr</sw   D F 6= 011 # # # $ggdll#6"l==>>G ! ! !!ceZdZdZdZdZdZ d+dedede fdZ d ed e j fd Z d Zd ed eefd Z d+d edeefdZ d+d edeefdZd edeefdZ d,deeefdeded eeeffdZded efdZddddddddddedeedeeeefdeeeefdeeeefdeeeefdeed eeeefd!eed efd"Zd efd#Z d$Z!d e"fd%Z#d ed ee$fd&Z%d ed ee&fd'Z'd eee&ffd(Z(d ed efd)Z)d ed e*efd*Z+dS)- JobManagerzProvide python APIs for job submission and management. It does not provide persistence, all info will be lost if the cluster goes down. g?N gcs_clientlogs_dirtimeout_check_timerc>||_||_t|||_|j|_|j|_t|_ tj t|_|p t|_t#|_ t't(jj||_n#t0$r d|_YnwxYwt3j|_t7|dSN) _gcs_client _logs_dirr_job_info_clientaddress _gcs_address cluster_idhex_cluster_id_hexr _log_clientrayremoter_supervisor_actor_clsr_timeout_check_timerr2monitored_jobsr r SourceTypeJOBS event_logger Exceptionasyncio_recover_running_jobs_eventr _recover_running_jobs)selfrArBrCs r;__init__zJobManager.__init__Js&! 4Z J J&.)488::.00%(Z %>%>"$7$B577!!ee % 01A1F Q QD   % % % $D    %,3=??(D668899999s($C C! C!job_idr$c<tjtd|}|jsv|j|}tj|}tjtj }| || ||S)zReturn job driver logger to log messages to the job driver log file. If this function is called for the first time, configure the logger. z.driver-) logging getLogger__name__handlersrNget_log_file_path FileHandler Formatter ray_constants LOGGER_FORMAT setFormatter addHandler)r[r]job_driver_loggerjob_driver_log_pathjob_driver_handlerjob_driver_formatters r;_get_job_driver_loggerz!JobManager._get_job_driver_logger^s $-.K.K6.K.KLL!) ="&"2"D"DV"L"L !(!45H!I!I #*#4]5P#Q#Q  + +,@ A A A  ( (); < < <  r=cdK |jd{V}|D]@\}}|js"t ||A |jdS#|jwxYw)zRecovers all running jobs from the status client. For each job, we will spawn a coroutine to monitor it. Each will be added to self._running_jobs and reconciled. N) rH get_all_jobsitemsstatus is_terminalr _monitor_jobrYr2)r[all_jobsr]job_infos r;rZz JobManager._recover_running_jobsos 3!2??AAAAAAAAH$,NN$4$4 C C 2244C'(9(9&(A(ABBB C  , 0 0 2 2 2 2 2D , 0 0 2 2 2 2s A4BB/c tjtj|tS#t $rYdSwxYw)Nr]) namespace)rO get_actorrformatr ValueErrorr[r]s r;_get_actor_for_jobzJobManager._get_actor_for_jobsX ='.f===8    44 s -0 >>job_supervisorcK||jvr td|ddS|j| |||d{Vn0#t $r#}td||d}~wwxYw |j|dS#|j|wxYw)zMonitors the specified job until it enters a terminal state. This is necessary because we need to handle the case where the JobSupervisor dies unexpectedly. zJob z is already being monitored.Nz&Unhandled exception in job monitoring!exc_info)rSloggerdebugadd_monitor_job_internalrWerrorremove)r[r]res r;rtzJobManager._monitor_jobs T( ( ( LLDDDD E E E F ''' /,,V^DD D D D D D D D D    LLAAL N N NG  E   & &v . . . . .D  & &v . . . .s*A$#B1$ B.B  BB11C c~ Kttjtt }d}d}d} t j|jd{V|j |dd{V}|tj kr|"|j |dd{V}|j|jdz z |krd|dtd}|jduo |jdkpH|jduo |jdkp4|jduo |jdkp |jduot+|jdk}|r|dz }|j |tj|t0jd d{Vt4|ne|||}|k|tj krt4d |d|j |tjd t0jd d{Vn|-|jd  }tCj"|gd\} } | rtCj|d}nOny#tF$rk} |j |dd{V}d} d} ||$rnDtK| tLr0t4d|dd| } t0j'} ntK| tPr2t4d|d| d| } t0j)} n{tK| tTr2t4d|d| d| } t0j+} n4t4d|d| d| d| } t0j,} tj}|j ||| | p t0j,d d{V| r|j-.|}tj/tj01|dte|d5}|3| dddn #1swxYwY|j4rLd|d|d}| r%|d| z }|j4|| n|j45|| Yd} ~ n d} ~ wwxYw|||}|tCj6|d!dSdS)"NT)timeoutz,Job supervisor actor failed to start within zM seconds. This timeout can be configured by setting the environment variable .ra) This may be because the job entrypoint's specified resources (entrypoint_num_cpus, entrypoint_num_gpus, entrypoint_resources, entrypoint_memory)aren't available on the cluster. Try checking the cluster's available resources with `ray status` and specifying fewer resources for the job entrypoint.)message error_typerz%Failed to get job supervisor for job z8Unexpected error occurred: failed to get job supervisor.)max_task_retriesr+z%Failed to set up runtime_env for job zruntime_env setup failed: zFailed to schedule job z6 because the supervisor actor could not be scheduled: z-Job supervisor actor could not be scheduled: zJob supervisor actor for z died: zJob supervisor actor died: zJob monitoring for job z failed unexpectedly: rzUnexpected error occurred: )exist_okazCompleted a ray job z with a status   submission_id) no_restart)7floatosenvirongetrrrXsleepJOB_MONITOR_LOOP_PERIOD_SrH get_statusr PENDINGget_inforRtime start_timeentrypoint_num_cpusentrypoint_num_gpusentrypoint_memoryentrypoint_resourceslen put_statusFAILEDr"JOB_SUPERVISOR_ACTOR_START_TIMEOUTrrr~"JOB_SUPERVISOR_ACTOR_START_FAILUREpingoptionsrPrOwaitrWrs isinstancerRUNTIME_ENV_SETUP_FAILUREr"JOB_SUPERVISOR_ACTOR_UNSCHEDULABLErJOB_SUPERVISOR_ACTOR_DIED$JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURErNrcmakedirspathdirnameopenwriterVinfokill)r[r]rr job_statusrv ping_obj_referr_msgresources_specifiedready_rtarget_job_error_messagetarget_job_error_typelog_pathlog_file event_logs r;rz JobManager._monitor_job_internals JNN51      u t  mD$BCCCCCCCCC#'#8#C#CD$D$$ !222 ')-)>)G)G"D*H**$$$$$$ 166888;NQU;UU!""Q&QQ)NQQQ !) $B!) =T I!K$'(E$F$F$J,$/ #!2G#3>>"%,$+'3'V$( ? W---!)%)%<%>"%,!@(4'V$(?          '#1#6#>#>)+$?$$fhh!8\NA>>>qGL)))#'LL!K K K #'#8#C#CD$D$$ ,.(@D%)j.D.D.F.F)!!%9::" %VV%V%V%VWWW3SPQ3S3S00<0V--#A'>?? AfAA=>AA PAOO1)K.-$A~66 %S%S%SPQ%S%STTT3TQR3T3T00<0V-- 2f22-.222%&% 4UQR3T3T0(M."+!1J/::" 8#8$M'L $ ;,A#/AA&IIHK 9 9DIIIIh,,A '?@@@AAAAAAAAAAAAAAA$PSvSSjSSS0P!%C)A%C%CC )// /PPPP)..y.OOOWK Uu p  !!44V< F >r=Fuser_runtime_envrrc|tj|ni}|d}|i}d|tj<|sd|t <d|t <||d<tjtddkrH|d}|t}|j |g|d<||d<|S)aPConfigure and return the runtime_env for the supervisor actor. Args: user_runtime_env: The runtime_env specified by the user. resources_specified: Whether the user specified resources in the submit_job() call. If so, we will skip the workaround introduced in #24546 for GPU detection and just use the user's resource requests, so that the behavior matches that of the user specifying resources for any other actor. Returns: The runtime_env for the supervisor actor. Nenv_varsr&1config log_files) copydeepcopyrrfRAY_WORKER_NICENESSr r rgetenvrr!rNrc)r[rrr runtime_envrrs r;_get_supervisor_runtime_envz&JobManager._get_supervisor_runtime_envus*0@/KDM* + + +QS  ??:..  H6923" DHead node ID found in GCS; scheduling job driver on head node F)node_idsoft) rrrrrrrrFr")r[r head_node_idscheduling_strategys r;_get_scheduling_strategyz#JobManager._get_scheduling_strategys  9 :>>F L LPS S S KK?AAA    9 .d.>????????   KK'    #,   KK,),,   #A$5### #"r=)rrmetadatarrrr_start_signal_actor entrypointrrrrrrrc K|d}|d}|d}|t}|jd{Vtd|t |t jttj dz|||||| } |j || dd{V} | std|d | |} t|duo|dk|duo|dk|duo|dk|difvg} || d{V}|jr |jd |d | | d |jdt'j|||||||||| t,d |||pi|j|j|j}|j| | t9|||n#t<$r|}t?j }| !d|d|d||j "|t j#d|d|d|tHj%d{VYd}~nd}~wwxYwtM| n#tM| wxYw|S)a^ Job execution happens asynchronously. 1) Generate a new unique id for this job submission, each call of this method assumes they're independent submission with its own new ID, job supervisor actor, and child process. 2) Create new detached actor with same runtime_env as job spec Actual setting up runtime_env, subprocess group, driver command execution, subprocess cleaning up and running status update to GCS is all handled by job supervisor actor. Args: entrypoint: Driver command to execute in subprocess shell. Represents the entrypoint to start user application. runtime_env: Runtime environment used to execute driver command, which could contain its own ray.init() to configure runtime env at ray cluster, task and actor level. metadata: Support passing arbitrary data to driver command in case needed. entrypoint_num_cpus: The quantity of CPU cores to reserve for the execution of the entrypoint command, separately from any tasks or actors launched by it. Defaults to 0. entrypoint_num_gpus: The quantity of GPUs to reserve for the entrypoint command, separately from any tasks or actors launched by it. Defaults to 0. entrypoint_memory: The amount of total available memory for workers requesting memory the entrypoint command, separately from any tasks or actors launched by it. Defaults to 0. entrypoint_resources: The quantity of various custom resources to reserve for the entrypoint command, separately from any tasks or actors launched by it. _start_signal_actor: Used in testing only to capture state transitions between PENDING -> RUNNING. Regular user shouldn't need this. Returns: job_id: Generated uuid for further job management. Only valid within the same ray cluster. Nrz!Starting job with submission_id: r) rrrrrrrrrrF) overwritezJob with submission_id z6 already exists. Please use a different submission_id.zStarted a ray job rrzRuntime env is setting up.detachedrx) lifetimenamenum_cpusnum_gpusmemory resourcesrrryenable_task_events)rr)rz)Failed to start supervisor actor for job z: 'z'. Full traceback: z!Failed to start supervisor actor )rr)'r<rYrrrrr rintrrHput_infor|rnanyrrVrQrrr{rrrPrJrMrGrunr rtrW traceback format_excwarningrrrrr)r[rrrrrrrrrrv new_key_added driver_loggerrr supervisorrtb_strs r; submit_jobzJobManager.submit_jobs_j  &"#   &"#   $ !   +--M.33555555555 G GGHHH!$49;;-..# 3 3/!5    #3<< 8u=         8-888  33MBB C 8"%'t3O8Ka8O't3O8Ka8O%T1K6G!6K(r : ## )-(E(E#))######   !&&9999'   ; < < <3;;#,3=III,,(.$7 <<0C9$)<fB!$  . N ! !$7$7 "    !!- !KK       )++F  ! !/M//a//&,//   '22 3 33!33*033(J3               ) 7 7 7 7 ( 7 7 7 7s2,EH-,K- J37A2J.)K.J33KKcj||}||jdSdS)zeRequest a job to exit, fire and forget. Returns whether or not the job was running. NTF)r~stoprP)r[r]job_supervisor_actors r;stop_jobzJobManager.stop_jobys? $66v>>  + ! % , , . . .45r=cK|j|d{V}||std|d|d|j|d{VdS)z2Delete a job's info and metadata from the cluster.NzAttempted to delete job 'z%', but it is in a non-terminal state rT)rHrrs RuntimeError delete_info)r[r]rs r; delete_jobzJobManager.delete_jobs0;;FCCCCCCCC  Z%;%;%=%= CFCC5?CCC  #//777777777tr=c|jSrE)rHr[s r;job_info_clientzJobManager.job_info_clients $$r=cFK|j|d{VS)zGet latest status of a job.N)rHrr}s r;get_job_statuszJobManager.get_job_statuss/*55f=========r=cFK|j|d{VS)zGet latest info of a job.N)rHrr}s r; get_job_infozJobManager.get_job_infos/*33F;;;;;;;;;r=cDK|jd{VS)zGet info for all jobs.N)rHrprs r; list_jobszJobManager.list_jobss-*77999999999r=c6|j|S)zGet all logs produced by a job.)rNget_logsr}s r; get_job_logszJobManager.get_job_logss((000r=cK||d{Vtd|dd}|j|23d{V}|V|rdS||d{V}|rd}t j|jd{V_d|WVx6dS)z/Return an iterator following the logs of a job.NzJob 'z' does not exist.FTr+) rrrN tail_logsrsrXrLOG_TAIL_SLEEP_Sr6)r[r] job_finishedlinesrrs r; tail_job_logszJobManager.tail_job_logss)$$V,, , , , , , , 4@v@@@AA A +55f== % % % % % % %%},FF#'#6#6v#>#>>>>>>>F))++,'+ mD$9::::::::::ggenn$$$$$>==s CrE)F),ra __module__ __qualname____doc__r rWAIT_FOR_ACTOR_DEATH_TIMEOUT_Sr strrr\r_LoggerrnrZrrr~rtrrWrrrboolrr#rrrrrrrrrr rrrrr rrrr=r;r?r?=s !%("RV::#:/2:IN::::(!S!W^!!!!"333"+1FDH///+3K+@////,DHI6I6I6+3K+@I6I6I6I6V  hy>Q     %* 00sCx.00" 0 c3h 0000d.##'.# .#.#.#.#h(,04-1;?;?+/;?59```` } ` d38n- ` 4S>* `&eCJ&78`&eCJ&78`$C=`'tCJ'78`&k2` ````D $       %!5%%%%>3>8I3F>>>><<'1B<<<<:c7l!3::::13131111%#%-2D%%%%%%r=r?)ErXrr_rr/r3rrtypingrrrrrrOray._private.ray_constants_privaterfray._common.utilsrr ray._private.accelerators.npur $ray._private.accelerators.nvidia_gpur ray._private.event.event_loggerr ray._rayletr ray.actorrray.core.generated.event_pb2rray.dashboard.constsrrrr ray.dashboard.modules.job.commonrrrr0ray.dashboard.modules.job.job_log_storage_clientr(ray.dashboard.modules.job.job_supervisorrray.dashboard.modules.job.utilsrray.dashboard.utilsrray.exceptionsrrrray.job_submissionrr ray.runtime_envr!ray.util.scheduling_strategiesr"r#r`rarrr<r?rr=r;r+s  <<<<<<<<<<<<<< 22222222288888888QQQQQQSSSSSS<<<<<<!!!!!!!!!!!!......  QPPPPPBBBBBB<<<<<<<<<<<<XXXXXXXXXX66666666,,,,,,  8 $ $ " " " " " % % % % % % % % % %r=