&`i9prddlZddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z ddlZddlmZddlmZmZmZddlZddlmZddlmZddlmZmZmZmZdd lmZm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;mZ>m?Z?ddl@mAZAddlBmCZDddlEmFZFejGeHZIeIJejKGddeLejMZNGddeejOZPGddZQGddeAZRdS)N)datetime) AsyncIteratorDictOptionalTuple)ClientResponse)RequestResponseStreamResponse)NodeID) build_address) BaseModelExtraField validator)get_or_create_event_loop load_class)KV_NAMESPACE_DASHBOARD)package_existspin_runtime_env_uriupload_package_to_gcs)#DASHBOARD_AGENT_ADDR_NODE_ID_PREFIXGCS_RPC_TIMEOUT_SECONDSRAY_CLUSTER_ACTIVITY_HOOK&TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDSWAIT_AVAILABLE_AGENT_TIMEOUT)JobDeleteResponseJobInfoStorageClientJobLogsResponseJobStopResponseJobSubmitRequestJobSubmitResponsehttp_uri_components_to_uri) JobDetailsJobType)find_job_by_idsget_driver_jobsget_head_node_idparse_and_validate_request)CURRENT_VERSIONVersionResponse)SubprocessModule)SubprocessRouteTable) ResponseTypeceZdZdZdZdZdS)RayActivityStatusACTIVEINACTIVEERRORN)__name__ __module__ __qualname__r1r2r3v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/dashboard/modules/job/job_head.pyr0r0;s FH EEEr8r0ceZdZUdZeddZeed<eddZe e ed<edd Z e ed <edd Z e e ed <edd dZdS)RayActivityResponsez Pydantic model used to inform if a particular Ray component can be considered active, and metadata about observation. .zWhether the corresponding Ray component is considered active or inactive, or if there was an error while collecting this observation.) description is_activeNz8Reason if Ray component is considered active or errored.reasonzwTimestamp of when this observation about the Ray component was made. This is in the format of seconds since unix epoch. timestampzTimestamp when last actvity of this Ray component finished in format of seconds since unix epoch. This field does not need to be populated for Ray components where it is not meaningful.last_activity_atT)alwaysc \d|vr'|dtjkr|td|S)Nr=z6Reason is required if is_active is "active" or "error")r0r2 ValueError)clsvvalueskwargss r9reason_requiredz#RayActivityResponse.reason_requiredasA & VK%8rstrr?floatr@rrHr7r8r9r;r;As $)5  J$$$I "E TFHSMu  AIu).  =)))huoYx%%%&%r8r;)extraceZdZdZdefdZdefdZdede fdZ d ede fd Z d ede fd Zd edefd Zd edeefd ZddZdS)JobAgentSubmissionClientzA local client for submitting and interacting with jobs on a specific node in the remote cluster. Submits requests over HTTP to the job agent on the specific node using the REST API. dashboard_agent_addresscD||_tj|_dSN)_agent_addressaiohttp ClientSession_session)selfrPs r9__init__z!JobAgentSubmissionClient.__init__qs 6-// r8respctK|j}|d{V}td|d|d)Nz Request failed with status code z: .)statustext RuntimeError)rWrYr\ error_texts r9 _raise_errorz%JobAgentSubmissionClient._raise_errorxsN99;;&&&&&& UfUU UUUVVVr8reqreturncKtd|jd|j|jdt j|4d{V }|jdkr8| d{V}tdi|cdddd{VS| |d{V dddd{VdS#1d{VswxYwYdS)Nz"Submitting job with submission_id=r[/api/job_agent/jobs/)jsonr7) loggerdebug submission_idrVpostrS dataclassesasdictr\rer"r`)rWrarY result_jsons r9submit_job_internalz,JobAgentSubmissionClient.submit_job_internal}s N#:KNNNOOO=%%" 8 8 8{?QRU?V?V&   . . . . . . . . {c!!$(IIKK////// (77;77  . . . . . . . . . . . . . .''---------- . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .s#1C'C C #C job_idcKtd|d|j|jd|d4d{V }|jdkr8|d{V}tdi|cdddd{VS||d{V dddd{VdS#1d{VswxYwYdS)NzStopping job with job_id=r[rdz/stoprfr7) rgrhrVrjrSr\rer r`rWrorYrms r9stop_job_internalz*JobAgentSubmissionClient.stop_job_internals ::::;;;=%%" E E E E E   . . . . . . . . {c!!$(IIKK////// &5555  . . . . . . . . . . . . . .''---------- . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .s 1CC C  C cKtd|d|j|jd|4d{V }|jdkr8|d{V}tdi|cdddd{VS||d{V dddd{VdS#1d{VswxYwYdS)NzDeleting job with job_id=r[rdrfr7) rgrhrVdeleterSr\rerr`rqs r9delete_job_internalz,JobAgentSubmissionClient.delete_job_internals ::::;;;=''" @ @ @ @   . . . . . . . . {c!!$(IIKK////// (77;77  . . . . . . . . . . . . . .''---------- . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .s 1B?B?? C  C cjK|j|jd|d4d{V }|jdkr8|d{V}t di|cdddd{VS||d{V dddd{VdS#1d{VswxYwYdS)Nrdz/logsrfr7)rVgetrSr\rerr`rqs r9get_job_logs_internalz.JobAgentSubmissionClient.get_job_logs_internals=$$" E E E E E   . . . . . . . . {c!!$(IIKK////// &5555  . . . . . . . . . . . . . .''---------- . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .s1B"3B"" B,/B,cLK|j|jd|dd{V} |d{V}|jt jjkr |jWVn7|jt jj krdS|jt jj kr w)z/Get an iterator that follows the logs of a job.rdz /logs/tailN) rV ws_connectrSreceivetyperT WSMsgTypeTEXTdataCLOSEDr3)rWrowsmsgs r9 tail_job_logsz&JobAgentSubmissionClient.tail_job_logss=++" J J J J J          $$$$$$Cx7,111hW.555W.444 r8TcrK |jd{VdS#t$r|sYdSwxYwrR)rVclose Exception)rW ignore_errors r9rzJobAgentSubmissionClient.closesl -%%'' ' ' ' ' ' ' ' ' '        s % 66N)T)r4r5r6rIrKrXrr`r!r"rnr rrrrurrxrrrr7r8r9rOrOks  0!$0000W~WWWW .-= .BS . . . . .c .o . . . . . .8I . . . ..#./....#-2D r8rOcteZdZdZdZfdZefdedefdZ defdZ dedefdZ d e de eeeffd Zejd d edefd Zejdd edefdZejdd efdZejdd edefdZejdd edefdZejdd edefdZejdd edefdZejdd edefdZejdd edefdZ ejde!j"d ede#fdZ$de%de&efdZ'ejd d e(j)jde(j)jfd!Z*d"ede+fd#Z,fd$Z-xZ.S)%JobHeadaRuns on the head node of a Ray cluster and handles Ray Jobs APIs. NOTE(architkulkarni): Please keep this class in sync with the OpenAPI spec at `doc/source/cluster/running-applications/job-submission/openapi.yml`. We currently do not automatically check that the OpenAPI spec is in sync with the implementation. If any changes are made to the paths in the @route decorators or in the Responses returned by the methods (or any nested fields in the Responses), you will need to find the corresponding field of the OpenAPI yaml file and update it manually. Also, bump the version number in the yaml file and in this class's `get_version`. ctj|i|d|_|jJtjjsJt|_ dSrR) superrX_job_info_client gcs_clientray experimental internal_kv_internal_kv_initializeddict_agents)rWargsrG __class__s r9rXzJobHead.__init__sh$)&))) $***+DDFFFFF@Dvv r8 timeout_srbc<K||d{VS)at Get a `JobAgentSubmissionClient`, which is a client for interacting with jobs via an agent process. Args: timeout_s: The timeout for the operation. Returns: A `JobAgentSubmissionClient` for interacting with jobs via an agent process. Raises: TimeoutError: If the operation times out. N)_get_head_node_agent)rWrs r9get_target_agentzJobHead.get_target_agents. ..y999999999r8cBKt|jd{V}|stdtj|}||jvrI||d{V\}}}dt||}t||j|<|j|S)Nz.Head node id has not yet been persisted in GCSzhttp://) r(rrr from_hexr_fetch_agent_infor rO)rWhead_node_id_hex head_node_idip http_port_agent_http_addresss r9_get_head_node_agent_oncez!JobHead._get_head_node_agent_onces!1$/!B!BBBBBBB NLMM M'788 t| + +%)%;%;L%I%IIIIIII B 1!I=Y+G+G!I!I )ABT)U)UDL &|L))r8cKtj|z}d}tj|kr |d{VS#t$rN}|}tdt dt jt d{VYd}~nd}~wwxYwtj|ktd|d|)aRetrieves HTTP client for `JobAgent` running on the Head node. If the head node does not have an agent, it will retry every `TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS` seconds indefinitely. Args: timeout_s: The timeout for the operation. Returns: A `JobAgentSubmissionClient` for interacting with jobs via the head node's agent process. Raises: TimeoutError: If the operation times out. Nz+Failed to get head node agent, retrying in z seconds...z%Failed to get head node agent within z seconds. The last exception is ) timerrrg exceptionrasynciosleep TimeoutError)rWr timeout_pointres r9rzJobHead._get_head_node_agents i/  ikkM)) L!;;========= L L L   uBhuuum$JKKKKKKKKKKKKKK  LikkM)) jI j j_h j j   sA B%AB  B%target_node_idcKt|}|j|tt d{V}|st d|dtj| S)z Fetches agent info by the Node ID. May raise exception if there's network error or the agent info is not found. Returns: (ip, http_port, grpc_port) ) namespacetimeoutNz-Agent info not found in internal KV for node z. It's possible that the agent didn't launch successfully due to port conflicts or other issues. Please check `dashboard_agent.log` for more details.) rhexrasync_internal_kv_getrrKeyErrorreloadsdecode)rWrkeyvalues r9rzJobHead._fetch_agent_info"s5Ln6H6H6J6JLLo;; ,+<          $$$$  z%,,..)))r8z /api/versionracKtttjtj|j}t tjtj |dtj j jS)N)version ray_version ray_commit session_nameapplication/jsonr] content_typer\)r+r*r __version__ __commit__rr redumpsrkrlrTwebHTTPOk status_code)rWrarYs r9 get_versionzJobHead.get_version8sn#~*     K.t4455+;%1    r8z'/api/packages/{protocol}/{package_name}cKt|jd|jd}td|d t |nF#t $r9t tjtj j j cYSwxYwt|s)t d|dtj jj St S) Nprotocol package_namerrz&Adding temporary reference to package r[r]r\zPackage  does not exist)r# match_inforgrhrrr traceback format_excrTrHTTPInternalServerErrorrr HTTPNotFound)rWra package_uris r9 get_packagezJobHead.get_packageHs0^J/7    LkLLLMMM   , , , ,   )++{:F     k** < <<<{/;  zzs AABBcKt|jd|jd}td|d |d{V}t dt||d{VnF#t$r9ttj tj jjcYSwxYwttj jjS)NrrrzUploading package z to the GCS.r)r\)r#rrginforeadrrun_in_executorrrr rrrTrrrr)rWrarrs r9upload_packagezJobHead.upload_package`s/0^J/7     BBBBCCC ######D*,,<<%             )++{:F     w{1=>>>>s A BACCz /api/jobs/cKt|td{V}t|tr|S|} |d{V}||d{V}n#t j$r(tdtj j j cYSttf$r9ttjtj jj cYSt"$r9ttjtj j j cYSwxYwtt%jt)j|dtj jj S)Nz9No available agent to submit job, please try again later.rrr)r)r! isinstancer rrnrrrTrrr TypeErrorrCrrHTTPBadRequestrrerrkrlr)rWraresultsubmit_requestjob_agent_clientrYs r9 submit_jobzJobHead.submit_jobws1#7GHHHHHHHH fh ' ' 6M/5N %)%:%:%<%<<<<<<< )==nMMMMMMMMDD#   P{:F   :&   )++{1=      )++{:F     K.t4455+;%1    s5A..4D.$AD.,?D.-D.z%/api/jobs/{job_or_submission_id}/stopcK|jd}t|j|j|d{V}|s)t d|dt jjjS|j tj ur%t dt jj jS | d{V}||jd{V}nF#t $r9t t#jt jjjcYSwxYwt t)jt-j|dS)Njob_or_submission_idJob rrz"Can only stop submission type jobsrr]r)rr&rrr rTrrrr|r% SUBMISSIONrrrrrirrrrrerrkrlrWrarjobrrYs r9stop_jobzJobHead.stop_jobs"~.DE# O  !           A0AAA{/;  87- - -9{1=   %)%:%:%<%<<<<<<< );;COPPPPPPPPDD   )++{:F     K.t4455DV    rc"K|jd}t|j|j|d{V}|s)t d|dt jjjSt tj | dS)Nrrrrrr) rr&rrr rTrrrrerr)rWrarrs r9 get_job_infozJobHead.get_job_infos"~.DE# O  !           A0AAA{/;  CHHJJ''+    r8cvKtjt|j|jd{V\\}}fd|D}ttj gd|Dd| DdS)Nc g|]e\}}tditj|||vr|jnd|t jdfS)N)riro driver_infor|r7)r$rkrlrwidr%r).0rirsubmission_job_driverss r9 z%JobHead.list_jobs..s    # s   $S)) + $:::.11-@@CC266}EE'        r8c6g|]}|Sr7r)rsubmission_jobs r9rz%JobHead.list_jobs..s$RRRn))++RRRr8c6g|]}|Sr7r)rjob_infos r9rz%JobHead.list_jobs..s KKK(hmmooKKKr8rr) rgatherr'rr get_all_jobsitemsr rerrF)rWra driver_jobssubmission_jobsrs @r9 list_jobszJobHead.list_jobssGN~ DO , ,d.C.P.P.R.RH H B B B B B B >-,     '6&;&;&=&=    RR/RRRKKk6H6H6J6JKKK ,    r8z%/api/jobs/{job_or_submission_id}/logscK|jd}t|j|j|d{V}|s)t d|dt jjjS|j tj ur%t dt jj jS | |}|r ||jd{Vnt!d}t t#jt'j|dS#t*$r9t t-jt jjjcYSwxYw) Nrrrr)Can only get logs of submission type jobsrr)rr&rrr rTrrrr|r%rrget_job_driver_agent_clientrxrirrerrkrlrrrr)rWrarrrpayloads r9 get_job_logszJobHead.get_job_logs s"~.DE# O  !           A0AAA{/;  87- - -@{1=   #??DD $)&<r?z-Failed to get activity status from user hook external_componentrrr7)queryrwisdigitint_get_job_activity_inforrosenvironrrr;rrgrr0r3reprrnowr?rTrr rerrr) rWrardriver_activity_inforYcluster_activity_callableexternal_activity_outputcomponent_typecomponent_activity_outputrs r9get_component_activitiesz JobHead.get_component_activitiesms)-- 400  w(( 'llGGG&*%@%@%@%Q%QQQQQQQ$3445 $ 2 2' ,6J89--),E+D+F+F(!":DAA:bj1J&K::7::A '?N4L*515H55"#<==551044M/N/N^,,$   ((WWW.0j9R.SWWW *;)@&*1gg)1)A)A)C)C00^,,,,,, (     EJ'@AEEE "3!8"1gg!)!9!9!;!;..)****** {##D!!+;%1$   sEA,F:53D)(F:) F63A9F1,F:1F66F:: IA6H??Irc6K |jdd|d{V}d}d}|D]\}t|j}|jjd}|jrt||jn|}|s|s|dz }]tj }||kr|dz }||ksJd|d||dkr tjn tj} t!| |rd |nd||r|nd S#t"$rp} t$d t!tjt+| tj cYd} ~ Sd} ~ wwxYw) NT)skip_submission_job_info_fieldskip_is_running_tasks_fieldrr_ray_internal_rizMost recent job end time z2 must be before or equal to the current timestamp zNumber of active drivers: )r=r>r?r@z-Failed to get activity status of Ray drivers.r)rasync_get_all_job_inforFboolis_deadconfig ray_namespace startswithend_timemaxrrr?r0r1r2r;rrgrr3r) rWrreplynum_active_driverslatest_job_end_timejob_table_entryr*in_internal_namespacecurrent_timestampr=rs r9rzJobHead._get_job_activity_infosQ 6 /@@/3,0AE "# "# #(<<>> , ,677(7(>(L(W(W$))% '/-C+_-EFFF,$ ,'<,&!+& (  8 8 : : #%666&9D&@#(,????T0CTT@QTT@??&))"((&/  '#%H4FHHH+9L!U!4!4QU          L M M M&+1Aww",..2244       sDD F(A%F FFcKtd{V|jst|j|_dSdSrR)rrunrrr)rWrs r9r7z JobHead.runs[ggkkmm$ J$8$I$ID ! ! ! J Jr8)/r4r5r6rIr rXrrLrOrrrr rrKrrroutesrwr r rrputrrjrrrtrrrrr. WEBSOCKETr rr$rrrTrr#r;rr7 __classcell__)rs@r9rrs  ,-( F F F F F ">::: !::::$ *1I * * * * E >V    :*f*sCQT}AU****,VZ  W         VZ9::W;:.VZ9::????;:?,V[ G     BV[899 ' h   :9 >V]566 G    76 >VZ233 g (   43 *VZ  7 x    6VZ788"g"("""98"HVZ4 @V*w*>****X , , * + , , , ,VZ+,,; ;&;  ; ; ; -,; z<C<rSs  777777777777))))))9999999999 333333JJJJJJJJJJJJBBBBBBBB====== JIIIIIII KJJJJJJJ>>>>>>LLLLLL999999  8 $ $ TY ''''')5;''''TUUUUUUUUpiJiJiJiJiJiJiJiJiJiJr8