&`ivS\ddlZddlZddlmZmZmZmZmZmZddl Z ddl Z ddl m Z ddlmZmZmZmZmZmZddlmZddlmZddlmZddlmZdd lmZdd lm Z  ddl!Z!ddl"Z"n#e#$rdZ!dZ"YnwxYwej$e%Z&e&'ej(Gd d e Z)dS) N)Any AsyncIteratorDictListOptionalUnion)SubmissionClient)JobDeleteResponseJobLogsResponse JobStatusJobStopResponseJobSubmitRequestJobSubmitResponse JobDetails)strip_keys_with_value_none)!get_address_for_submission_client) RuntimeEnv)_validate_no_local_paths) PublicAPIc^eZdZdZ d deededeeeefdeeeefd eeeefd ee eeff fd Z e d ddddddddddedeedeeeefdeeeefdeedee e e fdee e e fdee deeee fdefdZe d dedefdZe d dedefdZe d dedefdZe d deefdZe d dedefdZe d dedefdZe d dedeefdZxZS)!JobSubmissionClienta,A local client for submitting and interacting with jobs on a remote cluster. Submits requests over HTTP to the job server on the cluster using the REST API. Args: address: Either (1) the address of the Ray cluster, or (2) the HTTP address of the dashboard server on the head node, e.g. "http://:8265". In case (1) it must be specified as an address that can be passed to ray.init(), e.g. a Ray Client address (ray://:10001), or "auto", or "localhost:". If unspecified, will try to connect to a running local Ray cluster. This argument is always overridden by the RAY_API_SERVER_ADDRESS or RAY_ADDRESS environment variable. create_cluster_if_needed: Indicates whether the cluster at the specified address needs to already be running. Ray doesn't start a cluster before interacting with jobs, but third-party job managers may do so. cookies: Cookies to use when sending requests to the HTTP job server. metadata: Arbitrary metadata to store along with all jobs. New metadata specified per job will be merged with the global metadata provided here via a simple dict update. headers: Headers to use when sending requests to the HTTP job server, used for cases like authentication to a remote cluster. verify: Boolean indication to verify the server's TLS certificate or a path to a file or directory of trusted certificates. Default: True. NFTaddresscreate_cluster_if_neededcookiesmetadataheadersverifycjtj|_ tt d|4t |t stdt|t |tstdt||4t |tstdt||4t |tstdt||4t |tstdt|t |t s4t |tstdt|t|}t |||||||d d tj|jtjd kr"|d d |jd dSdS)NZThe Ray jobs CLI & SDK require the ray[default] installation: `pip install 'ray[default]'`zaddress must be a string, got z-create_cluster_if_needed must be a bool, got zcookies must be a dict, got zmetadata must be a dict, got zheaders must be a dict, got z"verify must be a str or bool, got )rrrrrrz1.9zeJobs API is not supported on the Ray cluster. Please ensure the cluster is running Ray 1.9 or higher. min_versionversion_error_messagez2.0zClient Ray version z is not compatible with the Ray cluster. Please ensure the cluster is running Ray 2.0 or higher or downgrade the client Ray version.)ray __version___client_ray_versionrequests RuntimeError isinstancestr TypeErrortypebooldictrsuper__init___check_connection_and_version packagingversionparse) selfrrrrrrapi_server_url __class__s q/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/dashboard/modules/job/sdk.pyr0zJobSubmissionClient.__init__?st$'? W  =   z'3'?'? LT']]LLMM M2D99 512255   z'4'@'@ J4==JJKK K   8T(B(B LDNNLLMM M  z'4'@'@ J4==JJKK K63'' Q:fd+C+C QOfOOPP P:7CC "%=     **#) +      " "4#; < r?r@rArBreturnc |rtd|s|s| r|dd|r|dd|pi}|pi}||j|||||d} | r5t| ts tdt| d tdi|}t||}|p|}t|||||||| } t!t#j| } td |d |d d| } | jdkr#t-di| jS|| dS)aM Submit and execute a job asynchronously. When a job is submitted, it runs once to completion or failure. Retries or different runs with different parameters should be handled by the submitter. Jobs are bound to the lifetime of a Ray cluster, so if the cluster goes down, all running jobs on that cluster will be terminated. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> client.submit_job( # doctest: +SKIP ... entrypoint="python script.py", ... runtime_env={ ... "working_dir": "./", ... "pip": ["requests==2.26.0"] ... } ... ) # doctest: +SKIP 'raysubmit_4LamXRuQpYdSMg7J' Args: entrypoint: The shell command to run for this job. submission_id: A unique ID for this job. runtime_env: The runtime environment to install and run this job in. metadata: Arbitrary data to store along with this job. job_id: DEPRECATED. This has been renamed to submission_id entrypoint_num_cpus: The quantity of CPU cores to reserve for the execution of the entrypoint command, separately from any tasks or actors launched by it. Defaults to 0. entrypoint_num_gpus: The quantity of GPUs to reserve for the execution of the entrypoint command, separately from any tasks or actors launched by it. Defaults to 0. entrypoint_memory: The quantity of memory to reserve for the execution of the entrypoint command, separately from any tasks or actors launched by it. Defaults to 0. entrypoint_resources: The quantity of custom resources to reserve for the execution of the entrypoint command, separately from any tasks or actors launched by it. Returns: The submission ID of the submitted job. If not specified, this is a randomly generated unique ID. Raises: RuntimeError: If the request to the job server fails, or if the specified submission_id has already been used by a job on this cluster. z=job_id kwarg is deprecated. Please use submission_id instead.z2.2z`entrypoint_num_cpus`, `entrypoint_num_gpus`, and `entrypoint_resources` kwargs are not supported on the Ray cluster. Please ensure the cluster is running Ray 2.2 or higher.r!z2.8zv`entrypoint_memory` kwarg is not supported on the Ray cluster. Please ensure the cluster is running Ray 2.8 or higher.worker_process_setup_hookz Invalid type z for `worker_process_setup_hook`. When a job submission API is used, `worker_process_setup_hook` only allows a string type (module name). Specify `worker_process_setup_hook` via ray.init within a driver to use a `Callable` type. )rCr>r=rr?r@rArBz"Submitting job with submission_id=.POST /api/jobs/) json_dataN)loggerwarningr1update_default_metadata_upload_working_dir_if_needed_upload_py_modules_if_neededgetr)r* ValueErrorr,rrto_dictrr dataclassesasdictdebug _do_request status_coderjsonr> _raise_error)r5rCr<r=rr>r?r@rArB setup_hookreqrJrs r8 submit_jobzJobSubmissionClient.submit_job}sIx   NNO     "5 9M   . .!'- /       . .!'- /   "'R >r./// **;777 ))+666!__%@AA  jS99 FZ 0 0FFF !//;//  ---!))++ %/ !'# 3 3/!5    /{/A#/F/FGG  J-JJJKKK   V\Y  G G =C  $00qvvxx00> >   a r9ctd|d|dd|d}|jdkr#t di|jS||dS) aRequest a job to exit asynchronously. Attempts to terminate process first, then kills process after timeout. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> sub_id = client.submit_job(entrypoint="sleep 10") # doctest: +SKIP >>> client.stop_job(sub_id) # doctest: +SKIP True Args: job_id: The job ID or submission ID for the job to be stopped. Returns: True if the job was running, otherwise False. Raises: RuntimeError: If the job does not exist or if the request to the job server fails. zStopping job with job_id=rGrHrIz/stoprKNrL)rMrXrYrZr r[stoppedr\r5r<r_s r8stop_jobzJobSubmissionClient.stop_jobs4  ::::;;;   V%?&%?%?%? @ @ =C  "..QVVXX..6 6   a r9ctd|d|dd|}|jdkr#t di|jS||dS)aDelete a job in a terminal state and all of its associated data. If the job is not already in a terminal state, raises an error. This does not delete the job logs from disk. Submitting a job with the same submission ID as a previously deleted job is not supported and may lead to unexpected behavior. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient() # doctest: +SKIP >>> job_id = client.submit_job(entrypoint="echo hello") # doctest: +SKIP >>> client.delete_job(job_id) # doctest: +SKIP True Args: job_id: submission ID for the job to be deleted. Returns: True if the job was deleted, otherwise False. Raises: RuntimeError: If the job does not exist, if the request to the job server fails, or if the job is not in a terminal state. zDeleting job with job_id=rGDELETErIrKNrL)rMrXrYrZr r[deletedr\rcs r8 delete_jobzJobSubmissionClient.delete_job!s:  ::::;;;   X'>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> submission_id = client.submit_job(entrypoint="sleep 1") # doctest: +SKIP >>> client.get_job_info(submission_id) # doctest: +SKIP JobDetails(status='SUCCEEDED', job_id='03000000', type='submission', submission_id='raysubmit_4LamXRuQpYdSMg7J', message='Job finished successfully.', error_type=None, start_time=1647388711, end_time=1647388712, metadata={}, runtime_env={}) Args: job_id: The job ID or submission ID of the job whose information is being requested. Returns: The JobDetails for the job. Raises: RuntimeError: If the job does not exist or if the request to the job server fails. GETrIrKNr rL)rYrZrr(r[r\rcs r8 get_job_infoz JobSubmissionClient.get_job_infoFs:   U$9$9$9 : : =C  !"A "--AFFHH---   a r9c|dd}|jdkr"|}d|D}|S||dS)aWList all jobs along with their status and other information. Lists all jobs that have ever run on the cluster, including jobs that are currently running and jobs that are no longer running. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> client.submit_job(entrypoint="echo hello") # doctest: +SKIP >>> client.submit_job(entrypoint="sleep 2") # doctest: +SKIP >>> client.list_jobs() # doctest: +SKIP [JobDetails(status='SUCCEEDED', job_id='03000000', type='submission', submission_id='raysubmit_4LamXRuQpYdSMg7J', message='Job finished successfully.', error_type=None, start_time=1647388711, end_time=1647388712, metadata={}, runtime_env={}), JobDetails(status='RUNNING', job_id='04000000', type='submission', submission_id='raysubmit_1dxCeNvG1fCMVNHG', message='Job is currently running.', error_type=None, start_time=1647454832, end_time=None, metadata={}, runtime_env={})] Returns: A list of JobDetails containing the job status and other information. Raises: RuntimeError: If the request to the job server fails. rjrIrKc&g|]}tdi|S)rLr).0 job_info_jsons r8 z1JobSubmissionClient.list_jobs..s30= ++]++r9N)rYrZr[r\)r5r_jobs_info_json jobs_infos r8 list_jobszJobSubmissionClient.list_jobspss<   UL 1 1 =C  VVXXNAOI    a r9c6||jS)aGet the most recent status of a job. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> client.submit_job(entrypoint="echo hello") # doctest: +SKIP >>> client.get_job_status("raysubmit_4LamXRuQpYdSMg7J") # doctest: +SKIP 'SUCCEEDED' Args: job_id: The job ID or submission ID of the job whose status is being requested. Returns: The JobStatus of the job. Raises: RuntimeError: If the job does not exist or if the request to the job server fails. )rkstatus)r5r<s r8get_job_statusz"JobSubmissionClient.get_job_statuss,  ((//r9c|dd|d}|jdkr#tdi|jS||dS)aGet all logs produced by a job. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> sub_id = client.submit_job(entrypoint="echo hello") # doctest: +SKIP >>> client.get_job_logs(sub_id) # doctest: +SKIP 'hello\n' Args: job_id: The job ID or submission ID of the job whose logs are being requested. Returns: A string containing the full logs of the job. Raises: RuntimeError: If the job does not exist or if the request to the job server fails. rjrIz/logsrKNrL)rYrZr r[logsr\rcs r8 get_job_logsz JobSubmissionClient.get_job_logssk,   U$>$>$>$> ? ? =C  "..QVVXX..3 3   a r9cKtj|j|j4d{V }||jd|d|jd{V} |d{V}|jtj j kr |j WVn|jtj j krWtd|d|j|jtjjkrt%d |jnN|jtj jkr3td |d |n dddd{VdS#1d{VswxYwYdS) aGet an iterator that follows the logs of a job. Example: >>> from ray.job_submission import JobSubmissionClient >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP >>> submission_id = client.submit_job( # doctest: +SKIP ... entrypoint="echo hi && sleep 5 && echo hi2") >>> async for lines in client.tail_job_logs( # doctest: +SKIP ... 'raysubmit_Xe7cvjyGJCyuCvm2'): ... print(lines, end="") # doctest: +SKIP hi hi2 Args: job_id: The job ID or submission ID of the job whose logs are being requested. Returns: The iterator. Raises: RuntimeError: If the job does not exist, if the request to the job server fails, or if the connection closes unexpectedly before the job reaches a terminal state. )rrNrIz /logs/tail)sslTzWebSocket closed for job z with close code z9WebSocket connection closed unexpectedly with close code zWebSocket error for job z!, treating as normal close. Err: )aiohttp ClientSession_cookies_headers ws_connect_address _ssl_contextreceiver, WSMsgTypeTEXTdataCLOSEDrMrX close_code WSCloseCodeABNORMAL_CLOSUREr(ERROR exception)r5r<sessionwsmsgs r8 tail_job_logsz!JobSubmissionClient.tail_job_logssT6(M4=            ))=>>F>>>DDU*B JJLL((((((8w0555(NNNNNX!2!999LL\F\\R]\\}(;(LLL*gXZXeggX!2!888LLl6ll\^\h\h\j\jll'                                sD-E++ E58E5)NFNNNT)__name__ __module__ __qualname____doc__rr*r-rrrr0rintfloatr`rdrhrrkrrsr rvryrr __classcell__)r7s@r8rr$s8"&).,0-1,0-1<<#<#'<$sCx.) < 4S>* < $sCx.) <sDy)*<<<<<<|Y""" !%04-1'+;?;?+/;?!!!! ! d38n- ! 4S>* ! }!&eCJ&78!&eCJ&78!$C=!'tCJ'78! !!!#"!BY"""!! !!!#"!BY""""!"! "!"!"!#""!HY"""'!'! '!'!'!#"'!RY"""&!4 +&!&!&!#"&!PY"""0S0Y000#"0.Y"""!3!3!!!#"!8Y"""4#4-2D444#"44444r9r)*rVloggingtypingrrrrrrpackaging.versionr2r$#ray.dashboard.modules.dashboard_sdkr ray.dashboard.modules.job.commonr r r r rr)ray.dashboard.modules.job.pydantic_modelsrray.dashboard.modules.job.utilsrray.dashboard.utilsrray.runtime_envrray.runtime_env.runtime_envrray.util.annotationsrr|r' ImportError getLoggerrrMsetLevelINFOrrLr9r8rsBBBBBBBBBBBBBBBB @@@@@@A@@@@@FFFFFFAAAAAA&&&&&&@@@@@@******NNNOOOOGHHH  8 $ $ _____*_____sA%% A10A1