
    &`iA              R       &   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d dlm Z m!Z! d dl"m#Z# d d	l$m%Z% d d
l&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/ d dl0m1Z1m2Z2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z:m;Z;m<Z<m=Z=m>Z> d dl?m@Z@ d dlAmBZBmCZCmDZDmEZE d dlFmGZH d dlImJZJmKZKmLZLmMZMmNZNmOZO d dlPmGZQ d dlRmSZS d dlTmUZU d dlVmWZW d dlXmYZY d dlZm[Z[ d dl\m]Z]m^Z^m_Z_ d dl`maZa d dlbmcZc er
d dldZed dl0m:Zf  ejg        eh          Zidee-ejeef         d eeeW                  fd!Zkd"eejelf         d eeY         fd#Zmdee-ejeef         d elfd$Zndee-ejeef         d elfd%Zod&eeee)f                  d elfd'Zp	 dcd)e+d*e:d+elfd,Zq	 dcd)e+d*d-d.elfd/Zrd ejs        fd0Ztd1ejfd2Zu G d3 d4e jv                  Zweadddddddd5dddddddd(ddddd d(dddd(d6ddddd d(edddejx        d7&d8eejeef         d9eej         d:eej         d;eej         d<eeeeUeejegelf         f                  d=eeeyezej{        f                  d>eeejef                  d?edeejeezeyef         f         e)f         d@eydAeej         dBedC         dDeeeMeLejf                  dEeeeEejf                  dFee          dGeeeye2e]f                  dHee:         dIeldJeee.gejf                  dKeee.gejf                  dLee!         dMee         dNeydOeldPeej         d"eeelejf                  dQeeY         dReldSeldTeee%                  dUeey         dVeey         dWeej         dXeydYeldZeld[eej         d\eel         d]eec         d^ed e#fPd_            Z|ea	 	 	 	 	 	 	 	 	 	 ddd`ee-eeee-ef                  f         dEeeE         dGeeeye2e]f                  dHee:         d"eeelejf                  dQeeY         dReldSeldaeldTeee%                  d\eel         fdb            Z}dS )e    N)	TYPE_CHECKINGAnyCallableDictMappingOptionalSequenceTypeUnion)usage)AirEntrypoint)_force_on_current_node)_DEPRECATED_VALUERAY_CHDIR_TO_TRIAL_DIR)CheckpointConfig
SyncConfig)ExperimentAnalysis)Callback)	TuneError)PlacementGroupFactory)TuneController)
ExperimentTrial_convert_to_experiment_list)IS_NOTEBOOKAirVerbosityget_air_verbosity)create_resolvers_mapinject_placeholders)TBXLoggerCallback)ProgressReporter_detect_progress_metrics_detect_reporter)_prepare_progress_reporter_for_ray_client_stream_client_output)get_trainable_cls)FIFOSchedulerPopulationBasedTrainingPopulationBasedTrainingReplayTrialScheduler)+_set_search_properties_backwards_compatible)BasicVariantGeneratorConcurrencyLimiterSearchAlgorithmSearcherSearchGeneratorcreate_searcher)_has_unresolved_values)Stopper)	Trainable)ResumeConfig)_create_default_callbacks)	Verbosityhas_verbosityset_verbosity)	PublicAPI)Queuer!   run_identifierreturnc                    t          | t                    r| j        } t          | t                    rt	          | t
                    sd S | }n9t          |           r| }n't          | t                    rt          |           }nd S |S N)	
isinstancer   r=   type
issubclassr4   callablestrr&   r=   trainable_clss     a/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/tune.py_get_trainablerI   U   s     .*-- 7'6.$'' 
.)44 	4&	.	!	! &	NC	(	( ).99t    resumec                 h   | du rdS | du rt                      S |                     d          }|d         }|dv rt          d| d          t                      }|d	d         D ]}|d
k    r t          t           j        j                  }(|dk    r t          t           j        j                  }N|dk    r0t          t           j        j        t           j        j                  }|dk    r0t          t           j        j        t           j        j                  }t          d| d          |S )zjConverts the legacy resume (str, bool) to a ResumeConfig object.
    Returns None if resume is False.
    FNT+r   )LOCALREMOTEPROMPTERRORED_ONLY'z=' is deprecated. Please pass in one of (True, False, 'AUTO').   ERRORED)erroredRESTART_ERROREDrQ   )
unfinishedrU   RESTART_ERRORED_ONLYzInvalid resume setting: ')r5   splitDeprecationWarning
ResumeTypeRESUMERESTARTSKIP
ValueError)rK   resume_settings
resume_strresume_configsettings        rH   '_build_resume_config_from_legacy_configrd   j   sm    t~~~~ ll3''O #JBBB ;
 ; ; ;
 
 	

 !NNM"122& E Ei(1H1OPPPMM)))(1H1PQQQMM&&('27$/6  MM ...('27$/7  MM
 CCCCDDDrJ   c                     t          |           }|sdS t          |d          o|j        j        t          j        j        k    S )NTdefault_resource_request)rI   hasattrrf   __code__r4   rF   s     rH   !_check_default_resources_overrideri      sM     #>22M t="<== .7-6	7rJ   c                 j    t          |           }|sdS t          |d          pt          |dd          S )NT
__mixins__	_is_mixinF)rI   rg   getattrrF   s     rH   _check_mixinrn      sF    ">22M t=,// 7{E4 4 rJ   	resourcesc                     | sdS t          | t                    r(t          | j                            dd                     S t          | t
                    r#t          |                     dd                     S d S )NFGPUgpu)rA   r   boolrequired_resourcesgetdictro   s    rH   _check_gpus_in_resourcesrx      s~      u)233 CI044UDAABBB)T"" 0IMM%..///0 0rJ   Frunnerreporterdonec                     |                                  }|                    ||          rG| j                                        }|                                 }|                    ||||           dS dS )zReports experiment progress.

    Args:
        runner: Trial runner to report on.
        reporter: Progress reporter.
        done: Whether this is the last progress report attempt.
    r{   N)
get_trialsshould_reportscheduler_algdebug_string_used_resources_stringreport)ry   rz   r{   trialssched_debug_strused_resources_strs         rH   _report_progressr      s       Ff400 K .;;==#::<<o7IJJJJJK KrJ   AirProgressReporterforcec                     |                                  }g }|                                 }|                    |            |j        |g|R d|i d S )Nr   )r~   r   appendprint_heartbeat)ry   rz   r   r   reporter_argsused_resources_strings         rH   _report_air_progressr      sj       FM"99;;.///HVAmAAA5AAAAArJ   c                     t          j        t           j                  t          j                    dt
          ffd} d}t          j                    t          j                    k    rd}|rut          t          j	        dd                    st          j         t           j        |            t          t           d          rt          j         t           j        |            S )Nsigc                     t                               d                                            t          j        t          j                   d S )NzStop signal received (e.g. via SIGINT/Ctrl+C), ending Ray Tune run. This will try to checkpoint the experiment state one last time. Press CTRL+C (or send SIGINT/SIGKILL/SIGTERM) to skip. )loggerwarningsetsignalSIGINT)r   frameexperiment_interrupted_eventoriginal_handlers     rH   signal_interrupt_tune_runz9_setup_signal_catching.<locals>.signal_interrupt_tune_run   sM    	
 	
 	
 	%((***fm%566666rJ   TFTUNE_DISABLE_SIGINT_HANDLER0SIGUSR1)r   	getsignalr   	threadingEventintcurrent_threadmain_threadosgetenvrg   r   )r   allow_signal_catchingr   r   s     @@rH   _setup_signal_catchingr      s    '66#,?#4#4 	7s 	7 	7 	7 	7 	7 	7 	7 !!!Y%:%<%<<< % E29:C@@AA 	DM&-)BCCC 69%% 	EM&.*CDDD''rJ   
entrypointc                    t           j                            d          dk    rt                              d           dS t          j                    s3t          j                     t                              d|  d           dS dS )z)Initialize Ray unless already configured.TUNE_DISABLE_AUTO_INIT1z$'TUNE_DISABLE_AUTO_INIT=1' detected.zmInitializing Ray automatically. For cluster usage or custom Ray initialization, call `ray.init(...)` before `z`.N)r   environru   r   inforayis_initializedinitr   s    rH   _ray_auto_initr      s    	z~~.//366:;;;;;!! 



;,6; ; ;	
 	
 	
 	
 	

 
rJ   c                       e Zd ZdefdZdS )_Configr>   c                     t           )z-Converts this configuration to a dict format.)NotImplementedError)selfs    rH   to_dictz_Config.to_dict  s    !!rJ   N)__name__
__module____qualname__rv   r    rJ   rH   r   r     s/        " " " " " " "rJ   r   rS   T)&namemetricmodestoptime_budget_sconfigresources_per_trialnum_samplesstorage_pathstorage_filesystem
search_alg	schedulercheckpoint_configverboseprogress_reporterlog_to_filetrial_name_creatortrial_dirname_creatorsync_configexport_formatsmax_failures	fail_fastrestorerK   rb   reuse_actorsraise_on_failed_trial	callbacksmax_concurrent_trialskeep_checkpoints_numcheckpoint_score_attrcheckpoint_freqcheckpoint_at_endchdir_to_trial_dir	local_dir_remote_remote_string_queue_entrypointrun_or_experimentr   r   r   r   r   r   r   r   r   r   zpyarrow.fs.FileSystemr   r   r   r   r   r   r   r   r   r   r   r   r   rb   r   r   r   r   r   r   r   r   r   r   r   r   r   c       &            t                                                      }'|'                    d           |&t          j        k    rdddd}(n4|&t          j        k    rdddd}(n|&t          j        k    rdd	d
d}(ndddd}(t          |(d                    |$(t          j	        j
        j                                        }$|%t          t          j                  pt          j        }|$rt          |          t"                              d            t          j        d          t(                    })t+          |)          })t-          |||%          \  }}*||'d<    |)j        d{ddi|'}+t/          |+||*           t          j        |+          S ~'d},t2          j                            d          r"t7          |,                    d                    t2          j                            d          r"t7          |,                    d                    |#t7          d          t          j        j        j                             d           tC          j"                     tC          j#        |&           tI          j$                    }-|r|dvrtK          d|(d          d          t          |          }.|.#tL          rt"                              d           d}.|.tO          d           ntO          |           |pi }tQ          |tR                    r|*                                }tQ          |tV                    s2tK          d|(d           d!|(d          d"tY          |           d#          |pt[                      }|pt]                      }|!t_          j0        d$t6                     ||_1        |ut_          j0        d%t6                     |2                    d&          r1t_          j0        d't6                     |d(d         |_3        d)|_4        n||_3        d*|_4        ||_5        | dk    r!t_          j0        d+t6                     | |_6        |!r!t_          j0        d,t6                     |!|_7        |"tp          k    rt7          d-tr           d.t6                    |d/k    rtt          j;        }tQ          |tx                    rdd0l=m>}/  |/|          }|pt                      }|j@        sNt3          jA        d1d2          }0|0r7t_          j0        d3tY          |           d4|0 d5           d6t2          j        d1<   tQ          |t          t          f          r|st_          j0        d7           t                      }1t          t          jF        |          |1          }tQ          | t                    r| }2n| g}2t          |2          D ]Z\  }3}4tQ          |4t                    s@t          d{i d8|d9|4d:|d;|d|d<|d=|d>|	d?|
d@|dA|dB|dC|dD|dE|dF|dG||2|3<   [|r|dk    rtK          dH          tQ          |tx                    rt          |          }t          jK        jL        M                                t          jK        jL        jN        k    }5|5rdI}|st          |pdJ          }n|s|5rtQ          |t                    rG|5sD|jQ        |k    rtK          dK| dL|jQ         dM          t"          R                    dK| dN           n\|dIk     rtK          dO| dP          tQ          |t                    rt          ||J          }n|5st"          R                    dQ           tQ          |t                    rt          |          }|rQt          |jV        |||fi |2d         jW        s1t          |          r"tK          dR|(d           dS|(d          dT          t          |jV        ||fi |2d         jW        s"tK          dU|(d          dV|(d          dW          t          t          |                     }6tC          j\        |2d         j]                   tC          j^        |           t          ||.|&||||6X          }t          j`                                        dYd          rFt          |Z          rn5t          |2d         jc                  rnt"          R                    d[           t                      }7|r|.t"          R                    d\           d}|."|&t          j        k    }8|pt          |8]          }||pt          |          }tW          ||1||2d         jg        |||||2d         jh        ||2d         j]        |&t          j        k    ^          }9t          d{i |9}:|:jj        s0|2D ]}4|k                    |4g           |:l                                 n1t"          m                    d_           |r|:n                    |           |:o                    |2|jp        `           tI          j$                    };d}<|.|q                    |;|jp        ||a           n;ddblrms}= |D ]2}>tQ          |>|=          r |>}<|<q                    |;|jp        c            n3|:jt        ju        }?|:jt        jv        }@t          dd |D                       r|?}And}A|<r9|<x                    |@|:jy        |jz        j{        |jz        j{        |jp        |Ae           	 |:|                                s|7}                                sy|:~                                 t          t          j                  rt          |:|           |.t          |:|<           |:|                                s|7}                                yn$# t          $ r |:                                  w xY wtI          j$                    |;z
  }BtI          j$                    }C	 |:                    dfdfg           t"                              dh|:jy         ditI          j$                    |Cz
  djdk           n.# t          $ r  t"                              dldfdfm           Y nw xY wt          t          j                  rt          |:|dfn           |.t          |:|<dfo           |:                                }D|:                                 g }E|DD ]}F|Fj        t          j        k    r|E|Fgz  }E|ErB|r%|7}                                st          dp|E          t"                              dq|E           tI          j$                    |-z
  }Gt          t          j                  r#t"                              dr|Gdsdt|Bdsdu           |7}                                rl|(dv                             |:jy        w          }H|&t          j        k    rt"          R                    dx|H            nt"          R                    dy|H            t          |:jy        |||D|2d         j]        j        z          S )|a)  Executes training.

    When a SIGINT signal is received (e.g. through Ctrl+C), the tuning run
    will gracefully shut down and checkpoint the latest experiment state.
    Sending SIGINT again (or SIGKILL/SIGTERM instead) will skip this step.

    Many aspects of Tune, such as the frequency of global checkpointing,
    maximum pending placement group trials and the path of the result
    directory be configured through environment variables. Refer to
    :ref:`tune-env-vars` for a list of environment variables available.

    Examples:

    .. code-block:: python

        # Run 10 trials (each trial is one instance of a Trainable). Tune runs
        # in parallel and automatically determines concurrency.
        tune.run(trainable, num_samples=10)

        # Run 1 trial, stop when trial has reached 10 iterations
        tune.run(my_trainable, stop={"training_iteration": 10})

        # automatically retry failed trials up to 3 times
        tune.run(my_trainable, stop={"training_iteration": 10}, max_failures=3)

        # Run 1 trial, search over hyperparameters, stop after 10 iterations.
        space = {"lr": tune.uniform(0, 1), "momentum": tune.uniform(0, 1)}
        tune.run(my_trainable, config=space, stop={"training_iteration": 10})

        # Resumes training if a previous machine crashed
        tune.run(
            my_trainable, config=space,
            storage_path=<path/to/dir>, name=<exp_name>, resume=True
        )

    Args:
        run_or_experiment: If function|class|str, this is the algorithm or
            model to train. This may refer to the name of a built-on algorithm
            (e.g. RLlib's DQN or PPO), a user-defined trainable
            function or class, or the string identifier of a
            trainable function or class registered in the tune registry.
            If Experiment, then Tune will execute training based on
            Experiment.spec. If you want to pass in a Python lambda, you
            will need to first register the function:
            ``tune.register_trainable("lambda_id", lambda x: ...)``. You can
            then use ``tune.run("lambda_id")``.
        metric: Metric to optimize. This metric should be reported
            with `tune.report()`. If set, will be passed to the search
            algorithm and scheduler.
        mode: Must be one of [min, max]. Determines whether objective is
            minimizing or maximizing the metric attribute. If set, will be
            passed to the search algorithm and scheduler.
        name: Name of experiment.
        stop: Stopping criteria. If dict,
            the keys may be any field in the return result of 'train()',
            whichever is reached first. If function, it must take (trial_id,
            result) as arguments and return a boolean (True if trial should be
            stopped, False otherwise). This can also be a subclass of
            ``ray.tune.Stopper``, which allows users to implement
            custom experiment-wide stopping (i.e., stopping an entire Tune
            run based on some time constraint).
        time_budget_s: Global time budget in
            seconds after which all trials are stopped. Can also be a
            ``datetime.timedelta`` object.
        config: Algorithm-specific configuration for Tune variant
            generation (e.g. env, hyperparams). Defaults to empty dict.
            Custom search algorithms may ignore this.
        resources_per_trial: Machine resources
            to allocate per trial, e.g. ``{"cpu": 64, "gpu": 8}``.
            Note that GPUs will not be assigned unless you specify them here.
            Defaults to 1 CPU and 0 GPUs in
            ``Trainable.default_resource_request()``. This can also
            be a PlacementGroupFactory object wrapping arguments to create a
            per-trial placement group.
        num_samples: Number of times to sample from the
            hyperparameter space. Defaults to 1. If `grid_search` is
            provided as an argument, the grid will be repeated
            `num_samples` of times. If this is -1, (virtually) infinite
            samples are generated until a stopping condition is met.
        storage_path: Path to store results at. Can be a local directory or
            a destination on cloud storage. Defaults to
            the local ``~/ray_results`` directory.
        search_alg: Search algorithm for
            optimization. You can also use the name of the algorithm.
        scheduler: Scheduler for executing
            the experiment. Choose among FIFO (default), MedianStopping,
            AsyncHyperBand, HyperBand and PopulationBasedTraining. Refer to
            ray.tune.schedulers for more options. You can also use the
            name of the scheduler.
        verbose: 0, 1, or 2. Verbosity mode.
            0 = silent, 1 = default, 2 = verbose. Defaults to 1.
            If the ``RAY_AIR_NEW_OUTPUT=1`` environment variable is set,
            uses the old verbosity settings:
            0 = silent, 1 = only status updates, 2 = status and brief
            results, 3 = status and detailed results.
        progress_reporter: Progress reporter for reporting
            intermediate experiment progress. Defaults to CLIReporter if
            running in command-line, or JupyterNotebookReporter if running in
            a Jupyter notebook.
        log_to_file: Log stdout and stderr to files in
            Tune's trial directories. If this is `False` (default), no files
            are written. If `true`, outputs are written to `trialdir/stdout`
            and `trialdir/stderr`, respectively. If this is a single string,
            this is interpreted as a file relative to the trialdir, to which
            both streams are written. If this is a Sequence (e.g. a Tuple),
            it has to have length 2 and the elements indicate the files to
            which stdout and stderr are written, respectively.
        trial_name_creator: Optional function that takes in a Trial and returns
            its name (i.e. its string representation). Be sure to include some unique
            identifier (such as `Trial.trial_id`) in each trial's name.
        trial_dirname_creator: Optional function that takes in a trial and
            generates its trial directory name as a string. Be sure to include some
            unique identifier (such as `Trial.trial_id`) is used in each trial's
            directory name. Otherwise, trials could overwrite artifacts and checkpoints
            of other trials. The return value cannot be a path.
        chdir_to_trial_dir: Deprecated. Set the `RAY_CHDIR_TO_TRIAL_DIR` env var instead
        sync_config: Configuration object for syncing. See tune.SyncConfig.
        export_formats: List of formats that exported at the end of
            the experiment. Default is None.
        max_failures: Try to recover a trial at least this many times.
            Ray will recover from the latest checkpoint if present.
            Setting to -1 will lead to infinite recovery retries.
            Setting to 0 will disable retries. Defaults to 0.
        fail_fast: Whether to fail upon the first error.
            If fail_fast='raise' provided, Tune will automatically
            raise the exception received by the Trainable. fail_fast='raise'
            can easily leak resources and should be used with caution (it
            is best used with `ray.init(local_mode=True)`).
        restore: Path to checkpoint. Only makes sense to set if
            running 1 trial. Defaults to None.
        resume: One of [True, False, "AUTO"]. Can
            be suffixed with one or more of ["+ERRORED", "+ERRORED_ONLY",
            "+RESTART_ERRORED", "+RESTART_ERRORED_ONLY"] (e.g. ``AUTO+ERRORED``).
            `resume=True` and `resume="AUTO"` will attempt to resume from a
            checkpoint and otherwise start a new experiment.
            The suffix "+ERRORED" resets and reruns errored trials upon resume -
            previous trial artifacts will be left untouched. It will try to continue
            from the last observed checkpoint.
            The suffix "+RESTART_ERRORED" will instead start the errored trials from
            scratch. "+ERRORED_ONLY" and "+RESTART_ERRORED_ONLY" will disable
            resuming non-errored trials - they will be added as finished instead. New
            trials can still be generated by the search algorithm.
        resume_config: [Experimental] Config object that controls how to resume
            trials of different statuses. Can be used as a substitute to the
            `resume` suffixes described above.
        reuse_actors: Whether to reuse actors between different trials
            when possible. This can drastically speed up experiments that start
            and stop actors often (e.g., PBT in time-multiplexing mode). This
            requires trials to have the same resource requirements.
            Defaults to ``False``.
        raise_on_failed_trial: Raise TuneError if there exists failed
            trial (of ERROR state) when the experiments complete.
        callbacks: List of callbacks that will be called at different
            times in the training loop. Must be instances of the
            ``ray.tune.callback.Callback`` class. If not passed,
            `LoggerCallback` (json/csv/tensorboard) callbacks are automatically added.
        max_concurrent_trials: Maximum number of trials to run
            concurrently. Must be non-negative. If None or 0, no limit will
            be applied. This is achieved by wrapping the ``search_alg`` in
            a :class:`ConcurrencyLimiter`, and thus setting this argument
            will raise an exception if the ``search_alg`` is already a
            :class:`ConcurrencyLimiter`. Defaults to None.
        _remote: Whether to run the Tune driver in a remote function.
            This is disabled automatically if a custom trial executor is
            passed in. This is enabled by default in Ray client mode.
        local_dir: Deprecated. Use `storage_path` instead.
        keep_checkpoints_num: Deprecated. use checkpoint_config instead.
        checkpoint_score_attr: Deprecated. use checkpoint_config instead.
        checkpoint_freq: Deprecated. use checkpoint_config instead.
        checkpoint_at_end: Deprecated. use checkpoint_config instead.
        checkpoint_keep_all_ranks: Deprecated. use checkpoint_config instead.
        checkpoint_upload_from_workers: Deprecated. use checkpoint_config instead.

    Returns:
        ExperimentAnalysis: Object for experiment analysis.

    Raises:
        TuneError: Any trials failed and `raise_on_failed_trial` is True.
    r   z<FrameworkTrainer>(...)param_spacez.<FrameworkTrainer>.restore(path="{path}", ...))r   search_space_argrestore_entrypointz
Tuner(...)z+Tuner.restore(path="{path}", trainable=...)tune.run_experiments(...)zexperiment=Experiment(config)z&tune.run_experiments(..., resume=True)ztune.run(...)r   ztune.run(..., resume=True)r   r   N[output] This uses the legacy output and progress reporter, as Ray client is not supported by the new engine. For more information, see https://github.com/ray-project/ray/issues/36949r   num_cpusr   FaX  The environment variable `{}` is deprecated. It is no longer used and will not have any effect. You should set the `storage_path` instead. Files will no longer be written to `~/ray_results` as long as `storage_path` is set.See the docs: https://docs.ray.io/en/latest/train/user-guides/persistent-storage.html#setting-the-local-staging-directoryTUNE_RESULT_DIRRAY_AIR_LOCAL_CACHE_DIRzThe `local_dir` argument is deprecated. You should set the `storage_path` instead. See the docs: https://docs.ray.io/en/latest/train/user-guides/persistent-storage.html#setting-the-local-staging-directorytune)minmaxz The `mode` parameter passed to `z` must be one of ['min', 'max']z[output] This uses the legacy output and progress reporter, as Jupyter notebooks are not supported by the new engine, yet. For more information, please see https://github.com/ray-project/ray/issues/36949zThe `r   z` passed to `z` must be a dict. Got 'z
' instead.zbkeep_checkpoints_num is deprecated and will be removed. use checkpoint_config.num_to_keep instead.zrcheckpoint_score_attr is deprecated and will be removed. use checkpoint_config.checkpoint_score_attribute instead.zmin-zusing min- and max- prefixes to specify checkpoint score order is deprecated. Use CheckpointConfig.checkpoint_score_order instead   r   r   zfcheckpoint_freq is deprecated and will be removed. use checkpoint_config.checkpoint_frequency instead.zecheckpoint_at_end is deprecated and will be removed. use checkpoint_config.checkpoint_at_end instead.z,`chdir_to_trial_dir` is deprecated. Use the zm environment variable instead. Set it to 0 to disable the default behavior of changing the working directory.)create_schedulerTUNE_RESULT_BUFFER_LENGTH zYou are using a z2 scheduler, but TUNE_RESULT_BUFFER_LENGTH is set (zf). This can lead to undesired and faulty behavior, so the buffer length was forcibly set to 1 instead.r   zrConsider boosting PBT performance by enabling `reuse_actors` as well as implementing `reset_config` for Trainable.r   runr   r   r   r   r   r   r   r   r   r   r   r   r   r   z)max_failures must be 0 if fail_fast=True.rS   )max_concurrentz*You have specified `max_concurrent_trials=zO`, but the `search_alg` is already a `ConcurrencyLimiter` with `max_concurrent=z3. FIX THIS by setting `max_concurrent_trials=None`.zc`, but the `search_alg` is already a `ConcurrencyLimiter`. `max_concurrent_trials` will be ignored.z=`max_concurrent_trials` must be greater or equal than 1, got .zYou have passed a `SearchGenerator` instance as the `search_alg`, but `max_concurrent_trials` requires a `Searcher` instance`. `max_concurrent_trials` will be ignored.zYou passed a `z` parameter to `z` with unresolved parameters, but the search algorithm was already instantiated with a search space. Make sure that `config` does not contain any more parameter definitions - include them in the search algorithm's search space if necessary.z-You passed a `metric` or `mode` argument to `z`, but the scheduler you are using was already instantiated with their own `metric` and `mode` parameters. Either remove the arguments from your scheduler or from `z` args.)air_verbosityr   r   r   r   progress_metricsrq   rw   a  Tune detects GPUs, but no trials are using GPUs. To enable trials to use GPUs, wrap `train_func` with `tune.with_resources(train_func, resources_per_trial={'gpu': 1})` which allows Tune to expose 1 GPU to each trial. For Ray Train Trainers, you can specify GPU resources through `ScalingConfig(use_gpu=True)`. You can also override `Trainable.default_resource_request` if using the Trainable API.zBAIR_VERBOSITY is set, ignoring passed-in ProgressReporter for now.)_trainer_api)r   placeholder_resolversr   stopperrb   r   r   r   trial_checkpoint_configr   storager   zYou have resumed the Tune run, which means that any newly specified `Experiment`s will be ignored. Tune will just continue what was previously running.)experimentstotal_num_samples)
start_timetotal_samplesr   r   r<   )r  r  c              3   @   K   | ]}t          |t                    V  d S r@   )rA   r    ).0cbs     rH   	<genexpr>zrun.<locals>.<genexpr>  s-      
A
A:b+,,
A
A
A
A
A
ArJ   )experiment_nameexperiment_pathsearcher_strscheduler_strr   tensorboard_pathT)r   waitzFWrote the latest version of all result files and experiment state to 'z' in z.4fzs.z%Experiment state snapshotting failed:)exc_info
stack_infor}   )r   zTrials did not completezTrials did not complete: %szTotal run time: z.2fz
 seconds (z seconds for the tuning loop).r   )pathzZTraining has been interrupted, but the most recent state was saved.
Resume training with: z^Experiment has been interrupted, but the most recent state was saved.
Resume experiment with: )experiment_checkpoint_pathdefault_metricdefault_moder   r   r   )localscopypopr   TRAINERTUNERTUNE_RUN_EXPERIMENTSr   r   utilclientis_connectedr   r   VERBOSEr7   V3_TRIAL_DETAILSr   r   remoter   r   r$   r%   ru   r   r   rZ   format_commonr   	usage_librecord_library_usage	air_usagetag_ray_air_env_varstag_air_entrypointtimer_   r   r9   rA   r   r   rv   rB   r   r   warningswarnnum_to_keep
startswithcheckpoint_score_attributecheckpoint_score_order
score_attrcheckpoint_frequencyr   r   r   sysmaxsizerE   ray.tune.schedulersr   r'   supports_buffered_resultsr   r(   r)   r   r   deepcopylist	enumerater   r1   _privateworker_mode
LOCAL_MODEr,   r-   r   r   r/   r0   searcher_set_search_propsset_search_propertiespublic_specr2   scheduler_set_search_propsr"   rI   tag_storage_typer   tag_callbacksr6   cluster_resourcesrx   ri   r=   r   r#   rd   r   r   r   resumedadd_configurationsupdate_max_pending_trialsdebugupdate_pending_trial_resourcessetup_experimentsr  setupray.tune.experimental.outputr!   _storageexperiment_driver_staging_pathexperiment_dir_nameanyexperiment_startedr  	__class__r   is_finishedis_setstepr8   V1_EXPERIMENTr   r   	Exceptioncleanup
checkpointerrorr~   statusr   
TERMINATEDr   r   r   )Ir   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rK   rb   r   r   r   r   r   r   r   r   r   r   r   r   r   remote_run_kwargserror_message_map
remote_runstring_queueremote_futureENV_VAR_DEPRECATION_MESSAGE	all_startr   r   env_result_buffer_lengthr   r   iexpis_local_moder   r   
is_trainerrunner_kwargsry   
tune_startair_progress_reporterr   callbackexperiment_local_pathrK  r  
tune_takenfinal_sync_start
all_trialsincomplete_trialstrial	all_takenr   sI                                                                            rH   r   r   
  s   R )$$$m+++3 -"R
 

 
+	+	+& -"O
 

 
:	:	:5 ?"J
 
 * (">
 
 /=>>>>(/%2244 $L$899WY=W &W%%1KKB   ,SZ+++C00
 ,J77
*Sw(<+
 +
'<
 2C-.)
)MM%M;LMM	
 	
 	

 w}%%%	F   
z~~'(( X !<!C!CDU!V!VWWW	z~~/00 
 '../HII
 
 	
  J
 
 	
 K44V<<< "$$$  ---	I 
N**,/@/N , , ,
 
 	

 &g..M [ >	
 	
 	
  a 	g\rF&'"" "!!fd## 
-%&89 - -!,/- -LL- - -
 
 	
 -K)?-=-?-? '9	
 	
 	

 )=%(H	
 	
 	
 !++F33 	=M #	   <QQRQSQS;T87<44;P87<4'<$B	
 	
 	

 2A. @?	
 	
 	

 /@+ ... !-! ! ! 
 
 	
 bk )S!! 0888888$$Y//	,]__I. :
 $&9-H"#M#M # 	:M!4	?? ! !,! ! !   7:BJ23 	968UVWW

 	A	
 	
 	
 122 f F #T** *'()K((  3#z** 	'   TC T ,m	
 v %8$7 (K *\ $6#5 (K #4"3 $6#5 '<&; (K  .~  *\!"  #KN(  F\Q&&DEEE*c"" 1$Z00
 L'--//3<3F3QQM " ! $*:O:TSTUUU

	 "- "j"455 !	  ,0EEE$808 8 &48 8 8   NNC0C C C   %q(( 404 4 4   *h// 
//D  

 # '   *h'' 1$Z00
 /(	 
 a.
$   "&)) 	L!23E!F L L%l3L L L   &' 9DQ9S  	
 U!,/U U ->l,K	U U U
 
 	
 0?P0Q0QRR{1~5666 I&&& *#)  I ""5!,, #.ABBB 	.{1~/LMM 	NN!
 
 
 $:#;#;  !]6P	
 	
 	
 ! M$99
- 
1A#2
 2
 2
 %X)PQW)X)X3A&# +A @!A& M$99  M ,,m,,F> G 	1 	1C))3%0000 	((****C	
 	
 	

  	G112EFFF :3K     J !$2	 	  	
 	
 	
 	
 	YXXXXX! 	 	H($788 (0%%++)9Q ,     #OJ /=

A
Ay
A
A
AAA  0 
00/"2#-6#-6(6- 	1 	
 	
 	

$$&& 	D/K/R/R/T/T 	DKKMMMY455 < ):;;;($V-BCCC $$&& 	D/K/R/R/T/T 	D     z)Jy{{	
4000T&T T-1Y[[;K-KST T T	
 	
 	
 	
  
 
 
3dt 	 	
 	
 	
 	
 	


 Y,-- ?!2>>>> V%:$GGGG""$$J
NN ) )<5+++%( K  	K)E)L)L)N)N 	K57HIIILL68IJJJ	i'IY,-- 
?y7 ? ?>? ? ?	
 	
 	

 $**,, ./CDKK' L 
 
 -///NN>);> >   
 NNH3EH H  
 #)#9&q>1D   s    B!n= =!oAq (r	r	r   
concurrentc                    |
(t           j        j        j                                         }
t	          d           |%t          t          j                  pt          j	        }|
rt          |          t                              d            t          j        d          t                    }t          |          }t          j        |                    | |||||||||	d                    S t!          |           } t#          ||||||||	t$          j        		  	        }|rt)          | fi |j        S g }| D ]}|t)          |fi |j        z  }|S )
af  Runs and blocks until all trials finish.

    Example:
        >>> from ray.tune.experiment import Experiment
        >>> from ray.tune.tune import run_experiments
        >>> def my_func(config): return {"score": 0}
        >>> experiment_spec = Experiment("experiment", my_func) # doctest: +SKIP
        >>> run_experiments(experiments=experiment_spec) # doctest: +SKIP
        >>> experiment_spec = {"experiment": {"run": my_func}} # doctest: +SKIP
        >>> run_experiments(experiments=experiment_spec) # doctest: +SKIP

    Returns:
        List of Trial objects, holding data for each executed trial.

    Nr   r   r   r   r   F)r   )	r   r   rK   rb   r   r   r   r   r   )r   r  r  r  r   r   r   r  r7   r  r   r   r  run_experimentsr   ru   r   rv   r   r  r   r   )r   r   r   r   rK   rb   r   r   rp  r   r   r[  tune_run_paramsr   rb  s                  rH   rr  rr  .  s   : (/%22449:::: $L$899WY=W 
W%%1KKB   ,SZ+++O<<
 ,J77
w!%   
 
 	
& .k::K+#!3!6
 
 
O  ;22/2299 	9 	9Cc#111188FFrJ   )F)
NNNNNFTTNN)~abcr  datetimeloggingr   r   r/  r   r&  r'  typingr   r   r   r   r   r   r	   r
   r   r   ray.air._internalr   r#  ray.air._internal.usager   ray.air.util.noder   ray.train.constantsr   r   ray.tuner   r   ray.tune.analysisr   ray.tune.callbackr   ray.tune.errorr   #ray.tune.execution.placement_groupsr   "ray.tune.execution.tune_controllerr   ray.tune.experimentr   r   r   rH  r   r   r   ray.tune.impl.placeholderr   r   ray.tune.loggerr    ray.tune.progress_reporterr!   r"   r#   r$   r%   ray.tune.registryr&   r1  r'   r(   r)   r*   ray.tune.schedulers.utilr+   r=  ray.tune.searchr,   r-   r.   r/   r0   r1   ray.tune.search.utilr:  !ray.tune.search.variant_generatorr2   ray.tune.stopperr3   ray.tune.trainabler4   ray.tune.tune_configr5   ray.tune.utils.callbackr6   ray.tune.utils.logr7   r8   r9   ray.util.annotationsr:   ray.util.queuer;   
pyarrow.fspyarrowr   	getLoggerr   r   rE   rI   rs   rd   ri   rn   rx   r   r   r   r   r   ABCr   TUNE_RUNr   float	timedeltar   rr  r   rJ   rH   <module>r     s
   



    				  



      
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 


 0 0 0 0 0 0 1 1 1 1 1 1 4 4 4 4 4 4 I I I I I I I I 1 1 1 1 1 1 1 1 0 0 0 0 0 0 & & & & & & $ $ $ $ $ $ E E E E E E = = = = = = N N N N N N N N N N U U U U U U U U U U O O O O O O O O - - - - - -              0 / / / / /                                     E D D D D D $ $ $ $ $ $ ( ( ( ( ( ( - - - - - - = = = = = = F F F F F F F F F F * * * * * *             UTTTTTT		8	$	$*c49:d9o   *(#t)(l( ( ( (V*c49:	   z3h'F!G D    
0d$99:;
0	
0 
0 
0 
0 FKK KK&6K>BK K K K$ LQB BB&;BDHB B B B(	 ( ( ( (B

s 

 

 

 

" " " " "cg " " "   NREI'+ 	"&<@BF6:48=A48;?>B(,)-!)-,0"&.2+/*.+/#0#",0 "/!7_` ` `S(D01` 3-` SM	`
 3-` 5'8S'ND4H+IIJ
K` E#uh.@"@AB` T#s(^$` gc5W!45568MM` ` 3-` !!89` x#=>?`  nc123!`"   01#`$ eCy89:%`&   01'`( )`* !5'3,!78+`, $HeWc\$:;-`. *%/`0 X&1`2 3`4 5`6 c]7`8 U49%&9`: L);`< =`>  ?`@ *+A`B $C=C`F #3-G`H $C=I`J K`L M`N O`P }Q`T d^U`X #5/Y`^ _`` a` ` ` `F  +/=A48)-,0"&.2"Z Zz7HU:w;N5O,PPQZ'Z eCy89:Z   01	Z
 U49%&Z L)Z Z  Z Z *+Z d^Z Z Z Z Z ZrJ   