&`iolddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZmZddlZddlmZmZmZddlmZmZddlmZddlmZdd lmZdd lm Z dd l!m"Z"m#Z#dd l$m%Z%dd l&m'Z'ddl(m)Z)m*Z*m+Z+ddl,m-Z- ddl.Z/ddl.m0Z0n#e1$rdZ/dZ0YnwxYwej2e3Z4e-dGddZ5dS)N)Number)Path)AnyDictListOptionalTupleUnion)EXPR_PROGRESS_FILEEXPR_RESULT_FILETRAINING_ITERATION)_exists_at_fs_pathget_fs_and_path) Checkpoint)"_find_newest_experiment_checkpoint)TuneController)Trial) CONFIG_PREFIXDEFAULT_METRIC) flatten_dict)TuneFunctionDecoder)is_nan is_nan_or_infunflattened_lookup) PublicAPI) DataFramebeta) stabilityc eZdZdZddddddeeejfdee j j dee e deedeef d Zd e e fd Zd e d efd Zd eeeffdZd/ded eeeffdZed efdZed e fdZed efdZed efdZed efdZed efdZdZed efdZed eeeffdZ ed efdZ!ed eeeffdZ" d0deedeed efdZ# d1d e deed e e$ee%ffd Z& d0d e deedeed eefd!Z' d2deedeed$ed%ed ee f d&Z( d3deedeed$ed eefd'Z) d4d eefd*Z*ded efd+Z+ded efd,Z, d0deedeed eee-ffd-Z.d eee-ffd.Z/dS)5ExperimentAnalysisaAnalyze results from a Ray Train/Tune experiment. To use this class, the run must store the history of reported metrics in log files (e.g., `result.json` and `progress.csv`). This is the default behavior, unless default loggers are explicitly excluded with the `TUNE_DISABLE_AUTO_CALLBACK_LOGGERS=1` environment variable. Parameters: experiment_checkpoint_path: Path to an `experiment_state.json` file, or a directory that contains an `experiment_state.json` file. default_metric: Default metric for comparing results. Can be overwritten with the ``metric`` parameter in the respective functions. default_mode: Default mode for comparing results. Has to be one of [min, max]. Can be overwritten with the ``mode`` parameter in the respective functions. trials: List of trials that can be accessed via `analysis.trials`. N)storage_filesystemtrialsdefault_metric default_modeexperiment_checkpoint_pathr!r"r#r$c ||_|r|dvrtd||_|j|j t|_|r||_nt |\|_}t |}|dr,tj ||_ ||_ nr||_ t|j |j}|Gtjd}td|d|jjd|j d ||_ |p||_||_||_dS) Nminmaxz2`default_mode` has to be None or one of [min, max]z.json)experiment_pathfs*z%No experiment snapshot file of form 'z' was found at: (z, z) Please check if you specified the correct experiment path, which should be a combination of the `storage_path` and `name` specified in your run.)r# ValueErrorr$r_fsrstrendswithospathdirname_experiment_fs_path_experiment_json_fs_pathrrCKPT_FILE_TMPLformat type_name _load_trialsr"_fetch_trial_dataframes_trial_dataframesget_all_configs_configs)selfr%r!r"r#r$experiment_json_fs_pathpatterns y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/analysis/experiment_analysis.py__init__zExperimentAnalysis.__init__8s-  SL>>QRR R(   &4+<+H"0D   )DHH3B*44 0DH0 &))C%D%D" % . .w 7 7 D')w7Q'R'RD $,FD ) )'AD $&H $ 8TX''' #'.(7>>sCC -G--*--.2.F----DD )3 1 1 3 3 !%!=!=!?!?,,.. returncb|j|j5}tj|t }dddn #1swxYwYt|j}g}|d}|D]\}}tj |d}| |tj |j } |j| _|j| _|j| _|| |||S)N)cls trial_dataTstub)r.open_input_streamr5jsonloadsreadallrrr4rfrom_json_staterestore_run_metadatacopystorageparentas_posixstorage_fs_pathr!nameexperiment_dir_name set_storageappend) r>fexperiment_stateexperiment_fs_pathr" trial_statestrial_json_statetrial_runtime_metadatatrial new_storages rAr9zExperimentAnalysis._load_trialslsi X ' '(E F F P!#z!))++;NOOO  P P P P P P P P P P P P P P P"$":;;' 5 8D ! ! 4 4)*:FFFE  & &'= > > >)EM22K*<*C*L*L*N*NK '-1XK *.@.EK +   k * * * MM%  s.AA!Ar_c `dti}|jtSt|jjt }t|jjt}t|jj |r|jj |5}| dd}|stcdddSd|dD}dddn #1swxYwYt!j|d}nt|jj |r|jj |5}| d} dddn #1swxYwYt!jt'j| |}n/t+d|d t d td |jj|S) Ntrial_idzutf-8 c6g|]}tj|S)rKrL).0rows rA z=ExperimentAnalysis._fetch_trial_dataframe..s LLLTZ__LLLrC/)sep)dtypezCould not fetch metrics for z: both z and z were not found at )r/ last_resultrrrQ trial_fs_pathr rSr rr!rJrMdecoderstripsplitpdjson_normalizeread_csvioStringIOFileNotFoundError) r>r_ force_dtype json_fs_path csv_fs_pathrYcontent json_listdfcsv_strs rA_fetch_trial_dataframez)ExperimentAnalysis._fetch_trial_dataframes!3'    $;; EM79IJJSSUU 5=68JKKTTVV em> M M 1CCLQQ MUV))++,,W55< pd.DataFrame Nrcc>g|]\}}d|dt|S)z- z: )repr)rfr_errors rArhz>ExperimentAnalysis._fetch_trial_dataframes..s3JJJ,e,,tE{{,,JJJrCzFailed to fetch metrics for z trial(s): ) r"r~rb ExceptionrXrjoinloggerwarninglen)r>failures trial_dfsr_efail_strs rAr:z*ExperimentAnalysis._fetch_trial_dataframess  [  E ,0,G,G,N,N %.))    +++,5KK %.)   yyJJJJJH NNTs8}}TT(TT   s- A.-A))A.Fprefixc*fd|jDS)zReturns all trial hyperparameter configurations. Args: prefix: If True, flattens the config dict and prepends `config/`. Returns: Dict[str, Dict]: Mapping trial_id -> config dict cdi|],}|jrtt|jin|j-Sre)rbrrconfig)rfr_rs rA z6ExperimentAnalysis.get_all_configs..sK    N?EW mU\:;;;5<   rCr")r>rs `rAr<z"ExperimentAnalysis.get_all_configss2         rCc|jS)zPath pointing to the experiment directory on persistent storage. This can point to a remote storage location (e.g. S3) or to a local location (path on the head node).)r4r>s rAr*z"ExperimentAnalysis.experiment_paths ''rCc||jr|jstd||j|jS)a)Get the best trial of the experiment The best trial is determined by comparing the last trial results using the `metric` and `mode` parameters passed to `tune.run()`. If you didn't pass these parameters, use `get_best_trial(metric, mode, scope)` instead. zTo fetch the `best_trial`, pass a `metric` and `mode` parameter to `tune.run()`. Alternatively, use the `get_best_trial(metric, mode)` method to set the metric and mode explicitly.)r#r$r-get_best_trialrs rA best_trialzExperimentAnalysis.best_trialsN" $*; '  ""4#68IJJJrCc||jr|jstd||j|jS)a8Get the config of the best trial of the experiment The best trial is determined by comparing the last trial results using the `metric` and `mode` parameters passed to `tune.run()`. If you didn't pass these parameters, use `get_best_config(metric, mode, scope)` instead. zTo fetch the `best_config`, pass a `metric` and `mode` parameter to `tune.run()`. Alternatively, use the `get_best_config(metric, mode)` method to set the metric and mode explicitly.)r#r$r-get_best_configrs rA best_configzExperimentAnalysis.best_configsN" $*; '  ##D$79JKKKrCc|jr|jstd|j}|s td|jd|jd|||j|jS)aGet the checkpoint path of the best trial of the experiment The best trial is determined by comparing the last trial results using the `metric` and `mode` parameters passed to `tune.run()`. If you didn't pass these parameters, use `get_best_checkpoint(trial, metric, mode)` instead. Returns: :class:`Checkpoint ` object. zTo fetch the `best_checkpoint`, pass a `metric` and `mode` parameter to `tune.run()`. Alternatively, use the `get_best_checkpoint(trial, metric, mode)` method to set the metric and mode explicitly.zONo best trial found. Please check if you specified the correct default metric (z ) and mode (z).)r#r$r-rget_best_checkpoint)r>rs rAbest_checkpointz"ExperimentAnalysis.best_checkpoints" $*; .  _  *+/+>**%***  '' +T->   rCcj|jr|jstd|j|jjS)aGet the full result dataframe of the best trial of the experiment The best trial is determined by comparing the last trial results using the `metric` and `mode` parameters passed to `tune.run()`. If you didn't pass these parameters, use `get_best_trial(metric, mode)` and use it to look for the dataframe in the `self.trial_dataframes` dict. zQTo fetch the `best_result`, pass a `metric` and `mode` parameter to `tune.run()`.)r#r$r-trial_dataframesrrbrs rAbest_dataframez!ExperimentAnalysis.best_dataframesE" $*; - $T_%=>>rCcT|jr|jstd|jjS)aHGet the last result of the best trial of the experiment The best trial is determined by comparing the last trial results using the `metric` and `mode` parameters passed to `tune.run()`. If you didn't pass these parameters, use `get_best_trial(metric, mode, scope).last_result` instead. zTo fetch the `best_result`, pass a `metric` and `mode` parameter to `tune.run()`. Alternatively, use `get_best_trial(metric, mode).last_result` to set the metric and mode explicitly and fetch the last result.)r#r$r-rrlrs rA best_resultzExperimentAnalysis.best_result)s>" $*; L  **rCcBtjddS)NTUNE_RESULT_DELIMri)r1environgetrs rA _delimiterzExperimentAnalysis._delimiter<sz~~13777rCctstdt|j|}tj|gdS)aMGet the best result of the experiment as a pandas dataframe. The best trial is determined by comparing the last trial results using the `metric` and `mode` parameters passed to `tune.run()`. If you didn't pass these parameters, use `get_best_trial(metric, mode, scope).last_result` instead. zD`best_result_df` requires pandas. Install with `pip install pandas`. delimiterrbindex)rqr-rrrr from_records)r>rs rAbest_result_dfz!ExperimentAnalysis.best_result_df?sa (  #4#3t?P?PQQQ |((+j(IIIrCc$d|jDS)z7Get the last result of the all trials of the experimentc(i|]}|j|jSre)rbrl)rfr_s rArz.ExperimentAnalysis.results..UsKKKe 1KKKrCrrs rAresultszExperimentAnalysis.resultsRsLKt{KKKKrCctstdtjfdjDdS)z/Get all the last results as a pandas dataframe.z@`results_df` requires pandas. Install with `pip install pandas`.c`g|]*}t|j+S)r)rrlr)rfr_r>s rArhz1ExperimentAnalysis.results_df.._sC   U.$//:K:KLLL   rCrbr)rqr-rrr"rs`rA results_dfzExperimentAnalysis.results_dfWsp R |((    ![    )   rCc|jS)zList of all dataframes of the trials. Each dataframe is indexed by iterations and contains reported metrics. )r;rs rArz#ExperimentAnalysis.trial_dataframesfs %%rCmetricmodec|r|dvrtd|r|std|||}|d}|D]@\}}||vr7||||||At jt|S)aReturns a pandas.DataFrame object constructed from the trials. This function will look through all observed results of each trial and return the one corresponding to the passed ``metric`` and ``mode``: If ``mode=min``, it returns the result with the lowest *ever* observed ``metric`` for this trial (this is not necessarily the last)! For ``mode=max``, it's the highest, respectively. If ``metric=None`` or ``mode=None``, the last result will be returned. Args: metric: Key for trial info to order on. If None, uses last result. mode: One of [None, "min", "max"]. Returns: pd.DataFrame: Constructed from a result dict of each trial. r'*If set, `mode` has to be one of [min, max]z^If a `mode` is passed to `ExperimentAnalysis.dataframe(), you'll also have to pass a `metric`!)rrT)r)logdir) r-_retrieve_rowsr<itemsupdaterqrlistvalues)r>rrrows all_configsr2rs rA dataframezExperimentAnalysis.dataframeos,  KD..IJJ J   8  ""&t"<<**$*77 '--// / /LD&t||T !!&)))T !!!...|D//000rCczp |jpt|jjj}d|D}fd|DS)aGet all checkpoints and a specified metric of a trial. Args: trial: The log directory of a trial, or a trial instance. metric: key for trial info to return, e.g. "mean_accuracy". "training_iteration" is used by default if no value was passed to ``self.default_metric``. Returns: List of [Checkpoint, metric] for all checkpoints of the trial. c*g|]}|j|jfSre) checkpointmetrics)rfcheckpoint_results rArhzIExperimentAnalysis._get_trial_checkpoints_with_metric..s4   ! )+<+D E   rCc:g|]\}}|t|fSre)r)rfrrrs rArhzIExperimentAnalysis._get_trial_checkpoints_with_metric..s=   # G+FG<< =   rC)r#r run_metadatacheckpoint_managerbest_checkpoint_results)r>r_rrbest_checkpointss ` rA"_get_trial_checkpoints_with_metricz5ExperimentAnalysis._get_trial_checkpoints_with_metricsuD4.D2D   1 I   %<        '7    rCc@|p |jpt}||}|||}t t d|}|s t d|ddS|dkrdndt|fd \}}|S) aJGets best persistent checkpoint path of provided trial. Any checkpoints with an associated metric value of ``nan`` will be filtered out. Args: trial: The log directory of a trial, or a trial instance. metric: key of trial info to return, e.g. "mean_accuracy". "training_iteration" is used by default if no value was passed to ``self.default_metric``. mode: One of [min, max]. Defaults to ``self.default_mode``. Returns: A :class:`Checkpoint ` object c.t|d SN)r)xs rAz8ExperimentAnalysis.get_best_checkpoint..s!-rCz)No checkpoints have been found for trial .Nr(rc|dzSrre)rscore_order_factors rArz8ExperimentAnalysis.get_best_checkpoint..s3E!3LrC)key) r#r _validate_moderrfilterrrr))r>r_rrcheckpoints_and_metricsr_rs @rArz&ExperimentAnalysis.get_best_checkpoints(D4.D2D""4(("&"I"I%QW"X"X#' --/F G G# # '  LLMUMMM N N N4#'5==RRa #)L)L)L)L   rClastTscopefilter_nan_and_infc$t|jdkr |jdS||}||}|dvr#t d||d}d}|jD]q}||jvr |dvr|j||}n|j||}|rt|rI||}|}P|dkr ||kr|}|}a|dkr ||kr|}|}r|st d |S) aRetrieve the best trial object. Compares all trials' scores on ``metric``. If ``metric`` is not specified, ``self.default_metric`` will be used. If `mode` is not specified, ``self.default_mode`` will be used. These values are usually initialized by passing the ``metric`` and ``mode`` parameters to ``tune.run()``. Args: metric: Key for trial info to order on. Defaults to ``self.default_metric``. mode: One of [min, max]. Defaults to ``self.default_mode``. scope: One of [all, last, avg, last-5-avg, last-10-avg]. If `scope=last`, only look at each trial's final step for `metric`, and compare across trials based on `mode=[min,max]`. If `scope=avg`, consider the simple average over all steps for `metric` and compare across trials based on `mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`, consider the simple average over the last 5 or 10 steps for `metric` and compare across trials based on `mode=[min,max]`. If `scope=all`, find each trial's min/max score for `metric` based on `mode`, and compare trials based on `mode=[min,max]`. filter_nan_and_inf: If True (default), NaN or infinite values are disregarded and these trials are never selected as the best trial. Returns: The best trial for the provided metric. If no trials contain the provided metric, or if the value for the metric is NaN for all trials, then returns None. rr)allravg last-5-avg last-10-avgzExperimentAnalysis: attempting to get best trial for metric {} for scope {} not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]. If you didn't pass a `metric` parameter to `tune.run()`, you have to pass one when fetching the best trial.N)rrrrr)r(zGCould not find best trial. Did you pass the correct `metric` parameter?) rr"_validate_metricrr-r7metric_analysisrrr) r>rrrrrbest_metric_scorer_ metric_scores rArz!ExperimentAnalysis.get_best_trialsL t{  q ;q> !&&v..""4(( K K KEFLVEFF    [ # #EU222DDD$4Vrrrrs rArz"ExperimentAnalysis.get_best_config4s.:((u== $.8z  D8rCtraining_iterationr)cj|p|||}||tdS)aGets the last checkpoint of the provided trial, i.e., with the highest "training_iteration". If no trial is specified, it loads the best trial according to the provided metric and mode (defaults to max. training iteration). Args: trial: If None, load the best trial automatically. metric: If no trial is specified, use this metric to identify the best trial and load the last checkpoint from this trial. mode: If no trial is specified, use the metric and this mode to identify the best trial and load the last checkpoint from it. Returns: Path for last checkpoint of trial r))rrr )r>r_rrs rAget_last_checkpointz&ExperimentAnalysis.get_last_checkpointTs7&:,,VT::''/A5IIIrCcD|s|jstd|p|jS)NzjNo `metric` has been passed and `default_metric` has not been set. Please specify the `metric` parameter.)r#r-)r>rs rArz#ExperimentAnalysis._validate_metricjs< d1 G ,,,rCcn|s|jstd|r|dvrtd|p|jS)NzdNo `mode` has been passed and `default_mode` has not been set. Please specify the `mode` parameter.r'r)r$r-)r>rs rArz!ExperimentAnalysis._validate_moders^ D- E   KD..IJJ J(t((rCc||dvsJ|r|sJi}|jD]\}}|jr ||vrd}nD|dkr||}n#|dkr||}nd} |j|||<|#t$r0t d |YwxYw|S)N)r)r(rr)r(z2Warning: Non-numerical value(s) encountered for {}) rremptyidxmaxidxminilocto_dict TypeErrorrrr7)r>rrrr2r|idxs rArz!ExperimentAnalysis._retrieve_rows|s+|t~5555!6!!!-3355  HD"x Rj''))j'')) WS\1133T    HOOPTUU   s"B&&7C C c|j}dtdtfdfd|dD|d<|S)zEnsure that trials are marked as stubs when pickling, so that they can be loaded later without the trainable being registered. r_rDc|jr|St|jd}|||S)NTrH)rIrtrainable_name __setstate__ __getstate__)r_ trial_copys rAmake_stub_if_neededz.make_stub_if_neededsLz  u3$???J  # #E$6$6$8$8 9 9 9 rCc&g|] }|Srere)rftrs rArhz3ExperimentAnalysis.__getstate__..s%KKKa..q11KKKrCr")__dict__rPr)r>staters @rArzExperimentAnalysis.__getstate__sc  ""$$ u     LKKK5?KKKh rC)F)NN)N)NNrT)NNr)Nrr))0__name__ __module__ __qualname____doc__r r/r1PathLikerpyarrowr+ FileSystemrrrBr9rr~rr:boolr<propertyr*rrrrrrrrrrrrr rrrrrrrrrrrrerCrAr r #sw.?C(,(,&*2/2/2/$)#r{*:$;2/%WZ%:; 2/ e% 2/ ! 2/sm2/2/2/2/hd5k*Ei<c9n)=4  d tCI    "((((X(KEKKKXK$LTLLLXL$    X <? ???X?"+T+++X+$888J JJJXJ$Lc4iLLLXL  I    X  &$sI~"6&&&X&CG%1%1sm%12:3-%1 %1%1%1%1P59   $,SM eJ&' (    B!%" &&& &sm & *  &&&&T!%"#' VV VsmV V ! V % VVVVt!%" 99 9sm9 9 $ 9999B=BJJ * JJJJ,-s-s----)3)3))))CGsm2:3- c3h6d38nrCr )6rPrtrKloggingr1numbersrpathlibrtypingrrrrr r pyarrow.fsrray.air.constantsr r r ray.train._internal.storagerrray.tuner#ray.tune.execution.experiment_stater"ray.tune.execution.tune_controllerrray.tune.experimentrray.tune.resultrrray.tune.utilsrray.tune.utils.serializationrray.tune.utils.utilrrrray.util.annotationsrpandasrqr ImportError getLoggerrrr rerCrArs5  ::::::::::::::::VVVVVVVVVVKKKKKKKKRRRRRR======%%%%%%99999999''''''<<<<<<IIIIIIIIII******        BIII  8 $ $ VB B B B B B B B B B s B BB