&`iLddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddlm Z ddl mZmZmZmZmZmZmZmZddlZddlZddlmZmZmZmZmZmZddl m!Z!dd l"m#Z#m$Z$dd l%m&Z&m'Z'ddl(Z(ej)e*Z+d Z,ejZ-e&Gd d e Z.e&e/dddfdegefdeee/eee/fde0de0dee de1f dZ2e&GddZ3e&Gdde4Z5e&dZ6dZ7dZ8e&defdZ9d ed!e:d"e:d#e:fd$Z;d%e:d&e:deefd'Z<e'd() d>d-eee0e:fd.e=d/e0d0e0d1ee=f d2Z>e& d?d3ed4eed5e0fd6Z?d7Z@e' d@d9ee:d:eeeefd;ede> NNF G G G >e> NNJ K K K >fn F k4  ))+++ !$'' N$$    JJLLLLL  c|j5t|jdt tjd|jdt tjj|jg} |j }n*#t$rt dYnwxYw|D]}|jdt|jzt |j|jdt|jzt |jddddS#1swxYwYdS)Ncpu_util_percent)intervalram_util_percentzGPUtil failed to retrieve GPUs.gpu_util_percentvram_util_percent)r/r'r,appendfloat cpu_percentvirtual_memorypercentrgetGPUs Exceptionr%debugstridload memoryUtil)r2gpu_listgpus r_read_utilizationzUtilMonitor._read_utilizationMs Y  ! ./66&,d;;;<< ./66&/119::{&D#{2244HH DDDLL!BCCCCCD#CK 2S[[ @AHHchK 3c#&kk ABIIcn--%                  s7BFB76F7$CFCBFFFc|jriS|j5tj|j}|jD] \}}|dd= dddn #1swxYwYdd|DiS)Nperfcdi|]-\}}t|dk|tj|.S)r)lennpmean).0kvs r z(UtilMonitor.get_data..ls2TTT41aQRSBGAJJr4)r$r/copydeepcopyr,items)r2 ret_valueskeyvals rget_datazUtilMonitor.get_datads < I Y  t{33J K--//  SFF                TT:3C3C3E3ETTTUUs>AA #A cd|_|js6|tj|j|j4dSdSNF)r$rItimesleepr*r2s rrunzUtilMonitor.runnsW , #  " " $ $ $ Jtz " " ", # # # # #r4cd|_dS)NT)r$r_s rstopzUtilMonitor.stopts  r4)Tr") __name__ __module__ __qualname____doc__r)rIrZr`rb __classcell__)r3s@rr!r!-s*.VVV### r4r!fnexception_type num_retries sleep_timetimeoutreturnc tjfd}t|D]}tj|}d|_||||r6t d|dzd|dtd dn sdStj|d S) Nc dS#$r9}t|Yd}~dSd}~wwxYwN)r%r&set)eerroredrkrjs r_try_fnzretry_fn.._try_fnsa  BDDDDD    NN1    KKMMMMMMMMM s A .AA )targetT)rnzProcess timed out (try ri/z): rcF)r-Eventrangeclearrr0r1joinis_aliver%rBgetattris_setr]r^) rjrkrlrmrnrviprocrus `` @rretry_fnrxs4oG;   w///   ' """ ==??  LL3!A#33 332z40033    !! 44 : 5r4c eZdZdZeejddZdZ dde de ed e e d e fd Z d Zd ZdS) warn_if_slowa)Prints a warning if a given operation is slower than 500ms. Example: >>> from ray.tune.utils.util import warn_if_slow >>> something = ... # doctest: +SKIP >>> with warn_if_slow("some_operation"): # doctest: +SKIP ... ray.get(something) # doctest: +SKIP TUNE_WARN_THRESHOLD_Sg?zTThe `{name}` operation took {duration:.3f} s, which may be a performance bottleneck.NFname thresholdmessagedisablech||_|p|j|_|p|j|_d|_||_dSr\)rDEFAULT_THRESHOLDrDEFAULT_MESSAGErtoo_slowr)r2rrrrs rr)zwarn_if_slow.__init__s; "   $ $%8 9 99r4c.|tjdfS)zConverts obj to a form that can be pinned in object store memory. Currently only numpy arrays are pinned in memory, if you have a strong reference to the array value. ri)rNzerosobjs r _to_pinnablers ! r4c|dS)z"Retrieve from _to_pinnable format.rrrs r_from_pinnabler(s  q6Mr4 trainablecddlmm}fd}td|d |dt |z|dtd d S#t $r}td |Yd }~nd }~wwxYwtd t |dt j|}t}|j r8tdt|j d||j |d|j r8tdt|j d||j |d|std|Std|d|S)aUtility for detecting why your trainable function isn't serializing. Args: trainable: The trainable object passed to tune.Tuner(trainable). Currently only supports Function API. Returns: bool | set of unserializable objects. Example: .. code-block:: python import threading # this is not serializable e = threading.Event() def test(): print(e) diagnose_serialization(test) # should help identify that 'e' should be moved into # the `test` scope. # correct implementation def test(): e = threading.Event() print(e) assert diagnose_serialization(test) is True r)_check_serializabilityregister_trainablecR|D]\}}d} ||d}nL#t$r?}d}|jjdt |}||Yd}~nd}~wwxYw|t |d|d||r ||dS)NPASSEDFAILEDz: z[name='z'']... )rVrAr3rcrCadd) objects failure_setprintervar_namevariablerstatusrtrs rcheck_variablesz/diagnose_serialization..check_variablesSs")--//   HhC *&&x:::! * * *!-99Q99)))))))) * Gs8}}FFXFFfFF G G G    s- A65A11A6zTrying to serialize z...z__test:F)warnzSerialization succeeded!TzSerialization failed: NzIInspecting the scope of the trainable by running `inspect.getclosurevars(z)`...z Detected z. global variables. Checking serializability...c&td|zSNz printrs rz(diagnose_serialization..tseai@P@Pr4z0 nonlocal variables. Checking serializability...c&td|zSrrrs rrz(diagnose_serialization..{s%PQ BRBRr4zNothing was found to have failed the diagnostic test, though serialization did not succeed. Feel free to raise an issue on github.z Variable(s) z was found to be non-serializable. Consider either removing the instantiation/imports of these objects or moving them into the scope of the trainable. ) ray.tune.registryrrrrCrAinspectgetclosurevarsrsglobalsrM nonlocals)rrrrtclosurerrs @rdiagnose_serializationr.s-FMLLLLLLL      / / / /000,9s9~~5yuMMMM ()))t ,,, *q**++++++++,  9#&y>> 9 9 9$Y//G%%KR  *GO,, * * *    6P6PQQQT  *G-.. * * *    );8R8RSSS        ;      s-A A8A33A8statecheckpoint_dir file_name tmp_file_namec4ddlm}tj||}t |d5}|||dddn #1swxYwYtj|tj||dS)aSAtomically saves the state object to the checkpoint directory. This is automatically used by Tuner().fit during a Tune job. Args: state: Object state to be serialized. checkpoint_dir: Directory location for the checkpoint. file_name: Final name of file. tmp_file_name: Temporary name of file. rNwb)ray.cloudpickle cloudpicklerpathr|opendumpreplace)rrrrrtmp_search_ckpt_pathfs r _atomic_savers*)))))7<< FF "D ) )#Q"""###############J#RW\\.)%L%LMMMMMsAA!Adirpath ckpt_patterncddlm}tjtj||}|sdSt |}t|d5}||}dddn #1swxYwY|S)ahReturns the most recently modified checkpoint. Assumes files are saved with an ordered name, most likely by :obj:atomic_save. Args: dirpath: Directory in which to look for the checkpoint file. ckpt_pattern: File name pattern to match to find checkpoint files. Returns: (dict) Deserialized state dict. rNrb) rrglobrrr|maxrrE)rrr full_pathsmost_recent_checkpointrcheckpoint_states r_load_newest_checkpointr s*)))))27<<>>??J  __ $d + +/q&++A../////////////// sA>>BBbeta) stability{Gz?gpu_id target_utilretrydelay_sgpu_memory_limitcL t}|td,tj}|std|dd t t r'rtn8d n5t ts tdtd fd  fd | D}|vrtd |d tt|D]x}t fd | D} | j |kr;td|d| j dt!j|vdStd)aOChecks if a given GPU has freed memory. Requires ``gputil`` to be installed: ``pip install gputil``. Args: gpu_id: GPU id or uuid to check. Must be found within GPUtil.getGPUs(). If none, resorts to the first item returned from `ray.get_gpu_ids()`. target_util: The utilization threshold to reach to unblock. Set this to 0 to block until the GPU is completely free. retry: Number of times to check GPU limit. Sleeps `delay_s` seconds between checks. delay_s: Seconds to wait before check. Returns: bool: True if free. Raises: RuntimeError: If GPUtil is not found, if no GPUs are detected or if the check fails. Example: .. code-block:: python def tune_func(config): tune.utils.wait_for_gpu() train() tuner = tune.Tuner( tune.with_resources( tune_func, resources={"gpu": 1} ), tune_config=tune.TuneConfig(num_samples=10) ) tuner.fit() Nz3GPUtil must be installed if calling `wait_for_gpu`.zPNo GPU ids found from `ray.get_gpu_ids()`. Did you set Tune resources correctly?rrDuuidzgpu_id (z) must be type str/int.c$t|Srr)r~)ggpu_attrs r gpu_id_fnzwait_for_gpu..gpu_id_fnsq(###r4c&h|] }|Srr)rPrrs r zwait_for_gpu.. s!666yy||666r4z% not found in set of available GPUs: zt. `wait_for_gpu` takes either GPU ordinal ID (e.g., '0') or UUID (e.g., 'GPU-04546190-b68d-65ac-101b-035f8faed77d').c3:K|]}|k|VdSrrr)rPrrrs r zwait_for_gpu..s5PP19O9O!9O9O9O9OPPr4zWaiting for GPU util to reach z. Util: z0.3fTzGPU memory was not freed.)r RuntimeErrorray get_gpu_ids isinstancerCisdigitintrrr@rznextrFr%infor]r^) rrrrrr gpu_id_listgpu_idsr gpu_objectrrs ` @@r wait_for_gpur+s^  F ~PQQQ ~o''  8 QH&# K >>   [[FF HH  $ $KIDLLIIIJJJ$$$$$ 7666V^^%5%5666G W G GG G G G   3u::    PPPPPV^^%5%5PPPPP  ; . . KK666#.566    Jw    44 2 3 33r4 trainable_clsconfignum_gpusctjs Jdtj||}||}||}ddlm}t dD]-}tj|j}.||s Jdtj|j|j tj|j}||dksJtj|j}||d ksJd S) aHelper method to check if your Trainable class will resume correctly. Args: trainable_cls: Trainable class for evaluation. config: Config to pass to Trainable when testing. num_gpus: GPU resources to allocate when testing. use_object_store: Whether to save and restore to Ray's object store. Recommended to set this to True if planning to use algorithms that pause training (i.e., PBT, HyperBand). zNeed Ray to be initialized.)r.)r-r)TRAINING_ITERATIONrhzQValidation will not pass because it requires `training_iteration` to be returned.rT) r!is_initializedremoteray.air.constantsr0rzrtrainrestoresave) r,r-r. remote_cls trainable_1 trainable_2r0_ress rvalidate_save_restorer= s~    >>!>>> .X...}==J##6#22K##6#22K444444 1XX22gk'..0011 77% & &  & GK  & &{'7'>'>'@'@ A ABBB '+#**,, - -C ! "a ' ' ' ' '+#**,, - -C ! "a ' ' ' ' 4r4ctj|}d} |in@#t$r3}tt |d}Yd}~nd}~wwxYw|S)zCheck if func({}) works.TFN)r signaturebindrAr%rBrC)funcfunc_siguse_config_singlerts r_detect_config_singlerDJs &&H" b """ SVV!" s. A+)A&&A+Tparameter_namespoints_to_evaluateevaluated_rewardsvalidate_point_name_lengthsc|rt|ts/tdt ||D]}t|t tfstd|d|r[t |t |ks;td|d|zdz|r|rt|ts/tdt |t |t |ks?td|d |zdzd Sd Sd S) a5Generic validation of a Searcher's warm start functionality. Raises exceptions in case of type and length mismatches between parameters. If ``validate_point_name_lengths`` is False, the equality of lengths between ``points_to_evaluate`` and ``parameter_names`` will not be validated. z1points_to_evaluate expected to be a list, got {}.z9points_to_evaluate expected to include list or dict, got .zDim of point {}z and parameter_names {}z do not match.z0evaluated_rewards expected to be a list, got {}.zDim of evaluated_rewards {}z and points_to_evaluate {}N)r#r+ TypeErrorrrdictrMr)rErFrGrHpoints rvalidate_warmstartrNVs,d33 CJJ+,,  (  EedD\22 $ $$$ + CJJ#oBVBV4V4V %,,U33/66GGH&'  / +T22 BII*++  $%%-?)@)@@@-445FGG.556HIIJ"#     A@r4)NrrrNr)T)BrTrrloggingrr-r] collectionsrrnumbersrrtypingrrr r r r r rnumpyrNr!ray._private.dictrrrrrrray.air._internal.jsonrray.air._internal.utilrrray.util.annotationsrrr' getLoggerrcr%rrr!rAr%rrrobjectrrrrrrCrr r<r+r=rDrNrr4rrZs   ######MMMMMMMMMMMMMMMMMMMM 7666668888888888888888  8 $ $ GGGGG&GGGTIR $ $$S$$y/8DO+DDE$$ $ f  $  $$$$N(S(S(S(S(S(S(S(SVI"I"I"I"I"&I"I"I"X::: \h\\\\~NNcNcNRUNNNN(S2 V(,(, a4a4 U38_ %a4a4 a4 a4 uo a4a4a4a4H"&&& TN&&&&&R     )- //#Y/U4:.///"& /// ///r4