&`ipddlZddlZddlZddlmZmZddlmZmZm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlZddlZddlmZddlmZddlmZmZdd l m!Z!m"Z"e r(ddl#Zdd l$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+ddl,m-Z-ddl.m/Z/edee0effZ1dZ2dZ3dZ4ej5e6Z7dddeee0efde0fdZ8eedGddZ9eedGddZ:eedGddZ;eedGd d!Z.equalsHs- =]24 =M15 %%(z, c3*K|]\}}|d|VdS)=Nr').0namer(s r* z"_repr_dataclass..YsE +e4%r,)) rgetattrr1getdefault isinstancer __class____name__joinitems) objr#non_default_valuesr+fieldr(r) is_requiredstrings r*_repr_dataclassrA2s"&&&33UZ((&**5:u}EE   >>  3ffUM:: 3-2 uz * & ) ) )F dii/A/G/G/I/IF cMF Mr,stable) stabilityceZdZUdZdZeeeefe d<dZ ee efe d<dZ ee efe d<dZeeeefe d<d Zeeefe d <dZeee d <d Zd ZdefdZddde fdZedZedZedZedZedZedZddZe ddZdS) ScalingConfigaM Configuration for scaling training. For more details, see :ref:`train_scaling_config`. Args: trainer_resources: Resources to allocate for the training coordinator. The training coordinator launches the worker group and executes the training function per worker, and this process does NOT require GPUs. The coordinator is always scheduled on the same node as the rank 0 worker, so one example use case is to set a minimum amount of resources (e.g. CPU memory) required by the rank 0 node. By default, this assigns 1 CPU to the training coordinator. num_workers: The number of workers (Ray actors) to launch. Each worker will reserve 1 CPU by default. The number of CPUs reserved by each worker can be overridden with the ``resources_per_worker`` argument. use_gpu: If True, training will be done on GPUs (1 per worker). Defaults to False. The number of GPUs reserved by each worker can be overridden with the ``resources_per_worker`` argument. resources_per_worker: If specified, the resources defined in this Dict is reserved for each worker. Define the ``"CPU"`` key (case-sensitive) to override the number of CPUs used by each worker. This can also be used to request :ref:`custom resources `. placement_strategy: The placement strategy to use for the placement group of the Ray actors. See :ref:`Placement Group Strategies ` for the possible options. accelerator_type: [Experimental] If specified, Ray Train will launch the training coordinator and workers on the nodes with the specified type of accelerators. See :ref:`the available accelerator types `. Ensure that your cluster has instances with the specified accelerator type or is able to autoscale to fulfill the request. Example: .. code-block:: python from ray.train import ScalingConfig scaling_config = ScalingConfig( # Number of distributed workers. num_workers=2, # Turn on/off GPU. use_gpu=True, # Assign extra CPU/GPU/custom resources per worker. resources_per_worker={"GPU": 1, "CPU": 1, "memory": 1e9, "custom": 1.0}, # Try to schedule workers on different nodes. placement_strategy="SPREAD", ) Ntrainer_resources num_workersFuse_gpuresources_per_workerPACKplacement_strategyaccelerator_typec|jrB|js|jdkrtd|jr|jdkrtddSdSdS)Nrz`use_gpu` is False but `GPU` was found in `resources_per_worker`. Either set `use_gpu` to True or remove `GPU` from `resources_per_worker.z`use_gpu` is True but `GPU` is set to 0 in `resources_per_worker`. Either set `use_gpu` to False or request a positive number of `GPU` in `resources_per_worker.)rJrInum_gpus_per_worker ValueErrorselfs r* __post_init__zScalingConfig.__post_init__s  $ < D$z@ScalingConfig._resources_per_worker_not_none..s*$$$Aa1ff1fffr,gMbP?)rJrIr; setdefaultrMr)rRrJ accelerators r*_resources_per_worker_not_nonez,ScalingConfig._resources_per_worker_not_nones  $ ,| 2).qz$$(-qz$$$$!%!:!@!@!B!B$$$  < 6 + +E1 5 5 5   @7P9NPPK + +K ? ? ?##r,c|j(|jr ddl}d}n#t$rd}YnwxYwd}d|i}n#d|jD}|S)NrrGrcc&i|]\}}|dk ||Srer'rfs r*riz=ScalingConfig._trainer_resources_not_none..s*!!!A1661666r,)rFrH google.colab ImportErrorr;)rRgoogletrainer_num_cpusrFs r*_trainer_resources_not_nonez)ScalingConfig._trainer_resources_not_nones  ! ) %)'''''($$")))'($$$) $% !&(8 9  !!!%!7!=!=!?!?!!! ! s  &&ctt|j}|jD]\}}||xx||jzz cc<t |S)z0Map of total resources required for the trainer.)rfloatrsrlr;rHdict)rRtotal_resource_maprgr(s r*total_resourceszScalingConfig.total_resourcesso)0PQQ;AACC > >HAu q ! ! !UT-=%= = ! ! ! !&'''r,c8|jddS)z%The number of CPUs to set per worker.rcrrlr5rQs r*num_cpus_per_workerz!ScalingConfig.num_cpus_per_worker266ua@@@r,c8|jddS)z%The number of GPUs to set per worker.rbrrzrQs r*rOz!ScalingConfig.num_gpus_per_workerr|r,cHd|jDS)z9Resources per worker, not including CPU or GPU resources.c"i|] \}}|dv || S))rcrbr'rfs r*rizAScalingConfig.additional_resources_per_worker..s3   1&& q&&&r,)rlr;rQs r*additional_resources_per_workerz-ScalingConfig.additional_resources_per_workers2  ;AACC    r,rcddlm}|j}|j}t t |t |z}i|g|g|jdz zz}|||jS)z>Returns a PlacementGroupFactory to specify resources for Tune.rrrG)strategy)#ray.tune.execution.placement_groupsrrsrlrvrrHrL)rRrtrainer_bundle worker_bundlecombined_bundlebundless r*r_z(ScalingConfig.as_placement_group_factory sMMMMMM9; w~669O9OOPP'=/T=MPQ=Q*RR$$Wt7NOOOOr,pgfc2t|j}|jd}|jd}t|dd}|j}t t |t |z }t|||||S)acCreate a ScalingConfig from a Tune's PlacementGroupFactory Note that this is only needed for ResourceChangingScheduler, which modifies a trial's PlacementGroupFactory but doesn't propagate the changes to ScalingConfig. TrainTrainable needs to reconstruct a ScalingConfig from on the trial's PlacementGroupFactory. rrbF)rFrHrIrJrL)lenrboolr5rrvrrE)clsrrHcombined_resourcesrJrIrLrFs r*from_placement_group_factoryz*ScalingConfig.from_placement_group_factorys#+&&  [^"{2+//u==>> \! & ' ''2F*G*G G  /#!51     r,)r$r)rrr$rE) r9 __module__ __qualname____doc__rFrrr SampleRange__annotations__rHintrIrrJrLstrrMrSrWr\r`propertyrlrsrxr{rOrr_ classmethodrr'r,r*rErEas33j=AxdK&7 89@@@+,KsK'(,,,(-GU4$ %---?C(5{):#;<CCC28c;./888&*hsm***"%%%ISIIIISSDSSSS $$X$,!!X!8((X(AAXAAAXA  X  P P P P! )! ! ! ! [! ! ! r,rEcReZdZUdZdZeed<dZee e fed<dZ dZ dZ d S) FailureConfigaeConfiguration related to failure handling of each training/tuning run. Args: max_failures: Tries to recover a run at least this many times. Will recover from the latest checkpoint if present. Setting to -1 will lead to infinite recovery retries. Setting to 0 will disable retries. Defaults to 0. fail_fast: Whether to fail upon the first error. If fail_fast='raise' provided, the original error during training will be immediately raised. fail_fast='raise' can easily leak resources and should be used with caution. r max_failuresF fail_fastct|jts5|jdkst d|jd|jr0|jdkr't dt |jddSdS)NRAISEz.fail_fast must be one of {bool, 'raise'}. Got .rz$max_failures must be 0 if fail_fast=)r7rrupperrPrreprrQs r*rSzFailureConfig.__post_init__Rs4>400 DN4H4H4J4Jg4U4UUDNUUU  > d/144NtDN7K7KNNN   44r,c t|SrUrVrQs r*rWzFailureConfig.__repr___rXr,ctdtddg|j|jgddddd S) NscrollableTable.html.j2z Max failuresz Fail fastSettingValuehtmlFkeystablefmt showindexheadersnonetable max_height)rrenderrrrrQs r*r\zFailureConfig._repr_html_bsi12299 . <"/@ :   r,N)r9rrrrrrrrrrrSrWr\r'r,r*rr?sx  L#"'IuT3Y'''   %%%      r,rc eZdZUdZdZeeed<dZee ed<e Z ee ed<dZ eeed<dZ eeed<eZeeed <eZeeed <d Zd Zd e fdZed ee fdZdS)CheckpointConfiga0 Configurable parameters for defining the checkpointing strategy. Default behavior is to persist all checkpoints to disk. If ``num_to_keep`` is set, the default retention policy is to keep the checkpoints with maximum timestamp, i.e. the most recent checkpoints. Args: num_to_keep: The number of checkpoints to keep on disk for this run. If a checkpoint is persisted to disk after there are already this many checkpoints, then an existing checkpoint will be deleted. If this is ``None`` then checkpoints will not be deleted. Must be >= 1. checkpoint_score_attribute: The attribute that will be used to score checkpoints to determine which checkpoints should be kept on disk when there are greater than ``num_to_keep`` checkpoints. This attribute must be a key from the checkpoint dictionary which has a numerical value. Per default, the last checkpoints will be kept. checkpoint_score_order: Either "max" or "min". If "max", then checkpoints with highest values of ``checkpoint_score_attribute`` will be kept. If "min", then checkpoints with lowest values of ``checkpoint_score_attribute`` will be kept. checkpoint_frequency: Number of iterations between checkpoints. If 0 this will disable checkpointing. Please note that most trainers will still save one checkpoint at the end of training. This attribute is only supported by trainers that don't take in custom training loops. checkpoint_at_end: If True, will save a checkpoint at the end of training. This attribute is only supported by trainers that don't take in custom training loops. Defaults to True for trainers that support it and False for generic function trainables. _checkpoint_keep_all_ranks: This experimental config is deprecated. This behavior is now controlled by reporting `checkpoint=None` in the workers that shouldn't persist a checkpoint. For example, if you only want the rank 0 worker to persist a checkpoint (e.g., in standard data parallel training), then you should save and report a checkpoint if `ray.train.get_context().get_world_rank() == 0` and `None` otherwise. _checkpoint_upload_from_workers: This experimental config is deprecated. Uploading checkpoint directly from the worker is now the default behavior. N num_to_keepcheckpoint_score_attributecheckpoint_score_orderrcheckpoint_frequencycheckpoint_at_end_checkpoint_keep_all_ranks_checkpoint_upload_from_workersc|jtkrtd|jtkrtd|j#|jdkrt d|jd|jttfvr t dtdtd|j dkrt d |j dS) NaThe experimental `_checkpoint_keep_all_ranks` config is deprecated. This behavior is now controlled by reporting `checkpoint=None` in the workers that shouldn't persist a checkpoint. For example, if you only want the rank 0 worker to persist a checkpoint (e.g., in standard data parallel training), then you should save and report a checkpoint if `ray.train.get_context().get_world_rank() == 0` and `None` otherwise.zThe experimental `_checkpoint_upload_from_workers` config is deprecated. Uploading checkpoint directly from the worker is now the default behavior.rzReceived invalid num_to_keep: z". Must be None or an integer >= 1.z'checkpoint_score_order must be either "z" or "z".z&checkpoint_frequency must be >=0, got ) r_DEPRECATED_VALUEDeprecationWarningrrrPrMAXMINrrQs r*rSzCheckpointConfig.__post_init__s  *.? ? ?$(     /3D D D$,    'D,<,A,A4#444   &sCj 8 8PcPPPPP   $q ( (T9RTT  ) (r,c t|SrUrVrQs r*rWzCheckpointConfig.__repr__rXr,r$c |jd}n|j}|jd}n|j}|jd}n|j}tdt gd|||j|j|gdddd d S) NAllz Most recentr)zNumber of checkpoints to keepzCheckpoint score attributezCheckpoint score orderzCheckpoint frequencyzCheckpoint at endrrFrrrr)rrrrrrrr)rRnum_to_keep_reprcheckpoint_score_attribute_reprcheckpoint_at_end_reprs r*r\zCheckpointConfig._repr_html_s   #$  #/   * 2.; + +.2.M +  ! )%' " "%)%; "12299   )731.   '*-:   r,c\|j|jSd}|jtkrd}||jS)zlSame as ``checkpoint_score_attr`` in ``tune.run``. Only used for Legacy API compatibility. Nrzmin-)rrr)rRprefixs r*"_tune_legacy_checkpoint_score_attrz3CheckpointConfig._tune_legacy_checkpoint_score_attrsB  * 22 2  &# - -F;$9;;;r,)r9rrrrrrrrrrrrrrrrrrSrWr\rrr'r,r*rrqs**X"&K#%%%04 444,/HSM///*+(3-+++(,x~,,,1BBBB6G#Xd^GGG"""H%%%' S' ' ' ' R  `bucket/path`). failure_config: Failure mode configuration. checkpoint_config: Checkpointing configuration. sync_config: Configuration object for syncing. See train.SyncConfig. verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = default, 2 = verbose. Defaults to 1. If the ``RAY_AIR_NEW_OUTPUT=1`` environment variable is set, uses the old verbosity settings: 0 = silent, 1 = only status updates, 2 = status and brief results, 3 = status and detailed results. stop: Stop conditions to consider. Refer to ray.tune.stopper.Stopper for more info. Stoppers should be serializable. callbacks: [DeveloperAPI] Callbacks to invoke. Refer to ray.tune.callback.Callback for more info. Callbacks should be serializable. Currently only stateless callbacks are supported for resumed runs. (any state of the callback will not be checkpointed by Tune and thus will not take effect in resumed runs). progress_reporter: [DeveloperAPI] Progress reporter for reporting intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in a Jupyter notebook. log_to_file: [DeveloperAPI] Log stdout and stderr to files in trial directories. If this is `False` (default), no files are written. If `true`, outputs are written to `trialdir/stdout` and `trialdir/stderr`, respectively. If this is a single string, this is interpreted as a file relative to the trialdir, to which both streams are written. If this is a Sequence (e.g. a Tuple), it has to have length 2 and the elements indicate the files to which stdout and stderr are written, respectively. Nr1 storage_pathstorage_filesystemfailure_configcheckpoint_configzray.train.SyncConfig sync_configrrverboserstopr callbacksz+ray.tune.progress_reporter.ProgressReporterprogress_reporterF log_to_file local_dircddlm}ddlm}ddlm}m}|jtd|j h||_ tj d}|@td|tjdt"d ||_ |jst'|_|js||_|jst-|_|j|_|j||jpd |_t5|j t6r |j |_ dSdS) Nr SyncConfig)DEFAULT_STORAGE_PATH)rget_air_verbosityzThe `RunConfig(local_dir)` argument is deprecated. You should set the `RunConfig(storage_path)` instead.See the docs: https://docs.ray.io/en/latest/train/user-guides/persistent-storage.html#setting-the-local-staging-directory RAY_STORAGEz8Using configured Ray Storage URI as the `storage_path`: zcThe `RAY_STORAGE` environment variable is deprecated. Please use `RunConfig(storage_path)` instead.) stacklevel) ray.trainrray.train.constantsrray.tune.experimental.outputrrrrrosenvironr5loggerinfowarningswarnrrrrrrr_verboseDEFAULTr7ras_posix)rRrrrrray_storage_uris r*rSzRunConfig.__post_init__Is((((((<<<<<<PPPPPPPP > %$N    $ 4D .0Z^^M-J-JO* )&)) D)  %4!" 2"///D  ,)z||D % 8%5%7%7D "  <  -,\-ABBGaDL d' . . = $ 1 : : < .J.J.L.L;     ! - LL233::-D4J4V4V4X4X;   01188::;q3u::~PQ?QR  33Q3566== I/3~!%#'#3  %''"G,   >   ,--44.//66%!75   r,)"r9rrrr1rrrrrpyarrowfs FileSystemrrrrrrrrrrr rrr rrrrrSrWr\r'r,r*rrs//bD(3-"&L(3-&&&:>!67>>>.2NH]+22248x 0188848K01888AEGXeC<= >EEEPTD(5)XsGnd6J-KKL MTTT,0IxZ()000QUx MNUUU5:KtS%S/12::: $Ix}###2=2=2=h    . S. . . . . . r,r)=loggingrr collectionsrr dataclassesrrrpathlibrtypingr r r r r rrrr pyarrow.fsrrayray._common.utilsr)ray._private.thirdparty.tabulate.tabulaterray.util.annotationsrr ray.widgetsrrray.tune.progress_reporterray.tune.callbackrrrrrray.tune.search.samplerray.tune.stopperrray.tune.utils.logrrrrrr getLoggerr9rrArErrrr'r,r*r s ,,,,,,,,8888888888                       888888>>>>>>AAAAAAAA66666666-%%%%******IIIIII999999------((((((,,,,,, Hd39o-.      8 $ $HL,,,HT#s(^,D,PS,,,,^  XY Y Y Y Y Y Y  Y x  X- - - - - - -  - `  XP<P<P<P<P<P<P< P