&`iddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m Z ejeZGdde ZGd d ZGd d e ZdS) N) defaultdict)Path)Dict)Callback)Trialc(eZdZdZ d dZdZdS) FailureInjectorCallbackz3Adds random failure injection to the TrialExecutor.~/ray_bootstrap_config.yaml皙?rFc||_t||_||_||_tj|_ dSN) probabilityr expanduseras_posix config_pathdisabletime_between_checkstime monotoniclast_fail_check)selfrrrrs g/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/utils/mock.py__init__z FailureInjectorCallback.__init__sY' ,,7799BBDD #6 #~//c tj|jsdSt j|j|jzkrdSt j|_ddl}ddl m }d}d}tj |j kr|j stj |j k}||kr ||jd|ddS#|jj$rS|dz }t dt'|t'||z YnwxYw||kdSdSdSdS)Nr) kill_nodeT)yeshardoverride_cluster_namez@Killing random node failed in attempt {}. Retrying {} more times)ospathexistsrrrrrclick ray.autoscaler._private.commandsrrandomrr exceptionsClickExceptionlogger exceptionformatstr)rinfor%rfailures max_failuresshould_terminates r on_step_beginz%FailureInjectorCallback.on_step_begin!sw~~d.//  F >  d2T5MM M M F#~// >>>>>> =??T- - -dl -%}1AA \))I( -.2  F'6MH$$117MM3|h/F+G+G22\)))) . - - -*)s9CAD10D1N)r r rF)__name__ __module__ __qualname____doc__rr2rrr r sK==2 0 0 0 0!!!!!rr cLeZdZdZdZdeeeffdZdefdZ de fdZ dS) TrialStatusSnapshotzvA sequence of statuses of trials as they progress. If all trials keep previous status, no snapshot is taken. cg|_dSr  _snapshot)rs rrzTrialStatusSnapshot.__init__Ks r new_snapshotcv|sdS|jr||jdkr|j|dSdS)z*May append a new snapshot to the sequence.N)r<append)rr=s rr@zTrialStatusSnapshot.appendNsO  F~ 01C!C!C N ! !, / / / / /"D!Crreturncd}|jD]4}d}|D]}||tjkr|dz }t||}5|S)zOutputs the max number of running trials at a given time. Usually used to assert certain number given resource restrictions. rr!)r<rRUNNINGmax)rresultsnapshotcounttrial_ids rmax_running_trialsz&TrialStatusSnapshot.max_running_trialsVs`  ( (HE$  H%66QJE''FF rcf|jsdS|jdtfdDS)z"True if all trials are terminated.Fr?c3DK|]}|tjkVdSr )r TERMINATED).0rH last_snapshots r z@TrialStatusSnapshot.all_trials_are_terminated..jsA  0000 C     4      rr9c$eZdZdZdefdZdZdS)TrialStatusSnapshotTakerzCollects a sequence of statuses of trials as they progress. If all trials keep previous status, no snapshot is taken. rFc||_dSr r;)rrFs rrz!TrialStatusSnapshotTaker.__init__us !rc tt}|D]}|j||j<|j|dSr )rr-statusrHr<r@)r iterationtrialskwargsr=trials r on_step_endz$TrialStatusSnapshotTaker.on_step_endxsK"3''  8 8E+0res ######&&&&&&%%%%%%  8 $ $33333h333l' ' ' ' ' ' ' ' T , , , , ,x , , , , ,r