
    &`iK&              	          d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d d	lmZ d d
lmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z&  ej'        e(          Z)dZ*dZ+e& G d de"                      Z,e&	 dde
e	ge	f         dee-         ded         fd            Z.dS )    N)partial)Number)AnyCallableDictOptionalType)RunnerThreadStartTraceback)_ERROR_FETCH_TIMEOUT)_TrainingResult)	TrialInfo_TrainSessionget_sessioninit_sessionshutdown_session)PlacementGroupFactory)DEFAULT_METRICRESULT_DUPLICATESHOULD_CHECKPOINT)	Trainable)_detect_config_single)DeveloperAPIz.null_markerz.temp_markerc                   x    e Zd ZdZdZd Zdeeef         fdZ	d Z
d Zd Zdd
efdZdefdZd Zd ZddZdS )FunctionTrainablezwTrainable that runs a user function reporting results.

    This mode of execution does not support checkpoint/restore.funcc                      t           fdt           j         j         j         j        j        d d  j        j                   j        dd d d d d d d            d  _        d S )Nc                  8                           j                  S N_trainable_funcconfigselfs   y/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/tune/trainable/function_trainable.py<lambda>z)FunctionTrainable.setup.<locals>.<lambda>,       $"6"6t{"C"C     nameid	resourceslogdir	driver_ipdriver_node_idexperiment_nameT)training_func
trial_infostoragesynchronous_result_reporting
world_rank
local_rank	node_ranklocal_world_size
world_sizedataset_shard
checkpoint)	r   r   
trial_nametrial_idtrial_resources_storagetrial_driver_staging_pathexperiment_dir_name_last_training_resultr$   r"   s   ` r%   setupzFunctionTrainable.setup*   s    CCCC _=.}># $ A   M)-!)	
 	
 	
 	
, AE"""r(   r"   c                     t           )z7Subclasses can override this to set the trainable func.)NotImplementedErrorrC   s     r%   r!   z!FunctionTrainable._trainable_funcC   s
     "!r(   c                       fd}t          | j        d           _         j                                         	  j                                         d S # t          $ r Y d S w xY w)Nc                  n    	                      j                  S # t          $ r} t          | d } ~ ww xY wr   )r!   r"   	Exceptionr   )er$   s    r%   
entrypointz,FunctionTrainable._start.<locals>.entrypointI   sE    ,++DK888 , , ,$!+,s    
4/4T)targeterror_queuedaemon)r
   _error_queue_runner_status_reporter_startstartRuntimeError)r$   rK   s   ` r%   rR   zFunctionTrainable._startH   s    	, 	, 	, 	, 	, $4+<T
 
 
 	$$&&&	L      	 	 	 DD	s   A 
A&%A&c                    t                      }|j        s|                                 |                                }|st	          d          |j        }t          |v r
d|t          <   || _        |j	        
d|t          <   |S )a=  Implements train() for a Function API.

        If the RunnerThread finishes without reporting "done",
        Tune will automatically provide a magic keyword __duplicate__
        along with a result with "done=True". The TrialRunner will handle the
        result accordingly (see tune/tune_controller.py).
        zShould not have reached here. The TuneController should not have scheduled another `train` remote call.It should have scheduled a `stop` instead after the training function exits.FNT)
r   training_startedrS   get_nextrT   metricsr   r   rB   r;   )r$   sessiontraining_resultrX   s       r%   stepzFunctionTrainable.step\   s     "-' 	MMOOO5<5E5E5G5G 	 5   ") w&&).G%&%4"%1
 *.G%&r(   c                      ||           S r    )r$   fns     r%   executezFunctionTrainable.execute   s    r$xxr(    checkpoint_dirc                 2    |rt          d          | j        S )Nz4Checkpoint dir should not be used with function API.)
ValueErrorrB   )r$   ra   s     r%   save_checkpointz!FunctionTrainable.save_checkpoint   s%     	USTTT
 ))r(   checkpoint_resultc                 :    t                      }|j        |_        d S r   )r   r;   loaded_checkpoint)r$   re   rY   s      r%   load_checkpointz!FunctionTrainable.load_checkpoint   s    --$5$@!!!r(   c                     t                      }	 |                    d           |                                 t                       d S # |                                 t                       w xY w)Nr   timeout)r   finish_report_thread_runner_errorr   )r$   rY   s     r%   cleanupzFunctionTrainable.cleanup   ss    --	 NN1N%%% //111 //111s   A
 
$A.c                     t                      }t          t          j                            dd                    }|                    |           |j                                        rdS |                     fdt           j
         j         j         j        j        d d  j        j                   j                   i  _        dS )	NTUNE_FUNCTION_THREAD_TIMEOUT_S   rj   Fc                  8                           j                  S r   r    r#   s   r%   r&   z0FunctionTrainable.reset_config.<locals>.<lambda>   r'   r(   r)   )r1   r2   r3   T)r   intosenvirongetrl   training_threadis_aliveresetr   r<   r=   r>   r?   trial_working_directoryrA   _last_result)r$   
new_configrY   thread_timeouts   `   r%   reset_configzFunctionTrainable.reset_config   s    -- RZ^^,LaPPQQ~..."++-- 	5CCCC _=.}<# $ A   M 	 	
 	
 	
 tr(   Fc                     	 | j                             |t                    }t          |# t          j        $ r Y d S w xY w)N)blockrk   )rO   rv   r   r   queueEmpty)r$   r   rJ   s      r%   rm   z-FunctionTrainable._report_thread_runner_error   sO    	!%%E;O%PPA a'{ 	 	 	DD	s   )+ >>N)r`   )F)__name__
__module____qualname____doc___namerD   r   strr   r!   rR   r[   r_   rd   r   rh   rn   r~   rm   r]   r(   r%   r   r   "   s        C C EE E E2"d38n " " " "
  (( ( (T  * *c * * * *A A A A A

 
 
  6     r(   r   
train_funcr*   returnc                 (    t           f}t           d          r
 j        |z   }t          j                   j        }t                     }|s"t          d                    |                    t           dd            G  fddg|R  }|S )N
__mixins__zUnknown argument found in the Trainable function. The function args must include a single 'config' positional parameter.
Found: {}
_resourcesc                       e Zd Z p ed          rj         ndZd ZfdZedee	e
f         dee         ffd            ZdS )	#wrap_function.<locals>.ImplicitFuncr   r   c                     | j         S r   )r   r#   s    r%   __repr__z,wrap_function.<locals>.ImplicitFunc.__repr__   s
    :r(   c                    t          |          }d }d }t          j                  r |            D ]} ||           n |            } ||           t                                          t
          di           |S )Nc                    | sd S t          | t                    r#t                                          |            d S t          | t                    r*t                                          t
          | i           d S t          d          )NzuInvalid return or yield value. Either return/yield a single number or a dictionary object in your trainable function.)
isinstancedictr   reportr   r   rc   )outputs    r%   handle_outputzJwrap_function.<locals>.ImplicitFunc._trainable_func.<locals>.handle_output   s     F-- 	MM((00000// MM((.&)ABBBBB$.  r(   T)r   inspectisgeneratorfunctionr   r   r   )r$   r"   r^   r   r   r   s        r%   r!   z3wrap_function.<locals>.ImplicitFunc._trainable_func   s    V,,B   F*:66 & bdd * *F!M&))))* f%%%
 MM  "2D!9:::Mr(   r"   r   c                 f    t          t                    st                    r |          S S r   )r   r   callable)clsr"   r,   s     r%   default_resource_requestz<wrap_function.<locals>.ImplicitFunc.default_resource_request  s=     i)>?? )HYDWDW ) y(((r(   N)r   r   r   hasattrr   r   r!   classmethodr   r   r   r   r   r   )r*   r,   r   s   r%   ImplicitFuncr      s         
#*7:z#B#BNJ 		 	 		 	 	 	 	> 
	c3h	+,	 	 	 	 	 
	 	 	r(   r   )
r   r   r   r   getfullargspecargsr   rc   formatgetattr)r   r*   inherit_from	func_argsuse_config_singler   r,   s   ``    @r%   wrap_functionr      s     &'Lz<(( <!,|;&z227I-j99 
y))
 
 	
 
L$77I- - - - - - - - - - - - - -^ r(   r   )/r   loggingrt   r   	functoolsr   numbersr   typingr   r   r   r   r	   ray.air._internal.utilr
   r   ray.air.constantsr   &ray.train._internal.checkpoint_managerr   ray.train._internal.sessionr   r   r   r   r   #ray.tune.execution.placement_groupsr   ray.tune.resultr   r   r   ray.tune.trainable.trainabler   ray.tune.utilsr   ray.util.annotationsr   	getLoggerr   loggerNULL_MARKERTEMP_MARKERr   r   r   r]   r(   r%   <module>r      s(     				              6 6 6 6 6 6 6 6 6 6 6 6 6 6 ? ? ? ? ? ? ? ? 2 2 2 2 2 2 B B B B B B              F E E E E E O O O O O O O O O O 2 2 2 2 2 2 0 0 0 0 0 0 - - - - - -		8	$	$
  ` ` ` ` `	 ` ` `F <@C C#$C,4SMC	
C C C C C Cr(   