
    &`i                         d dl Z d dlZd dlmZ d dlmZmZmZ d dlZd dl	m
Z
 d dlmZmZmZmZmZ d dlmZ d dlmZ  e j        e          Z G d d	          ZdS )
    N)defaultdict)AnyDictList)Dataset)ActorStatusEnumRunStatusEnumTrainDatasetInfoTrainRunInfoTrainWorkerInfo)check_for_failure)WorkerGroupc                       e Zd ZdZddZ	 ddededed	ed
edeeef         dede	de
eee	f                  deddfdZded	ededefdZdedeeef         ddfdZdS )TrainRunStateManagerzA class that aggregates and reports train run info to TrainStateActor.

    This manager class is created on the train controller layer for each run.
    returnNc                 F    || _         t          t                    | _        d S N)state_actorr   dicttrain_run_info_dict)selfr   s     {/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/train/_internal/state/state_manager.py__init__zTrainRunStateManager.__init__   s     &#.t#4#4        run_idjob_idrun_name
run_statuscontroller_actor_iddatasetsworker_groupstart_time_ms	resourcesstatus_detailc                   	 | j         st                              d           dS 	fdfdt          t	                              D             }t          |          \  }}|st                              d|            dS t          j        |          }t          |d           }d |
                                D             }t          |||||||||
		
  
        }i | j        |<   |                     ||           dS )
z0Collect Train Run Info and report to StateActor.zDUnable to register train run since `TrainStateActor` is not started.Nc                     t           j                                        } t           j                                        }t          |                                 |                                 |                                 |	                                |
                                t           j                                        t          j                    t          j                    d         t           j        
  
        S )Nr   )

world_rank
local_rank	node_rankactor_idnode_idnode_ipgpu_idspidr$   status)raytrainget_contextruntime_contextget_runtime_contextr   get_world_rankget_local_rankget_node_rankget_actor_idget_node_idutilget_node_ip_addressget_gpu_idsosgetpidr   ALIVE)train_contextcore_contextr$   s     r   collect_train_worker_infozJTrainRunStateManager.register_train_run.<locals>.collect_train_worker_info4   s    I1133M.BBDDL"(7799(7799'5577%2244$00224466))IKK#A,&,   r   c                 <    g | ]}                     |          S  )execute_single_async).0indexrC   r"   s     r   
<listcomp>z;TrainRunStateManager.register_train_run.<locals>.<listcomp>D   s:     
 
 
 --e5NOO
 
 
r   z>Failed to collect run information from the Ray Train workers:
c                     | j         S r   )r(   )infos    r   <lambda>z9TrainRunStateManager.register_train_run.<locals>.<lambda>R   s    T_ r   )keyc                 `    g | ]+\  }}t          ||j        j        |j        j                   ,S ))namedataset_namedataset_uuid)r
   _plan_dataset_name_dataset_uuid)rG   ds_namedss      r   rI   z;TrainRunStateManager.register_train_run.<locals>.<listcomp>T   sP     
 
 
  X3X3  
 
 
r   )
idr   rO   r    workersr!   r#   r   r%   r$   )r   loggerwarningrangelenr   errorr1   getsorteditemsr   r   _update_train_run_info)r   r   r   r   r   r    r!   r"   r#   r$   r%   futuressuccess	exceptionworker_info_listdataset_info_listupdatesrC   s          ` `       @r   register_train_runz'TrainRunStateManager.register_train_run   s     	NNV   F	 	 	 	 	 
 
 
 
 
s<0011
 
 
 /w77 	LL)&) )   F77++!"28T8TUUU
 
  (~~//
 
 
  3$&'!'
 
 
 ,. (##FG44444r   end_time_msc                 V    t          |||          }|                     ||           dS )z:Update the train run status when the training is finished.)r   r%   ri   N)r   ra   )r   r   r   r%   ri   rg   s         r   end_train_runz"TrainRunStateManager.end_train_runn   s?     !'#
 
 

 	##FG44444r   rg   c                     || j         v rj| j         |                             |           t          di | j         |         }t          j        | j        j                            |                     dS dS )z=Update specific fields of a registered TrainRunInfo instance.NrE   )r   updater   r1   r^   r   rh   remote)r   r   rg   train_run_infos       r   ra   z+TrainRunStateManager._update_train_run_info}   sy    T---$V,33G<<<)MMD,DV,LMMNGD$7>>~NNOOOOO .-r   )r   N)r   )__name__
__module____qualname____doc__r   strr   r   r   floatr   rh   r	   intrk   r   ra   rE   r   r   r   r      sX        
5 5 5 5  M5 M5M5 M5 	M5
 M5 !M5 sG|$M5 "M5 M5 S%Z()M5 M5 
M5 M5 M5 M5^55 "5 	5
 5 5 5 5PS P4S> Pd P P P P P Pr   r   )loggingr>   collectionsr   typingr   r   r   r1   ray.datar    ray.train._internal.state.schemar   r	   r
   r   r   ray.train._internal.utilsr    ray.train._internal.worker_groupr   	getLoggerrp   rY   r   rE   r   r   <module>r      s&    				 # # # # # # " " " " " " " " " " 



                    8 7 7 7 7 7 8 8 8 8 8 8		8	$	$mP mP mP mP mP mP mP mP mP mPr   