
    &`ioJ                     L   d dl Z d dlmZmZmZmZmZ d dlmZm	Z	 d dl
Z
d dlmc mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ erd dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'  e j(        e)          Z*e*+                    e j,                    G d de          Z-dS )    N)TYPE_CHECKINGDictListOptionalTuple)RequestResponse)gcs_service_pb2_grpc)JobInfoStorageClient)find_jobs_by_job_ids)SubprocessModule)SubprocessRouteTable)get_http_session_to_module)DeveloperAPI)
JobDetails)DecoratedTrainRunDecoratedTrainRunAttemptDecoratedTrainWorker	RunStatusTrainRunTrainRunAttemptTrainWorkerc                       e Zd Z fdZ ej        d           ej                    ede	de
fd                                    Zded         ded         fd	Zd
ee         deedf         fdZded         ded         fdZded         ded         fdZdddedee         f         fdZ ej        d           ej                    ede	de
fd                                    Zdee         fdZd Z fdZd Zd Z xZS ) 	TrainHeadc                      t                      j        |i | d | _        d | _        d | _        d | _        d | _        d S N)super__init___train_stats_actor_train_v2_state_actor_job_info_client_gcs_actor_info_stub_node_head_http_session)selfargskwargs	__class__s      z/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/dashboard/modules/train/train_head.pyr   zTrainHead.__init__!   sO    $)&)))"&%)" $$(! (,$$$    z/api/train/v2/runs/v1reqreturnc                 J  K   	 ddl m} n;# t          $ r. t                              d           t          dd          cY S w xY w|                                  d{V }|t          dd          S 	 |j                                         d{V }| 	                    |
                                           d{V } ||          }n5# t          j        j        $ r}t          d	d
|           cY d}~S d}~ww xY wt          |                                d          S )z#Get all TrainRuns for Ray Train V2.r   TrainRunsResponse`Train is not installed. Please run `pip install ray[train]` when setting up Ray on your cluster.  statustextNTrain state data is not available. Please make sure Ray Train is running and that the Train state actor is enabled by setting the RAY_TRAIN_ENABLE_STATE_TRACKING environment variable to "1".
train_runs  ^Failed to get a response from the train stats actor. The GCS may be down, please retry later: application/jsonr3   content_type)#ray.train.v2._internal.state.schemar.   ImportErrorlogger	exceptionr	   get_train_v2_state_actorget_train_runsremote_decorate_train_runsvaluesray
exceptionsRayTaskErrorjson)r$   r*   r.   state_actorr6   decorated_train_runsdetailses           r(   get_train_v2_runszTrainHead.get_train_v2_runs.   s     	MMMMMMM 		 		 		7   7     		 !99;;;;;;;;W   #.#=#D#D#F#FFFFFFF
-1-F-F%%''. . ( ( ( ( ( ($ ,+7KLLL>. 
 
 
  HDEH H        	
 +
 
 
 	
s.    5AA4AC C?!C:4C?:C?r6   r   r   c           	      P  K   ddl m} g }|                                  d{V }|j                                         d{V }|                     d |D                        d{V }|D ]}||j                                                 }|                     |           d{V }	||j	                 }
| 
                    |           d{V \  }} |j        i |                                |	|
||d          }|                    |           t          |d d          }|S )	zDecorate the train runs with run attempts, job details, status, and status details.

        Returns:
            List[DecoratedTrainRun]: The decorated train runs in reverse chronological order.
        r   )r   Nc                     g | ]	}|j         
S  job_id).0	train_runs     r(   
<listcomp>z2TrainHead._decorate_train_runs.<locals>.<listcomp>t   s    $R$R$R)Y%5$R$R$Rr)   )attemptsjob_detailsr2   status_detailc                     | j         S r   )start_time_nsruns    r(   <lambda>z0TrainHead._decorate_train_runs.<locals>.<lambda>   s	    C- r)   Tkeyreverse)r<   r   r@   get_train_run_attemptsrB   	_get_jobsidrD   _decorate_train_run_attemptsrR   _get_run_status	parse_objdictappendsorted)r$   r6   r   rJ   rI   all_train_run_attemptsjobsrT   train_run_attemptsdecorated_train_run_attemptsrW   r2   status_detailsdecorated_train_runs                 r(   rC   zTrainHead._decorate_train_runsd   s      	JIIIII8: 99;;;;;;;;'2'I'P'P'R'R!R!R!R!R!R!R^^$R$Rz$R$R$RSSSSSSSS# 	= 	=I "8	!E!L!L!N!N 778JKKKKKKKK ) y/0K+/+?+?	+J+J%J%J%J%J%J%J"FN"="3"=nn&& <#.$%3  # # !''(;<<<<  & -- 
  
  

 $#r)   job_idsr   c                 H   K   t          | j        | j        |           d {V S r   )r   
gcs_clientr!   )r$   rp   s     r(   rb   zTrainHead._get_jobs   sD      )O!
 
 
 
 
 
 
 
 	
r)   rl   r   r   c                    K   ddl m} g }|D ]_}|                     |j                   d {V } |j        i |                                d|i          }|                    |           `|S )Nr   )r   workers)r<   r   _decorate_train_workersrt   rf   rg   rh   )r$   rl   r   rm   train_run_attemptdecorated_train_workersdecorated_train_run_attempts          r(   rd   z&TrainHead._decorate_train_run_attempts   s       	QPPPPPGI$!3 
	M 
	M
 223D3LMMMMMMMM $ +M*B*LP$))++PY8OPP+ +' )//0KLLLL++r)   train_workersr   r   c                 
  
K   ddl m} g }d |D             }t                              d| d           |                     |           d {V }|D ]
|                    
j        d           }|rY
fd|d         D             }
fd|D             } |j        i 
                                |d	         |d
         |d          }	n" |j        
                                          }	|	                    |	           |S )Nr   )r   c                     g | ]	}|j         
S rP   actor_idrS   workers     r(   rU   z5TrainHead._decorate_train_workers.<locals>.<listcomp>   s    AAAV_AAAr)   +Getting all actor info from GCS (actor_ids=)c                 F    g | ]}j         d  |d         D             v |S )c                     g | ]
}|d          S pidrP   rS   processs     r(   rU   z@TrainHead._decorate_train_workers.<locals>.<listcomp>.<listcomp>   s    KKK7KKKr)   processesPidsr   rS   gputrain_workers     r(   rU   z5TrainHead._decorate_train_workers.<locals>.<listcomp>   sT       #'KKc/6JKKKL L L L Lr)   gpusc                 P    g | ]"}i |d fd|d         D             d         i#S )processInfoc                 6    g | ]}|d          j         k    |S r   r   )rS   r   r   s     r(   rU   z@TrainHead._decorate_train_workers.<locals>.<listcomp>.<listcomp>   s5     ( ( ( '&u~1AAA $AAAr)   r   r   rP   r   s     r(   rU   z5TrainHead._decorate_train_workers.<locals>.<listcomp>   su     
" 
" 
" % ( ( ( (+.+?( ( ( 	( 
" 
" 
"r)   stateprocessStatsr2   r   r   )
r<   r   r>   info_get_actor_infosgetr}   rf   rg   rh   )r$   ry   r   rw   	actor_idstrain_run_actorsactorr   formatted_gpusdecorated_train_workerr   s             @r(   ru   z!TrainHead._decorate_train_workers   s      	MLLLLL>@AA=AAA	N)NNNOOO!%!6!6y!A!AAAAAAA) (	C (	CL$(()>EEE #   $V}  
" 
" 
" 
"  $
" 
" 
" *H)=)G&++--"'.(-n(= .	  * *&& *H)=)G %%''* *& $**+ABBBB&&r)   rT   r   c                   K   ddl m}m} |                     |j        g           d {V }||j                 }|r|                    d          nd }||j        k    r|j        |j        k    r|j	        }d}||fS |j        |j
        fS )Nr   )ActorStatusr   r   6Terminated due to system errors or killed by the user.)r<   r   r   r   controller_actor_idr   DEADr2   RUNNINGABORTEDrX   )	r$   rT   r   r   actor_infoscontroller_actor_infocontroller_actor_status
run_statusrX   s	            r(   re   zTrainHead._get_run_status   s       	ONNNNNNN !1193P2QRRRRRRRR +I,I J 3HQ!%%g...T 	  ${'777 I$555"*JTM..  )"9::r)   z/api/train/v2/runsc                   K   	 ddl m} n;# t          $ r. t                              d           t          dd          cY S w xY w|                                  d{V }|t          dd          S 	 |j                                         d{V }| 	                    |           d{V }t          |d d	
          }t          | j        | j        d |D                        d{V }|D ]!}|                    |j                  |_        " ||          }n5# t"          j        j        $ r}	t          dd|	           cY d}	~	S d}	~	ww xY wt          |                                d          S )z'Get all TrainRunInfos for Ray Train V1.r   r-   r/   r0   r1   Nr4   c                     | j         S r   )start_time_msr[   s    r(   r]   z*TrainHead.get_train_runs.<locals>.<lambda>1  s	    C$5 r)   Tr^   c                     g | ]	}|j         
S rP   rQ   )rS   r\   s     r(   rU   z,TrainHead.get_train_runs.<locals>.<listcomp>7  s    CCCCSZCCCr)   r5   r7   r8   r9   r:   ) ray.train._internal.state.schemar.   r=   r>   r?   r	   get_train_stats_actorget_all_train_runsrB   '_add_actor_status_and_update_run_statusri   r   rr   r!   r   rR   rW   rE   rF   rG   rH   )
r$   r*   r.   stats_actorr6   train_runs_with_detailsrW   r\   rK   rL   s
             r(   rA   zTrainHead.get_train_runs  se     
	JJJJJJJ 		 		 		7   7     		 !6688888888W   #.#A#H#H#J#JJJJJJJ
FFzRRRRRRRR ( +1+55 + + +'
 %9O)CC+BCCC% %      
 3 B BC&1oocj&A&ACOO++7NOOO>. 
 
 
  HDEH H        	
 +
 
 
 	
s.    5AA4B(D E1E
E
Er   c                   K   | j         *t          d| j        j        | j        j                  | _         d                    |          }d| d}| j                             |          4 d {V }|                                 |                                 d {V }d d d           d {V  n# 1 d {V swxY w Y   |d         d         S )NNodeHead,z$http://localhost/logical/actors?ids=z
&nocache=1dataactors)	r#   r   _config
socket_dirsession_namejoinr   raise_for_statusrH   )r$   r   actor_ids_qs_strurlresp	resp_jsons         r(   r   zTrainHead._get_actor_infosM  s]     '/+EDL3T\5N, ,D( 88I..Q5EQQQ/33C88 	* 	* 	* 	* 	* 	* 	*D!!###"iikk))))))I	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*  **s   //B00
B:=B:c           	        K   ddl m}m}m}m} g }|                                D ]}g }d |j        D             }	t                              d|	 d           | 	                    |	           d {V }
|j        D ]|

                    j        d           }|r^fd|d         D             }fd|D             }|                    i                                 |d	         |d
         |d          }n'|                                                              }|                    |           |                    i |                                d|i          }|

                    |j                  }|r|
                    d	          nd }||j        k    r#|j        |j        k    r|j        |_        d|_        |                    |           |S )Nr   )ActorStatusEnumRunStatusEnumTrainRunInfoWithDetailsTrainWorkerInfoWithDetailsc                     g | ]	}|j         
S rP   r|   r~   s     r(   rU   zETrainHead._add_actor_status_and_update_run_status.<locals>.<listcomp>f  s    IIIVIIIr)   r   r   c                 F    g | ]}j         d  |d         D             v |S )c                     g | ]
}|d          S r   rP   r   s     r(   rU   zPTrainHead._add_actor_status_and_update_run_status.<locals>.<listcomp>.<listcomp>t  s    OOOwGENOOOr)   r   r   rS   r   worker_infos     r(   rU   zETrainHead._add_actor_status_and_update_run_status.<locals>.<listcomp>p  sS       &?OO#o:NOOOP P P P Pr)   r   c                 P    g | ]"}i |d fd|d         D             d         i#S )r   c                 6    g | ]}|d          j         k    |S r   r   )rS   r   r   s     r(   rU   zPTrainHead._add_actor_status_and_update_run_status.<locals>.<listcomp>.<listcomp>~  s4     , , ,$+#*5>[_#D#D !(#D#D#Dr)   r   r   rP   r   s     r(   rU   zETrainHead._add_actor_status_and_update_run_status.<locals>.<listcomp>{  su     
& 
& 
&  !) , , , ,/2?/C, , ,  	,! 
& 
& 
&r)   r   r   r   rt   r   )r   r   r   r   r   rD   rt   r>   r   r   r   r}   rf   rg   rh   r   r   r   r   r   rX   )r$   r6   r   r   r   r   r   rT   worker_infos_with_detailsr   r   r   r   r   worker_info_with_detailstrain_run_with_detailsr   r   s                    @r(   r   z1TrainHead._add_actor_status_and_update_run_statusY  s     	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 BD#**,, G	C G	CIJL%IIy7HIIIIKKRiRRRSSS%)%:%:9%E%EEEEEEE(0 (K (K(,,[-A4HH #   #(=  D
& 
& 
& 
& $(
& 
& 
&N 0J/S/S)..00&+Gn,1.,A$2	  0 0,, 0J/S/S#((**0 0, *001IJJJJ%<%F%FJ9>>##JY0IJJ& &" %(()FGGE<A&Keii&8&8&8t#'?+???(M,AAA4A4I&1L '4 $**+ABBBB&&r)   c                    K   t                                                       d {V  | j        st          | j                  | _        | j        }t          j        |          | _        d S r   )	r   r\   r!   r   rr   aiogrpc_gcs_channelr
   ActorInfoGcsServiceStubr"   )r$   gcs_channelr'   s     r(   r\   zTrainHead.run  so      ggkkmm$ 	J$8$I$ID!.$8$P%
 %
!!!r)   c                    K   	 ddl m} | j         |            | _        | j        S # t          $ r t                              d           Y nw xY wdS )zS
        Gets the train stats actor and caches it as an instance variable.
        r   get_state_actorNr/   )%ray.train._internal.state.state_actorr   r   r=   r>   r?   r$   r   s     r(   r   zTrainHead.get_train_stats_actor  s      	MMMMMM&.*9/*;*;'** 	 	 	7    	
 t   "' $AAc                    K   	 ddl m} | j         |            | _        | j        S # t          $ r t                              d           Y nw xY wdS )zS
        Gets the Train state actor and caches it as an instance variable.
        r   r   Nr/   )(ray.train.v2._internal.state.state_actorr   r    r=   r>   r?   r   s     r(   r@   z"TrainHead.get_train_v2_state_actor  s      	PPPPPP)1-<_->->*-- 	 	 	7    	
 tr   )__name__
__module____qualname__r   routesr   dashboard_optional_utilsinit_ray_and_catch_exceptionsr   r   r	   rM   r   rC   strr   rb   rd   ru   r   r   re   rA   r   r   r\   r   r@   __classcell__)r'   s   @r(   r   r       sP       , , , , , VZ'((;;==1
7 1
x 1
 1
 1
 \ >= )(1
f0$z*0$	!	"0$ 0$ 0$ 0$d
tCy 
T#|:K5L 
 
 
 
,"&'8"9,	(	), , , ,*6'!-06'	$	%6' 6' 6' 6'p;#;	{HSM)	*; ; ; ;B VZ$%%;;===
 =
H =
 =
 =
 \ >= &%=
~
+S	 
+ 
+ 
+ 
+S' S' S'j
 
 
 
 
  $      r)   r   ).loggingtypingr   r   r   r   r   aiohttp.webr   r	   rE   ray.dashboard.optional_utils	dashboardoptional_utilsr   ray.core.generatedr
    ray.dashboard.modules.job.commonr   ray.dashboard.modules.job.utilsr   !ray.dashboard.subprocesses.moduler   !ray.dashboard.subprocesses.routesr   r    ray.dashboard.subprocesses.utilsr   ray.util.annotationsr   )ray.dashboard.modules.job.pydantic_modelsr   r<   r   r   r   r   r   r   r   	getLoggerr   r>   setLevelINFOr   rP   r)   r(   <module>r      s    = = = = = = = = = = = = = = ) ) ) ) ) ) ) ) 



 ? ? ? ? ? ? ? ? ? 3 3 3 3 3 3 A A A A A A @ @ @ @ @ @ > > > > > > L L L L L L G G G G G G - - - - - - 
DDDDDD                  
	8	$	$    z z z z z  z z z z zr)   