
    &`ir#                         d dl Z d dlZd dlZd dlmZ d dlmZmZmZ d dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ  ej        e          Zd ZdZ G d	 d
e          ZdS )    N)RLock)AnyDictOptional)build_address)NodeLaunchException)NodeProvider)NODE_KIND_HEADNODE_KIND_WORKERSTATUS_SETTING_UPSTATUS_UP_TO_DATETAG_RAY_NODE_KINDTAG_RAY_NODE_NAMETAG_RAY_NODE_STATUSTAG_RAY_USER_NODE_TYPEzray.head.defaultc                       e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	ed
ee         fdZd Zd Zd Zdeeef         deeef         ded
eeeef                  fdZd Zd Zd Zd Zed             ZdS )SparkNodeProviderzCA node provider that implements provider for nodes of Ray on spark.c           
         t          j        | ||           t                      | _        t	          t
                    dt          t          t          t          t          t
          t          t          iii| _        d| _        | j        d         | _        | j        d         }dt#          | j        |           | _        | j        d         | _        | j        d         | _        d S )Ntagsr   ray_head_ipspark_job_server_portzhttp://ray_head_portcluster_unique_id)r	   __init__r   lockstrHEAD_NODE_IDr   r
   r   HEAD_NODE_TYPEr   r   r   _nodes_next_node_idprovider_configr   r   spark_job_server_urlr   
cluster_id)selfr!   cluster_namer   s       /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/spark/node_provider.pyr   zSparkNodeProvider.__init__    s    dO\BBBGG	 %~*N%|'):	 	
 /> !% 45L MNmD$46KLLNN 	! "1/B./BC    c                 r    | j         5  | xj        dz  c_        | j        cd d d            S # 1 swxY w Y   d S )N   )r   r    )r$   s    r&   get_next_node_idz"SparkNodeProvider.get_next_node_id<   s    Y 	& 	&!#%	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&s   ,00c                    | j         5  g }g }| j        D ]}|t          t                    k    rd}n|                     |          }|dk    re| j        |         d         t
                   t          k    r>t          | j        |         d         t
          <   t          	                    d| d           |dk    r|
                    |           |                     |          }d}|                                D ] \  }}	|                    |          |	k    rd}!|r|
                    |           |D ]}
| j                            |
           |cd d d            S # 1 swxY w Y   d S )Nrunningr   zSpark node provider node z starts running.
terminatedTF)r   r   r   r   _query_node_statusr   r   r   loggerinfoappend	node_tagsitemsgetpop)r$   tag_filtersnodes
died_nodesnode_idstatusr   okkvdied_node_ids              r&   non_terminated_nodesz&SparkNodeProvider.non_terminated_nodesA   s   Y $	 $	EJ; . .c,////&FF!44W==FY&&G,V45HI,- -
 . G,V4/ QQQQ   \))%%g....>>'22DB + 1 1 3 3 ' '188A;;!++!&B .W--- * . .----I$	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	 $	s   EE  E$'E$c                     |                      |          }t          j        | j        dz   d|i          }|                                 |j                            d          }t          j        |          }|d         S )Nz/query_task_statusspark_job_group_idurljsonzutf-8r:   )	_gen_spark_job_group_idrequestspostr"   raise_for_statuscontentdecoderD   loads)r$   r9   rA   responsedecoded_respjson_ress         r&   r.   z$SparkNodeProvider._query_node_statush   s    !99'BB=),@@&(:;
 
 
 	!!###'..w77:l++!!r'   c                     | j         5  || j        v o&| j        |         d         t                   t          k    	 cd d d            S # 1 swxY w Y   d S Nr   )r   r   r   r   r$   r9   s     r&   
is_runningzSparkNodeProvider.is_runningu   s    Y 	 	4;& %K(01DE$%	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   1AA
A
c                 V    | j         5  || j        vcd d d            S # 1 swxY w Y   d S Nr   r   rQ   s     r&   is_terminatedzSparkNodeProvider.is_terminated}   sx    Y 	. 	.$+-	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.s   	""c                 j    | j         5  | j        |         d         cd d d            S # 1 swxY w Y   d S rP   rU   rQ   s     r&   r2   zSparkNodeProvider.node_tags   s}    Y 	0 	0;w'/	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0s   (,,r9   returnc                     |S rT    rQ   s     r&   _get_ipzSparkNodeProvider._get_ip   s    r'   c                 ,    |                      |          S rT   r[   rQ   s     r&   external_ipzSparkNodeProvider.external_ip       ||G$$$r'   c                 ,    |                      |          S rT   r]   rQ   s     r&   internal_ipzSparkNodeProvider.internal_ip   r_   r'   c                 h    || j         v sJ | j         |         d                             |           d S rP   )r   update)r$   r9   r   s      r&   set_node_tagszSparkNodeProvider.set_node_tags   s;    $+%%%%GV$++D11111r'   node_configr   countc                      t          d          )Nz!This method should not be called.)AssertionError)r$   re   r   rf   s       r&   create_nodezSparkNodeProvider.create_node   s     @AAAr'   c                 ,    d| j          d| j         d| S )Nzray-cluster--z-worker-node-)r   r#   rQ   s     r&   rE   z)SparkNodeProvider._gen_spark_job_group_id   s6    &4- & & & &#& &	
r'   c                 Z    t          |          D ]}|                     ||||           d S rT   )range&_create_node_with_resources_and_labels)r$   re   r   rf   	resourceslabels_s          r&   %create_node_with_resources_and_labelsz7SparkNodeProvider.create_node_with_resources_and_labels   sI     u 	 	A77T9f   	 	r'   c                 <   ddl m} | j        5  |                                }|t                   }t          |                                           }t          |          |d<   | j                                        }|	                    d          }	|	                    d          }
|	                    d          }|	                    d          } ||d         |          |d<   t          j        | j        d	z   |                     |          d
| dt          | j        | j                   |d         | j        | j        |d         |	|
|||d         |d         |d         d          }	 |                                 n4# t$          $ r' t'          dd| dt)          j                              w xY wdt,          t.          t          |t0          |t2          t4          ii| j        |<   t8                              d| d           d d d            d S # 1 swxY w Y   d S )Nr   )_append_resources_configNODE_ID_AS_RESOURCECPUGPUmemoryobject_store_memoryworker_node_optionsz/create_nodezGThis job group is for spark job which runs the Ray cluster worker node z connecting to ray head node using_stage_schedulingray_temp_dircollect_log_to_path)rA   spark_job_group_descr{   r   r   r|   num_cpus_per_nodenum_gpus_per_nodeheap_memory_per_nodeobject_store_memory_per_noderz   r}   r9   rB   zNode creation failurezStarting ray worker node z failedr   z!Spark node provider creates node .)ray.util.spark.cluster_initrt   r   copyr   r   r*   intr!   r5   rF   rG   r"   rE   r   r   r   rH   	Exceptionr   sysexc_infor   r   r   r   r   r   r/   r0   )r$   re   r   ro   rp   rt   	node_typer9   confr   r   r   r   rL   s                 r&   rn   z8SparkNodeProvider._create_node_with_resources_and_labels   s    	IHHHHHY >	H >	H!((I34I $//1122G/27||I+,',,..D )e 4 4 )e 4 4#,==#:#: +4==9N+O+O(*B*B*+Y+ +D&'  }->*.*F*Fw*O*O[/6[ [%243CTEW%X%X[ [ /33K.L#'#3%)%7$($8):):,@4P+/0E+F+/0E+F()>?#   H.
 ))++++   )+@@@@LNN   %'7*I%w'):	$DK  KKFGFFFGGG}>	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	H >	Hs+   EH+F ?H 1F11AHHHc                 d   || j         v rGt          j        | j        dz   d|                     |          i          }|                                 | j        5  || j         v r| j                             |           d d d            n# 1 swxY w Y   t          	                    d|            d S )Nz/terminate_noderA   rB   z$Spark node provider terminates node )
r   rF   rG   r"   rE   rH   r   r5   r/   r0   )r$   r9   rL   s      r&   terminate_nodez SparkNodeProvider.terminate_node   s    dk!!}-0AA*D,H,H,Q,QR  H %%'''Y 	) 	)$+%%(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	D7DDEEEEEs   $BBBc                     | S rT   rZ   )cluster_configs    r&   bootstrap_configz"SparkNodeProvider.bootstrap_config   s    r'   N)__name__
__module____qualname____doc__r   r*   r?   r.   rR   rV   r2   r   r   r[   r^   ra   rd   r   r   r   ri   rE   rr   rn   r   staticmethodr   rZ   r'   r&   r   r      s       MMD D D8& & &
% % %N" " "  . . .0 0 0s x}    % % %% % %2 2 2BS>B15c3hBHKB	$sCx.	!B B B B

 
 
  CH CH CHJF F F   \  r'   r   )rD   loggingr   	threadingr   typingr   r   r   rF   ray._common.network_utilsr   $ray.autoscaler.node_launch_exceptionr   ray.autoscaler.node_providerr	   ray.autoscaler.tagsr
   r   r   r   r   r   r   r   	getLoggerr   r/   r   r   r   rZ   r'   r&   <module>r      sO     



       & & & & & & & & & &  3 3 3 3 3 3 D D D D D D 5 5 5 5 5 5	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 
	8	$	$#] ] ] ] ] ] ] ] ] ]r'   