
    &`in                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
mZmZ d dlmZmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* d dl+m,Z, d d	l-m.Z. d d
l/m0Z0 d dl1m2Z2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZBmCZCmDZD d dlEmFZF d dlGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZR d dlSmTZT d dlUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z` d dlambZb  ejc        ed          ZeefZg edg d          ZheeMeIeeL         f         Zie G d d                      Zj G d d          Zk edd          Zl G d d           ZmdS )!    N)Counterdefaultdict
namedtuple)	dataclassfield)Enum)	AnyCallableDict	FrozenSetListOptionalSetTupleUnion$PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME)
AUTOSCALER_HEARTBEAT_TIMEOUT_S"AUTOSCALER_MAX_CONCURRENT_LAUNCHESAUTOSCALER_MAX_LAUNCH_BATCHAUTOSCALER_MAX_NUM_FAILURESAUTOSCALER_STATUS_LOGAUTOSCALER_UPDATE_INTERVAL_SDISABLE_LAUNCH_CONFIG_CHECK_KEYDISABLE_NODE_UPDATERS_KEYFOREGROUND_NODE_LAUNCH_KEYWORKER_LIVENESS_CHECK_KEY)EventSummarizer)legacy_log_info_string)LoadMetrics)LocalNodeProvider!record_local_head_state_if_needed)BaseNodeLauncherNodeLauncher)NodeAvailabilitySummaryNodeProviderAvailabilityTracker)NodeTracker)AutoscalerPrometheusMetrics)_get_node_provider)ResourceDemandSchedulerResourceDictget_bin_pack_residual$placement_groups_to_resource_demands)NodeUpdaterThread)ConcurrentCounter	NodeCountNodeIDNodeIPNodeTypeNodeTypeConfigDictformat_info_stringhash_launch_confhash_runtime_confvalidate_configwith_head_node_ip)NodeProvider)NODE_KIND_HEADNODE_KIND_UNMANAGEDNODE_KIND_WORKERSTATUS_UP_TO_DATESTATUS_UPDATE_FAILEDTAG_RAY_FILE_MOUNTS_CONTENTSTAG_RAY_LAUNCH_CONFIGTAG_RAY_NODE_KINDTAG_RAY_NODE_STATUSTAG_RAY_RUNTIME_CONFIGTAG_RAY_USER_NODE_TYPE)RpcErrorUpdateInstructionsnode_idsetup_commandsray_start_commandsdocker_configc                      e Zd ZU eeef         ed<   eeeef                  ed<   ee	e
eef                  ed<   eeef         ed<   ee	e
ef                  ed<    ed           Zeed<   d	Zeeee	e
ee         f         f                  ed
<    ed           Zeeef         ed<   d	Zeeeef                  ed<   dZeed<   d	S )AutoscalerSummaryactive_nodes
idle_nodespending_nodespending_launchesfailed_nodesc                       t          i           S N)r%        v/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/_private/autoscaler.py<lambda>zAutoscalerSummary.<lambda>i   s     7 ; ; rW   )default_factorynode_availability_summaryNnode_activitiesc                      i S rU   rV   rV   rW   rX   rY   zAutoscalerSummary.<lambda>m   s    b rW   pending_resourcesnode_type_mappingFlegacy)__name__
__module____qualname__r   r3   int__annotations__r   r   r   r2   
NodeStatusr   r[   r%   r\   strr^   r_   r`   boolrV   rW   rX   rN   rN   a   sF        x}%%%%hm,----fh
:;<<<<8S=))))uVX-.////9>;;: : :6    FJOXd3fd3i.?(@#@ABIII(-jj(I(I(ItCH~III 37xS#X/666FDrW   rN   c                   :    e Zd ZdZdefdZdee         ddfdZdS )NonTerminatedNodeszBClass to extract and organize information on non-terminated nodes.providerc                    t          j                     }|                    i           | _        g | _        d | _        | j        D ]Z}|                    |          t                   }|t          k    r| j                            |           H|t          k    r|| _        [t          j                     |z
  | _
        t                              dt          | j
        d           d           d S )NThe autoscaler took    z3 seconds to fetch the list of non-terminated nodes.)timenon_terminated_nodesall_node_ids
worker_idshead_id	node_tagsrB   r=   appendr;   non_terminated_nodes_timeloggerinforound)selfrk   
start_timenode	node_kinds        rX   __init__zNonTerminatedNodes.__init__y   s    Y[[
$99"== )+)-% 	$ 	$D **4001BCI,,,&&t,,,,n,,# *.z)A&B5)G#K#K B B B	
 	
 	
 	
 	
rW   terminating_nodesreturnNc                     fd}t          t          || j                            | _        t          t          || j                            | _        dS )zMRemove nodes we're in the process of terminating from internal
        state.c                     | vS rU   rV   )r|   r   s    rX   not_terminatingzDNonTerminatedNodes.remove_terminating_nodes.<locals>.not_terminating   s    000rW   N)listfilterrr   rq   )rz   r   r   s    ` rX   remove_terminating_nodesz+NonTerminatedNodes.remove_terminating_nodes   s\    	1 	1 	1 	1 	1 votGGHH 9J!K!KLLrW   )	ra   rb   rc   __doc__r:   r~   r   r1   r   rV   rW   rX   rj   rj   v   sc        LL
 
 
 
 
>M$v, M4 M M M M M MrW   rj   KeepOrTerminatezkeep terminate decide_laterc                      e Zd ZdZdeeeeedddf	de	e
eg ef         f         dedddee
         d	ed
edededededee         dee         fdZedee
         fd            Zd Zd ZdefdZdedee
         deddfdZd Zdee         fdZ de!e"ef         ddfd Z#d! Z$d" Z%d# Z&d$ee'         fd%Z(d&ee         d'e!e
ef         dee         fd(Z)d)ee         de*e         fd*Z+ded+e!e"ef         de,e-ee
         f         fd,Z.d- Z/d. Z0dCd/Z1d0 Z2d1 Z3dededefd2Z4defd3Z5d4 Z6d5 Z7de
de
fd6Z8de
d7e
defd8Z9d9 Z:d: Z;d; Z<d< Z=d=ed>e
ddfd?Z>d@ Z?dee@         fdAZAdB ZBdS )DStandardAutoscalera  The autoscaling control loop for a Ray cluster.

    There are two ways to start an autoscaling cluster: manually by running
    `ray start --head --autoscaling-config=/path/to/config.yaml` on a instance
    that has permission to launch other instances, or you can also use `ray up
    /path/to/config.yaml` from your laptop, which will configure the right
    AWS/Cloud roles automatically. See the Ray documentation
    (https://docs.ray.io/en/latest/) for a full definition of autoscaling behavior.
    StandardAutoscaler's `update` method is periodically called in
    `monitor.py`'s monitoring loop.

    StandardAutoscaler is also used to bootstrap clusters (by adding workers
    until the cluster size that can handle the resource demand is met).
    NFconfig_readerload_metrics
gcs_clientzray._raylet.GcsClientsession_namemax_launch_batchmax_concurrent_launchesmax_failuresprocess_runnerupdate_interval_sprefix_cluster_infoevent_summarizerprom_metricsc                 r   t          t                    rfd}|| _        n| _        t                      | _        |
| _        d| _        |pt          |          | _        d| _	        | 
                    d           || _        || _        || _        || _        || _        |pt!                      | _        i | _        t'          t(                    | _        t'          t(                    | _        d| _        d| _        |	| _        d| _        g | _        | j        d                             t<          d	          | _        t@          !                    t<           d
| j                    | j        d                             tD          d	          | _#        t@          !                    tD           d
| j#                    | j        d                             tH          d	          | _%        t@          !                    tH           d
| j%                    | j        d                             tL          d          | _'        t@          !                    tL           d
| j'                    d| _(        d| _)        tU                      | _+        | j%        r:tY          | j        | j+        | j        | j        || j-        | j                  | _(        nt]          j/                    | _)        ta          j1        |te          |          z            }tg          t)          |                    D ]X}ti          | j        | j)        || j+        | j        | j        || j-        | j        	  	        }d|_5        |6                                 Yto                      | _8        d | j        d         9                                D             | j        d<   || _:        | j        d         ;                                D ]#}tx          j=        >                    |          sJ $t@          !                    d?                    | j                             dS )a  Create a StandardAutoscaler.

        Args:
            config_reader: Path to a Ray Autoscaler YAML, or a function to read
                and return the latest config.
            load_metrics: Provides metrics for the Ray cluster.
            session_name: The current Ray session name when this autoscaler
                is deployed.
            max_launch_batch: Max number of nodes to launch in one request.
            max_concurrent_launches: Max number of nodes that can be
                concurrently launched. This value and `max_launch_batch`
                determine the number of batches that are used to launch nodes.
            max_failures: Number of failures that the autoscaler will tolerate
                before exiting.
            process_runner: Subproc-like interface used by the CommandRunner.
            update_interval_s: Seconds between running the autoscaling loop.
            prefix_cluster_info: Whether to add the cluster name to info strs.
            event_summarizer: Utility to consolidate duplicated messages.
            prom_metrics: Prometheus metrics for autoscaler-related operations.
            gcs_client: client for interactions with the GCS. Used to drain nodes
                before termination.
        c                      t                    5 } t          j        |                                           }d d d            n# 1 swxY w Y   |S rU   )openyaml	safe_loadread)f
new_configr   s     rX   read_fnz,StandardAutoscaler.__init__.<locals>.read_fn   s    -(( :A!%!9!9J: : : : : : : : : : : : : : :!!s   'AAAN)r   Terrors_fatalr   g        rk   F:)rk   pendingr   "node_provider_availability_trackerr   
node_typesr   )	rk   queueindexr   r   r   r   r   r   c                 T    i | ]%\  }}|t           j                            |          &S rV   )ospath
expanduser).0remotelocals      rX   
<dictcomp>z/StandardAutoscaler.__init__.<locals>.<dictcomp>`  s>     &
 &
 &
 BG&&u--&
 &
 &
rW   file_mountszStandardAutoscaler: {})@
isinstancerg   r   r&   r   r   rk   r(   r   resource_demand_schedulerresetr   r   r   r   r   r   r   updatersr   rd   num_failed_updatesnum_successful_updatesnum_failureslast_update_timer   rp   nodes_to_terminateconfiggetr   disable_node_updatersrw   rx   r   disable_launch_config_checkr   foreground_node_launchr   worker_liveness_checkforeground_node_launcherlaunch_queuer/   rR   r#   available_node_typesr   Queuemathceilfloatranger$   daemonstartr'   node_trackeritemsr   valuesr   r   existsformat)rz   r   r   r   r   r   r   r   r   r   r   r   r   r   max_batchesinode_launcher
local_paths    `                rX   r~   zStandardAutoscaler.__init__   sB   N mS)) 		/" " " " "
 ")D!.D2Q2S2S/#6   ) 
,G%-
 -
 -
 *.&


%%%(( 0'>$, 0 EO4E4E :<5@5E5E9DS9I9I# #!2 CG! 13
 &*[%<%@%@%u&
 &
" 	0OO43MOOPPP ,0;z+B+F+F+U,
 ,
( 	.SS1QSS	
 	
 	
 '+k*&=&A&A&'
 '
# 	1QQD4OQQRRR &*[%<%@%@%t&
 &
" 	0OO43MOOPPP EI%CG 1 3 3& 	&,<-!%!6373Z)4!.- - -D)) !&D)$;eDT>U>U$UVVK3{++,, & & ,!]+ 1%)%:7;7^!-#8!%!2
! 
! 
! (,$##%%%% (MM
&
 &
!%]!;!A!A!C!C&
 &
 &
M"
 %+m4;;== 	. 	.J7>>*------,33DK@@AAAAArW   r   c                 @    | j         d                                         S )Nr   )r   keys)rz   s    rX   all_node_typesz!StandardAutoscaler.all_node_typesk  s    {1277999rW   c                 t   	 |                      d           |                                  d S # t          $ r}| j        j                                         t                              d           | xj        dz  c_        | j        | j	        k    rt          
                    d           |Y d }~d S d }~ww xY w)NFr   z-StandardAutoscaler: Error during autoscaling.   z+StandardAutoscaler: Too many errors, abort.)r   _update	Exceptionr   update_loop_exceptionsincrw   	exceptionr   r   critical)rz   es     rX   updatezStandardAutoscaler.updateo  s    		JJEJ***LLNNNNN 	 	 	488:::LMMM" 4#444 MNNN 544444		s   *. 
B7A4B22B7c                 *     j         sJ  j        sJ t          j                    }| j        z
   j        k     rd S | _        t           j                    _         j                                         s%t          	                    d j         d           d S g  _
        t          r,t          	                                                                t            j        j                    j                                         s                     |            j        r j        r                     |           nX                                                                     j        r                     |                                             t1           j        j                  } j        j                            |            j                             fd j        j        D                         j                             j        j         j         !                                 j        "                                 j        #                                 j        $                                 j        %                                 j        &                                 j'        (                                          \  }} )                    |            j                                         s *                    |            j         +                                 t          j                     j        z
  }t          	                    dtY          |d           d            j        j-        .                    |           d S )	Nz4Backing off of autoscaler update. Will try again in z	 seconds.c                 D    g | ]}j                             |          S rV   rk   internal_ipr   rI   rz   s     rX   
<listcomp>z.StandardAutoscaler._update.<locals>.<listcomp>  s9        ))'22  rW   )
active_ips)ensure_min_cluster_sizer[   rm   rn   z* seconds to complete the update iteration.)/rk   r   ro   r   r   rj   rp   safe_to_scalerw   rx   r   r   info_stringr   rr   is_readonly-terminate_nodes_to_enforce_config_constraintsr   r   terminate_unhealthy_nodesprocess_completed_updatesupdate_nodes"attempt_to_recover_unhealthy_nodesset_prometheus_updater_datalenr   running_workerssetr   prune_active_ipsrq   get_nodes_to_launchrR   	breakdownget_resource_demand_vectorget_resource_utilizationget_pending_placement_groupsget_static_node_resources_by_ipget_resource_requestsr   summary_report_pending_infeasiblelaunch_required_nodespost_processry   update_timeobserve)rz   nownum_workers	to_launchunfulfilledr   s   `     rX   r   zStandardAutoscaler._update{  s   }}----ikk &&)???F # %7t}$E$E! }**,, 	KKH&*&<H H H   F #% ! 	,KK((**+++tT%>%IJJJ}((** 	3>>sCCC) 3 - 8223777..000!!### - A;;C@@@00222 $3>??)--k::: 	**   #8E   	+ 	
 	
 	
 "&!?!S!S%2!++--88::6688::<<==??$($5$K$K$M$M&*&M&U&U&W&W "T 	"
 	"
	; 	''444}((** 	2&&y111 	""$$$ ikkD$9995a#8#8 9 9 9	
 	
 	
 	%--k:::::rW   r  c                      j         sJ  j        sJ  j        j        }d j        d         z  }||z
  }                      j         j        |          }                     |          }t          t                    dt          ddf fd}g }|D ]r}	                     |	          \  }
}|
t          j        k    r"                     |	|t          j                   N|
t          j        k    s|	|v r!                     |	          r ||	            j                            |	          }||v ry||         |k     rm                     |	dt          j                   t)          j        t)          j        ||                             }t                              d| d	                                |	          s#                     |	d
t          j                   R ||	           |                    |	           tt1           j         j                  }|t1           j                  z
   j        d         z
  }|t1          |          k    r=t                              d| dt1          |           d           t1          |          }|dk    r1|| d         }|D ]#}	                     |	dt          j                   $                                  dS )a  Terminates nodes to enforce constraints defined by the autoscaling
        config.

        (1) Terminates nodes in excess of `max_workers`.
        (2) Terminates nodes idle for longer than `idle_timeout_minutes`.
        (3) Terminates outdated nodes,
                namely nodes whose configs don't match `node_config` for the
                relevant node type.

        Avoids terminating non-outdated nodes required by
        autoscaler.sdk.request_resources().
        <   idle_timeout_minutesrI   r   Nc                     j         sJ j                             |           }t          |v r|t                   }|xx         dz  cc<   d S d S )Nr   rk   rt   rE   )rI   tags	node_typenode_type_countsrz   s      rX   	keep_nodezSStandardAutoscaler.terminate_nodes_to_enforce_config_constraints.<locals>.keep_node  se    =  ==**733D%-- !78	 +++q0+++++ .-rW   idlezNode last used: .outdatedmax_workersz(StandardAutoscaler: trying to terminate z nodes, while only z6 are safe to terminate. Inconsistent config is likely.r   zmax workers)rp   rk   r   ray_nodes_last_used_time_by_ipr   _sort_based_on_last_usedrr   '_get_nodes_needed_for_request_resourcesr   rd   r1   _keep_worker_of_node_typer   	terminateschedule_node_terminationrw   rx   keeplaunch_config_okr   ro   asctime	localtimeru   r   r   warningterminate_scheduled_nodes)rz   r  	last_usedidle_timeout_slast_used_cutoffsorted_node_idsnodes_not_allowed_to_terminater  nodes_we_could_terminaterI   should_keep_or_terminatereasonnode_ipformatted_last_used_timer  num_extra_nodes_to_terminateextra_nodes_to_terminater  s   `                @rX   r   z@StandardAutoscaler.terminate_nodes_to_enforce_config_constraints  sm    ((((}}%D	dk*@AA/
 77%0)
 

 *.)U)U*
 *
&
 's++	1v 	1$ 	1 	1 	1 	1 	1 	1 	1 24 & 	9 	9G 04/M/M)0 0,$f (?+DDD..wLLL(O,@@@<<<''00 =	'"""m//88G)##	'(:=M(M(M..wLLL+/<N9W#566, ,( J/GJJJKKKK**733 9..w
FKPPPP	'"""(//8888 $3>??#d5666]9SS 	% (#.F*G*GGGNN2/2 2/002 2 2   ,//G+H+H(
 (!++'?--..($ 4 T T..wv{SSSS&&(((((rW   rI   
reason_optlogger_methodc           	      v   | j         sJ |t          d          |}| j                             |          } |d| d| d| d           | j                            d|                     |          z   d                    |          z   dt          j        	           | j        	                    |           d S )
Nzreason should be not None.z1StandardAutoscaler: Terminating the node with id z and ip z. ()zRemoving {} nodes of type z ({}).r   quantity	aggregate)
rk   r   r   r   add_get_node_typer   operatorr   ru   )rz   rI   r+  r,  r&  r'  s         rX   r  z,StandardAutoscaler.schedule_node_terminationB  s     }}8999 -++G44,3     	
 	
 	
 	!!(!!'**+oof%%& l 	" 	
 	
 	
 	&&w/////rW   c                 ~   | j         sJ | j        sJ | j        sdS |                     | j                   | j                             | j                   | j        D ]:}| j                            |           | j        j        	                                 ;| j        
                    | j                   g | _        dS )z@Terminate scheduled nodes and clean associated autoscaler state.N)rk   rp   r   drain_nodes_via_gcsterminate_nodesr   untrackr   stopped_nodesr   r   )rz   r|   s     rX   r  z,StandardAutoscaler.terminate_scheduled_nodes\  s     }}((((& 	F 	  !8999%%d&=>>>+ 	2 	2D%%d++++//1111 	!::4;RSSS"$rW   provider_node_ids_to_drainc                 b     j         sJ t                      }d}|D ]b}	  j                             |          }|                    |           3# t          $ r# t
                              d| d           d}Y _w xY w|r j        j        	                                 | j
        j                                        z  } fd|D             }|sdS t
                              dt          |           d           	 t           j                            |d	
                    }||z
  }	|	rK j        j        	                                 t
                              dt          |	           d           dS dS # t$          $ rd}
|
j        t(          j        j        k    rn> j        j        	                                 t
                              d           Y d}
~
dS Y d}
~
dS d}
~
wt          $ r<  j        j        	                                 t
                              d           Y dS w xY w)a  Send an RPC request to the GCS to drain (prepare for termination)
        the nodes with the given node provider ids.

        note: The current implementation of DrainNode on the GCS side is to
        de-register and gracefully shut down the Raylets. In the future,
        the behavior may change to better reflect the name "Drain."
        See https://github.com/ray-project/ray/pull/19350.
        Fz!Failed to get ip of node with id z during scale-down.Tc                 4    h | ]}j         j        |         S rV   )r   node_id_by_ip)r   iprz   s     rX   	<setcomp>z9StandardAutoscaler.drain_nodes_via_gcs.<locals>.<setcomp>  s1     
 
 
46D+B/
 
 
rW   Nz	Draining z raylet(s).   )timeoutzFailed to drain z-Failed to drain Ray nodes. Traceback follows.)rk   r   r   r2  r   rw   r   r   drain_node_exceptionsr   r   r=  r   rx   r   r   drain_nodeserrorrF   rpc_coderay_rayletGRPC_STATUS_CODE_UNIMPLEMENTED)rz   r:  node_ipsfailed_ip_fetchprovider_node_idr>  connected_node_ipsnode_ids_to_draindrained_node_idsfailed_to_drainr   s   `          rX   r6  z&StandardAutoscaler.drain_nodes_via_gcsr  s    }} 55 : 	' 	'']../?@@R     ' ' '  >(> > >   #''  	:377999 &(9(G(L(L(N(NN

 
 
 
:L
 
 
 ! 	FC$5 6 6CCCDDD	N  #++,=q+II    02BBO S!7;;===QO0D0DQQQRRRRRS S  	R 	R 	RzS[GGG !7;;===  !PQQQQQQQQQ	 
  	N 	N 	N 377999LMMMMMM	Ns2   /A*A=<A=?A9E< <
H.AG%%AH.-H.r  c                 p    |r1|                                 D ]\  }}|                     ||           d S d S )N)r  )r   launch_new_node)rz   r  r  counts       rX   r   z(StandardAutoscaler.launch_required_nodes  s_     	A$-OO$5$5 A A 	5$$Ui$@@@@	A 	AA ArW   c                     g } fd j         j        D             D ]\  }}}}|{                     |          }                     |          }t                              | d           |                    t          j         j	        ||||||f                     |D ]}|
                                 |D ]}|                                 dS )z\Run NodeUpdaterThreads to run setup commands, sync files,
        and/or start Ray.
        c              3   B   K   | ]}                     |          V  d S rU   )should_updater   s     rX   	<genexpr>z2StandardAutoscaler.update_nodes.<locals>.<genexpr>  sM       K
 K
 w''K
 K
 K
 K
 K
 K
rW   Nz: Starting new thread runner.)targetargs)rp   rr   _node_resources_node_labelsrw   debugru   	threadingThreadspawn_updaterr   join)	rz   TrI   rJ   rK   rL   	resourceslabelsts	   `        rX   r   zStandardAutoscaler.update_nodes  s-    K
 K
 K
 K
4?K
 K
 K
 	 	FG^%7 " 0099	**733FFFGGG$#1#*.%")
 
 
    	 	AGGIIII 	 	AFFHHHH	 	rW   c                 F   g }| j                                         D ].\  }}|                                s|                    |           /|rg }|D ]`}| j         |         }|j        dk    r| j        |xx         dz  cc<   | j        j                                         |j	        r| j        j
                                         |j        r$| j        j                            |j                   | j                            | j                            |                     n|                    |           | j        |xx         dz  cc<   | j        j                                         |j	        r| j        j                                         | j                            |           | j         |= b|rk|D ]P}|| j        j        v r"|                     |dt4          j                   2t4                              d| d           Q|                                  dS dS dS )z&Clean up completed NodeUpdaterThreads.r   r   zlaunch failedzStandardAutoscaler: z:: Failed to update node. Node has already been terminated.N)r   r   is_aliveru   exitcoder   r   successful_updatesr   for_recoverysuccessful_recoveriesr   worker_update_timer  r   mark_activerk   r   r   failed_updatesfailed_recoveriesr   r8  rp   rr   r  rw   rD  r  r  )rz   completed_nodesrI   updaterrS   s        rX   r   z,StandardAutoscaler.process_completed_updates  s    $ 3 3 5 5 	0 	0GW##%% 0&&w/// *	1L* + +-0#q((/888A=888%8<<>>>+ F)?CCEEE* )<DD#/  
 %11$-2K2KG2T2TUUUU ''000+G4449444%488:::+ B);??AAA%--g666M'** 1
  ,  G$";"FFF66#_fl    A7 A A A   
 ..00000U*	1 *	121 1rW   c                     | j         j                            t          | j                             d}| j                                        D ]}|j        r|dz  }| j         j                            |           dS )zwRecord total number of active NodeUpdaterThreads and how many of
        these are being run to recover nodes.
        r   r   N)r   updating_nodesr   r   r   r   rh  recovering_nodes)rz   num_recoveringro  s      rX   r   z.StandardAutoscaler.set_prometheus_updater_data  s     	(,,S-?-?@@@}++-- 	$ 	$G# $!#*..~>>>>>rW   r  c           
         | j         sJ g }|D ]M}t          d |D                       }|r| j                             |          s|                    |           N|rg|D ]f}| j                            d                    |          d                    t          |                                                    d           edS dS )aC  Emit event messages for infeasible or unschedulable tasks.

        This adds messages to the event summarizer for warning on infeasible
        or "cluster full" resource requests.

        Args:
            unfulfilled: List of resource demands that would be unfulfilled
                even after full scale-up.
        c              3   4   K   | ]}d |v p
|t           k    V  dS )_group_Nr   )r   ks     rX   rV  z@StandardAutoscaler._report_pending_infeasible.<locals>.<genexpr>6  sE       " " QK!'K"K" " " " " "rW   z~Error: No available node types can fulfill resource request {}. Add suitable node types to this cluster to resolve this issue.zinfeasible_{}   )key
interval_sN)	r   anyis_feasibleru   r   add_once_per_intervalr   sortedr   )rz   r  
infeasiblebundleplacement_grouprequests         rX   r   z-StandardAutoscaler._report_pending_infeasible(  s    ----
! 	* 	*F! " "" " "  O  1==fEE *!!&))) 	%  %;;**0&//'..vgmmoo/F/FGG! <    	 	 rW   nodesr  c                 v     t          j        |          ddt          f fd}t          ||d          S )zSort the nodes based on the last time they were used.

        The first item in the return list is the most recently used.
        rI   c                 f    j         sJ j                             |           }|vrS |         S rU   r   )rI   r'  last_used_copyleast_recently_usedrz   s     rX   last_time_usedzCStandardAutoscaler._sort_based_on_last_used.<locals>.last_time_usedT  sA    =  =m//88Gn,,**%g..rW   T)ry  reverse)copydeepcopyr1   r~  )rz   r  r  r  r  r  s   `   @@rX   r  z+StandardAutoscaler._sort_based_on_last_usedH  sd     y11 !	/F 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ e>>>>rW   r"  c                    | j         sJ t                      }t          | j                                                  \  }}|                    | j                                                   |s|st          |          S | j                                        }t          j
        | j        | j        d                  d                   }|s:| j                             | j        j                  }|                    |i           }|g}g }	|D ]}
| j                             |
          }t$          |v r|t$                   }t          j
        | j        |         d                   }|s0| j                             |
          }|                    |i           }|                    |           |	                    |
           t          j
        |          }|D ]}t)          ||d          \  }}|r|}t)          ||          \  }}|                    d           |                    d           t-          |	          D ]5\  }}
||         ||         k    r	||         r |                    |
           6t          |          S )aJ  Returns the nodes NOT allowed to terminate due to get_resource_requests() and get_pending_placement_groups().

        Args:
            sorted_node_ids: the node ids sorted based on last used (LRU last).

        Returns:
            FrozenSet[NodeID]: a set of nodes (node ids) that
            we should NOT terminate.
        head_node_typera  T)strict_spreadr   )rk   r   r-   r   r   extendr   	frozensetr   r  r  r   r   r   rp   rs   r   rt   rE   ru   r,   pop	enumerater2  )rz   r"  r#  resource_demandsstrict_spreadsstatic_node_resourceshead_node_resourceshead_node_ipnode_total_resources&resource_demand_vector_worker_node_idsrI   r  r  node_resourcesr'  node_remaining_resourcesr  r   updated_node_remaining_resources_r   s                        rX   r  z:StandardAutoscaler._get_nodes_needed_for_request_resources^  s    }}69ee&+O::<<,
 ,
(. 	 1 G G I IJJJ 	= 	=;<<< ==?? 	 -1M%dk2B&CD[Q-
 -

 # 	N=44T5N5VWWL"7";";L""M"M4G3H13.& 	G 	GG=**733D%-- !78	/3}-i8E0 0 & L"m77@@G%:%>%>w%K%KN$++N;;;6==gFFF $(=1E#F#F + 	H 	HM<Q(-t= = =9K9  'G$$&;$&6'
 '
## 	  ### $$Q'''#$JKK 
	< 
	<JAw(+/CA/FFF(+ G .227;;;;7888rW   r  c                 N   | j         sJ | j                             |          }t          |v r|t                   }| j                            |i                               dd          }| j                            |i                               dd          }|| j        vr7t          | j                                                  }t          j        d| fS ||         dz   }|t          ||          k    rt          j
        dfS ||k    rt          j        dfS t          j        dfS )a  Determines if a worker should be kept based on the min_workers
        and max_workers constraint of the worker's node_type.

        Returns KeepOrTerminate.keep when both of the following hold:
        (a) The worker's node_type is present among the keys of the current
            config's available_node_types dict.
        (b) Deleting the node would violate the min_workers constraint for that
            worker's node_type.

        Returns KeepOrTerminate.terminate when both the following hold:
        (a) The worker's node_type is not present among the keys of the current
            config's available_node_types dict.
        (b) Keeping the node would violate the max_workers constraint for that
            worker's node_type.

        Return KeepOrTerminate.decide_later otherwise.

        Args:
            node_type_counts(Dict[NodeType, int]): The non_terminated node
                types counted so far.
        Returns:
            KeepOrTerminate: keep if the node should be kept, terminate if the
            node should be terminated, decide_later if we are allowed
            to terminate it, but do not have to.
            Optional[str]: reason for termination. Not None on
            KeepOrTerminate.terminate, None otherwise.
        min_workersr   r  znot in available_node_types: r   Nmax_workers_per_type)rk   rt   rE   r   r   r   r   r   r  minr  decide_later)	rz   rI   r  r  r  r  r  r   	new_counts	            rX   r  z,StandardAutoscaler._keep_worker_of_node_type  s?   > }}}&&w//!T))34I377	2FFJJq K 377	2FFJJq K  999 (,D,E,J,J,L,L'M'M$#-J4HJJ  )3a7IC[9999&+T11;&&&02HHH+T11rW   c                     | j                             |                              t                    }| j        r/| j                            |i                               di           S i S )Nra  rk   rt   r   rE   r   rz   rI   r  s      rX   rY  z"StandardAutoscaler._node_resources  s`    M++G44889OPP	$ 	,00B??CCKQSTTTIrW   c                     | j                             |                              t                    }| j        r/| j                            |i                               di           S i S )Nrb  r  r  s      rX   rZ  zStandardAutoscaler._node_labels  s_    M++G44889OPP	$ 	,00B??CCHbQQQIrW   c                 V   d}t          | d          r| j                            dd          }	 |                                 }|t	          | dd           k    rb	 t          |           nQ# t          $ rD}| j        j        	                                 t                              d|           Y d }~nd }~ww xY wt                              d| dt          |                      t          |d         |d	         |d
         |d         g|          \  }}|| _        || _        || _        | j        s+t#          | j        d         | j        d                   | _        t%          | j        t&                    rt)          | j                   | j        d         | _        | j                            d          }| j                            d          dk    }| j                            d          }	|rt-          |          }nT|rd}t                              d           n5|	r1dt1          |	d          z  dz
  }t                              d           nd}| j        r@| j                            | j        | j        | j        d         | j        d         |           d S t7          | j        | j        | j        d         | j        d         |          | _        d S # t          $ rG}| j        j        	                                 |r|t                              d           Y d }~d S d }~ww xY w)NFr   file_mounts_sync_continuouslyzCluster config validation failed. The version of the ray CLI you launched this cluster with may be higher than the version of ray being run on the cluster. Some new features may not be available until you upgrade ray on your cluster.)exc_infozNew config after validation: z, of type: r   cluster_synced_filesworker_setup_commandsworker_start_ray_commands)"generate_file_mounts_contents_hashrk   cluster_namer   upscaling_speedautoscaling_mode
aggressivetarget_utilization_fractioni z^Legacy aggressive autoscaling mode detected. Replacing it by setting upscaling_speed to 99999.r   gMbP?zLegacy target_utilization_fraction config detected. Replacing it by setting upscaling_speed to 1 / target_utilization_fraction - 1.g      ?r  r  z)StandardAutoscaler: Error parsing config.)hasattrr   r   r   getattrr8   r   r   config_validation_exceptionsr   rw   r[  typer7   runtime_hashfile_mounts_contents_hashrk   r)   r   r!   r"   r   r   r  maxr   reset_configr*   reset_exceptionsr   )
rz   r   sync_continuouslyr   r   new_runtime_hashnew_file_mounts_contents_hashr  r  r  s
             rX   r   zStandardAutoscaler.reset   s   !4"" 	X $0OQV W WZ	N++--JWT8T::::#J////  	 	 	%BFFHHHLLK
 "# !        	 LL0
 0 0!*--0 0   AR=)1267:; 4EA A A=< %DK 0D-JD*=  2K
+T[-H! ! $-):;; A1$-@@@(,4J(KD%"koo.?@@O);<<LJ*.+//:W*X*X' &"'"8"8  &"'   
 - &"#c*Eu&M&M"MPQ"Q=    #&-  .;;M-K.K 01#     2IM-K.K 01#2 2...  	N 	N 	N.22444 N  !LMMMMMMMMM	NsH   )K A) (K )
B73:B2-K 2B77G$K 8K 
L(!<L##L(c                    | j         rdS | j                            |          }|                    t                    }|                    t
                    }|| j        vrdS t          j        | j	                            di                     }|r,|
                    | j	        d         |         d                    t          || j	        d                   }||k    rdS dS )NTFworker_nodesr   node_configauth)r   rk   rt   r   rA   rE   r   r  r  r   r   r6   )rz   rI   rt   tag_launch_confr  launch_configcalculated_launch_hashs          rX   r  z#StandardAutoscaler.launch_config_ok`  s    + 	4M++G44	#--(=>>MM"899	D555 5 dkoonb&I&IJJ 	  23I>}M   "2-VAT!U!U!_445trW   c           	      V   | j                             |          }|                    t                    }|                    t                    }|| j        k    s| j        H| j        |k    r=t                              d	                    |||| j        | j                             dS dS )Nz>StandardAutoscaler: {}: Runtime state is ({},{}), want ({},{})FT)
rk   rt   r   rD   r@   r  r  rw   rx   r   )rz   rI   rt   applied_config_hash!applied_file_mounts_contents_hashs        rX   files_up_to_datez#StandardAutoscaler.files_up_to_datex  s    M++G44	'mm,BCC,5MM:V,W,W)$"333*6.2SSSKK==CV'5%2> >	 	 	 5trW   c                     | j         sJ | j                             |          }|| j        j        v r$| j        j        |         }||z
  }|t          k     rdS dS )zDetermine whether we've received a heartbeat from a node within the
        last AUTOSCALER_HEARTBEAT_TIMEOUT_S seconds.
        TF)rk   r   r   last_heartbeat_time_by_ipr   )rz   rI   r  ry  last_heartbeat_timedeltas         rX   heartbeat_on_timez$StandardAutoscaler.heartbeat_on_time  si    
 }}m''00$#==="&"3"Mc"R--E555turW   c                    | j         sJ | j        sJ | j        j        D ]}| j                             |          t                   }|t
          k    s3| j                             |          }|| j        j        vr| j        	                    |           | 
                    ||          r|                     |dt          j                   |                                  dS )z}Terminated nodes for which we haven't received a heartbeat on time.
        These nodes are subsequently terminated.
        zlost contact with rayletN)rk   rp   rr   rt   rC   r>   r   r   r  rk  r  r  rw   r  r  )rz   r  rI   node_statusr>  s        rX   r   z,StandardAutoscaler.terminate_unhealthy_nodes  s    
 }}((((0; 	 	G-11'::;NOK "333 **733B*DDD!--b111%%gs33 **3V^    	&&(((((rW   c                 P    | j         j        D ]}|                     ||           d S rU   )rp   rr   recover_if_needed)rz   r  rI   s      rX   r   z5StandardAutoscaler.attempt_to_recover_unhealthy_nodes  s:    0; 	1 	1G""7C0000	1 	1rW   c           	      P   |                      |          sd S |                     ||          rd S t                              d                    |                     | j                            d|                     |          z   dz   dt          j                   | j	        
                    | j        j                  }t          di d|d| j        d         d| j	        d	| j        d
         d| j        d         di dg dg dt          | j        d         |          d| j        d| j        d| j        ddddd|                     |          d|                     |          d|                     |          dd}|                                 || j        |<   d S )NzIStandardAutoscaler: {}: No recent heartbeat, restarting Ray to recover...zRestarting {} nodes of type z (lost contact with raylet).r   r/  rI   provider_configrk   auth_configr  r  r   initialization_commandsrJ   rK   r  r  r  r   use_internal_ipTis_head_nodeFrL   r  node_labelsrh  rV   )
can_updater  rw   r  r   r   r2  r3  r4  rk   r   rp   rs   r.   r   r9   r  r  r    _get_node_specific_docker_configrY  rZ  r   r   )rz   rI   r  r  ro  s        rX   r  z$StandardAutoscaler.recover_if_needed  s>   w'' 	F!!'3// 	F++16'??	
 	
 	

 	!!*!!'**+,- l 	" 	
 	
 	
 }001J1RSS# 
 
 
G
 K
33
 ]]
 F++	

 ^44
 
 %'B
 2
  178,     
 **
 '+&D&D
  ..
 !D
  !
" ??HHH#
$  //888%
& ))'222'
( )
, 	!(grW   c                 x    | j         sJ | j                             |          }t          |v r|t                   S dS )Nunknown_node_typer
  )rz   rI   rt   s      rX   r3  z!StandardAutoscaler._get_node_type  sA    }}M++G44	!Y..344&&rW   
fields_keyc                     | j         sJ | j        |         }| j                             |          }t          |v rB|t                   }|| j        vrt          d| d          | j        |         }||v r||         }|S )NzUnknown node type tag: r  )rk   r   rt   rE   r   
ValueError)rz   rI   r  fieldsrt   r  node_specific_configs          rX   _get_node_type_specific_fieldsz1StandardAutoscaler._get_node_type_specific_fields  s    }}Z(M++G44	!Y..!"89I 999 !G9!G!G!GHHH#'#<Y#G 111-j9rW   c                     d| j         vri S t          j        | j                             di                     }|                     |d          }|                    |           |S )Ndocker)r   r  r  r   r  r   )rz   rI   rL   node_specific_dockers       rX   r  z3StandardAutoscaler._get_node_specific_docker_config  sd    4;&&Idkooh&C&CDD#BB7HUU1222rW   c                    |                      |          st          d d d d           S | j                            |                              t
                    }|t          k    r'|                     |          rt          d d d d           S | j                            |d          dk    }|r+| j	                            dd          rg }| j	        d         }nY|r4| j	                            dd          r| 
                    |d          }g }n#| 
                    |d          }| j	        d         }|                     |          }t          ||||          S )Nr   restart_onlyFr  
no_restartr  rH   )r  rG   rk   rt   r   rC   r>   r  r   r   r  r  )rz   rI   statussuccessful_updatedrJ   rK   rL   s          rX   rU  z StandardAutoscaler.should_update
  s~   w'' 	>%dD$===((11556IJJ&&&4+@+@+I+I&%dD$===!8<<WaHH1L 	J$+//.%"H"H 	JN!%-H!I 		JDKOOL%$H$H 		J!@@0 N "$!@@0 N "&-H!I==gFF!)1'	
 
 
 	
rW   c                 Z   t                               d| d           | j                            |          }|                     |          }| j                            |||           | j                            | j        j                  }	t          di d|d| j
        d         d| j        d| j
        d         d| j
        d         d	| j
        d	         d
t          |                     |d
          |	          dt          ||	          dt          ||	          d| j        d| j        ddd| j
        d         d| j
                            d          | j
                            d          dd| j        ddd|d|d|}
|
                                 |
| j        |<   d S )Nz5Creating new (spawn_updater) updater thread for node r  rI   r  rk   r  r  r  r   r  rJ   rK   r  r  r  Fr  rsync_optionsrsync_excludersync_filter)r  r  r   r  TrL   r  r  rV   )rw   rx   rk   r   r3  r   trackrp   rs   r.   r   r9   r  r  r  r   r   r   r   )rz   rI   rJ   rK   r  r  rL   r>  r  r  ro  s              rX   r^  z StandardAutoscaler.spawn_updater)  s;    	RRRR	
 	
 	
 ]&&w//''00	Y777}001J1RSS# 
 
 
G
 K
33
 ]]
 F++	

 ^44
 M22
 %633G=VWW% % %
 -^\JJJ
  11C\RRR
 **
 '+&D&D
 
  "&-C!D!D!
$ "&!A!A $ ? ?  #
*  ..+
, !D-
. (-/
0 *>1
2 $3
6 	!(grW   c                     | j         rdS || j        v rdS |                     |          sdS | j                            |d          dk    rdS t
                              | d           dS )NFr   z@ is not being updated and passes config check (can_update=True).T)r   r   r  r   r   rw   r[  )rz   rI   s     rX   r  zStandardAutoscaler.can_updateW  s    % 	5dm##5$$W-- 	5"&&w22Q665 5 5 5	
 	
 	
 trW   rR  r  c                    t                               d                    |                     | j                            ||           t          j        | j                  }| j        r'| j	        J | j	        
                    |||           d S | j        J |dk    rB| j                            |t          || j                  |f           || j        z  }|dk    @d S d S )Nz1StandardAutoscaler: Queue {} new nodes for launchr   )rw   rx   r   rR   r   r  r  r   r   r   launch_noder   putr  r   )rz   rR  r  r   s       rX   rQ  z"StandardAutoscaler.launch_new_nodef  s    GNNuUUVVV!!)U333t{++& 	/0<<<)55feYOOOOO$000!))!%%S(=>>	J   .. !))))))rW   c                    t                               d           |                                 }|rW| j                            |           |D ]:}| j                            |           | j        j        	                                 ;t                               d
                    t          |                               d S )Nz*StandardAutoscaler: kill_workers triggeredz)StandardAutoscaler: terminated {} node(s))rw   rD  workersrk   r7  r   r8  r   r9  r   r   r   )rz   r  r|   s      rX   kill_workerszStandardAutoscaler.kill_workersx  s    ABBB 	6M))%000 6 6!))$///!/335555@GGE

SSTTTTTrW   c                    | j         sJ | j        sdS t                      }g }g }t                      }i }t	          j                    }| j        j        D ]}| j                             |          }| j                             |          t          fdt          t          t          fD                       set                   t          k    r|t                   }	|	||<   |                     ||          }
|
r&||	xx         dz  cc<   |                    |           ̉t                   }t          t           g}||v}|r.|                    |||	|f           |                    |           | j                            |          }i }| j                                                                        D ]\  }	}|r|||	<   i }| j                            d |D             |i           d         D ]8}|                                D ]!\  }}||                    |d          z   ||<   "9t5          t7          |          dd |D             ||| j                                        ||d		  	        S )
a  Summarizes the active, pending, and failed node launches.

        An active node is a node whose raylet is actively reporting heartbeats.
        A pending node is non-active node whose node tag is uninitialized,
        waiting for ssh, syncing files, or setting up.
        If a node is not pending or active, it is failed.

        Returns:
            AutoscalerSummary: The summary.
        Nc              3       K   | ]}|v V  	d S rU   rV   )r   tagrt   s     rX   rV  z-StandardAutoscaler.summary.<locals>.<genexpr>  s<         y      rW   r   c                     g | ]	\  }}}}|
S rV   rV   )r   rI   r  s      rX   r   z.StandardAutoscaler.summary.<locals>.<listcomp>  s!    AAA/w1a7AAArW   )r  rQ   unused_resources_by_ipr   c                 "    g | ]\  }}}}|||fS rV   rV   )r   r  r>  r  r  s        rX   r   z.StandardAutoscaler.summary.<locals>.<listcomp>  s4       ,DAr9fY'  rW   T)	rO   rP   rQ   rR   rS   r[   r^   r_   r`   )rk   rp   r   r   ro   rq   r   rt   allrB   rE   rC   r<   r  r2  r>   r?   ru   r   get_all_failed_node_inforR   r   r   r   calculate_node_resourcesr   rN   dictr   r   )rz   rO   rQ   rS   
non_failedr_   r  rI   r>  r  	is_activer  completed_states
is_pendingrR   rR  r^   r  ry  valuert   s                       @rX   r   zStandardAutoscaler.summary  s    }}( 	4,3IIUU
ikk0=  	,  	,G**733B//88I     &*'     
 *+/BBB!"89I$-b!..w<<I 	,Y'''1,'''w''''"#67$57K#L #+;;
 ,!(('2y&)IJJJNN7+++(AA*MM  $ 5 ? ? A A G G I I 	4 	4Iu 4.3 +"<UUAA=AAA* $& V 
 
  		O 		ON -2244 O O
U).1B1F1FsA1N1N)N!#&&O !l++ HU   .%&*&M&U&U&W&W//
 
 
 	
rW   c                     | j                                         }|                                 }|sJ dt          ||          z   S )N
)r   r   r5   )rz   
lm_summaryautoscaler_summarys      rX   r   zStandardAutoscaler.info_string  sG    &..00
!\\^^!!!!(5GHHHHrW   )F)Cra   rb   rc   r   r   r   r   
subprocessr   r   rg   r
   r  r    r   rd   r	   rh   r   r(   r~   propertyr   r   r   r   r   r   r1   r  r  r   r6  r   r3   r   r   r   r   r+   r   r  r   r  r   r   r  rY  rZ  r   r  r  r  r   r   r  r3  r  r  rU  r^  r  rQ  r  rN   r   r   rV   rW   rX   r   r      s        * '+ ;'I7(!=$)6:>BmB mB S(2t8"445mB "	mB
 ,mB smmB mB "%mB mB mB mB "mB #?3mB :;mB mB mB mB^ :C : : : X:
 
 
X; X; X;tk) k) k) k) k)Z00+3C=0IQ0	0 0 0 04% % %,ONd6l ON ON ON ONbAtHcM/B At A A A A
! ! !F01 01 01d	? 	? 	?d<6H    @?&\?.23:.>?	f? ? ? ?,W9#F|W9	6	W9 W9 W9 W9r929215hm1D92	-	.92 92 92 92v    ^N ^N ^N ^N@  0  * e      )U ) ) ) )81 1 1*) *) *)X'c 'c ' ' ' 'c s s      
 
 
>,) ,) ,)\  /S /S /T / / / /$U U U[
"34 [
 [
 [
 [
zI I I I IrW   r   )nr  loggingr   r4  r   r   r  r\  ro   collectionsr   r   r   dataclassesr   r   enumr   typingr	   r
   r   r   r   r   r   r   r   r   rF  ray._common.utilsr   !ray.autoscaler._private.constantsr   r   r   r   r   r   r   r   r   r   (ray.autoscaler._private.event_summarizerr   *ray.autoscaler._private.legacy_info_stringr   $ray.autoscaler._private.load_metricsr    +ray.autoscaler._private.local.node_providerr!   r"   %ray.autoscaler._private.node_launcherr#   r$   :ray.autoscaler._private.node_provider_availability_trackerr%   r&   $ray.autoscaler._private.node_trackerr'   $ray.autoscaler._private.prom_metricsr(   !ray.autoscaler._private.providersr)   1ray.autoscaler._private.resource_demand_schedulerr*   r+   r,   r-   ray.autoscaler._private.updaterr.   ray.autoscaler._private.utilr/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   ray.autoscaler.node_providerr:   ray.autoscaler.tagsr;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   ray.exceptionsrF   	getLoggerra   rw   rg   rf   rG   NodeLaunchDatarN   rj   r   r   rV   rW   rX   <module>r'     s       				           8 8 8 8 8 8 8 8 8 8 ( ( ( ( ( ( ( (       T T T T T T T T T T T T T T T T T T T T T T  



 B B B B B B                        E D D D D D M M M M M M < < < < < <        Q P P P P P P P        = < < < < < L L L L L L @ @ @ @ @ @            > = = = = =                          6 5 5 5 5 5                          $ # # # # #		8	$	$ 
  ZHHH  
 )9hx6HHI        (*M *M *M *M *M *M *M *Mf $(*GHHwI wI wI wI wI wI wI wI wI wIrW   