
    &`iM                    0   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
mZ d dlmZ d dlmZmZmZmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZ d dlmZ d dlm Z  d dl!m"Z"m#Z# d dl$m%Z%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2  ej3        e4          Z5e
 G d d                      Z6e
 G d d                      Z7 G d de          Z8 G d de          Z9 G d de          Z:e
 G d d                      Z; G d de8          Z<dS )     N)ABCabstractmethod)defaultdict)	dataclassfield)Enum)DictListOptionalTuple)message_to_dict)AUTOSCALER_CONSERVE_GPU_NODES)UtilizationScore_fits_inplace_subtract)AutoscalerEventLogger)InstanceUtil)NodeTypeConfig)AutoscalerInstanceNodeType)ProtobufUtilResourceRequestUtil)ClusterResourceConstraintGangResourceRequestResourceRequestResourceRequestByCount)LabelSelectorOperator)InstanceLaunchRequestNodeKindTerminationRequestc                   j   e Zd ZU eed<    ee          Zee	e
f         ed<   dZee         ed<   dZee         ed<    ee          Zee         ed<    ee          Zee         ed<    ee          Zee         ed	<    ee          Zee         ed
<    ee          Zee	ef         ed<   dS )SchedulingRequestdisable_launch_config_checkdefault_factorynode_type_configsNmax_num_nodesidle_timeout_sresource_requestsgang_resource_requestscluster_resource_constraintscurrent_instancescloud_resource_availabilities)__name__
__module____qualname__bool__annotations__r   dictr'   r	   r   r   r(   r   intr)   floatlistr*   r
   r   r+   r   r,   r   r-   r   r.        o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/autoscaler/v2/scheduler.pyr#   r#   /   s?         "&%%%8=d8S8S8StHn45SSS#'M8C='''&*NHUO*** 7<eD6Q6Q6Qt23QQQ8=d8S8S8SD!45SSSDIEE E E $'@"A    38%2M2M2Mt./MMM <A5QU;V;V;V!4%#8VVVVVr9   r#   c                       e Zd ZU  ee          Zee         ed<    ee          Z	ee
         ed<    ee          Zee         ed<    ee          Zee         ed<    ee          Zee         ed<   dS )SchedulingReplyr%   	to_launchto_terminateinfeasible_resource_requests!infeasible_gang_resource_requests'infeasible_cluster_resource_constraintsN)r/   r0   r1   r   r7   r=   r
   r   r3   r>   r!   r?   r   r@   r   rA   r   r8   r9   r:   r<   r<   I   s          &+U4%@%@%@ItM"@@@-2U4-H-H-HL$)*HHH:?%PT:U:U:U $"7UUUCH5D D D%t,?'@    PUuP P P+T2K-L     r9   r<   c                   2    e Zd ZdZededefd            ZdS )IResourceSchedulerz~
    Interface for a resource scheduler.

    Implements the `instance_manager.proto ResourceSchedulerService` interface.
    requestreturnc                     dS )z
        Given the resource requests and the current cluster state, calculate the
        target cluster shape by trying to schedule the resource requests on the
        nodes.
        Nr8   )selfrD   s     r:   schedulezIResourceScheduler.scheduleb   s	     	r9   N)r/   r0   r1   __doc__r   r#   r<   rH   r8   r9   r:   rC   rC   [   sM           1 o    ^  r9   rC   c                       e Zd ZdZdZdZdZdS )SchedulingNodeStatusz<
    The status of a scheduling node (`SchedulingNode`)
    	TO_LAUNCHSCHEDULABLETO_TERMINATEN)r/   r0   r1   rI   rL   rM   rN   r8   r9   r:   rK   rK   l   s+         
 I  K!LLLr9   rK   c                       e Zd ZdZdZdZdS )ResourceRequestSourcez-
    The source of the resource request.
    PENDING_DEMANDCLUSTER_RESOURCE_CONSTRAINTN)r/   r0   r1   rI   rQ   rR   r8   r9   r:   rP   rP   {   s*          &N #@r9   rP   c                   
   e Zd ZU dZeed<   eed<    ed           Ze	e
ee         f         ed<    ee          Ze	e
e	eef         f         ed<    ee          Ze	eef         ed<    ee          Ze	eef         ed	<   d
Zee         ed<   d
Zee         ed<   d
Zee         ed<   d
Zeej        j                 ed<   d
Zee         ed<   dZeed<   d
Zee         ed<   e j!        Z"e ed<   dd
ddde j!        d
fdede	eef         de	eef         d	e	eef         dededeej        j                 dededede dee         fdZ#de
fdZ$de
fdZ%dede
fdZ&e'de(de	ee)f         de*ded          fd             Z+e'de(de*fd!            Z,e'	 	 d.d"e)dede dee         dee         dd fd#            Z-d$ Z.d%ee         de
de/ee         e0f         fd&Z1de
de0fd'Z2dee         defd(Z3dede
de*fd)Z4d*ed+efd,Z5defd-Z6d
S )/SchedulingNodea  
    A abstraction of a node that can be scheduled on by the resource scheduler.

    A scheduling node is expected to be used as:

        node  = SchedulingNode.new(instance, node_configs)
        remaining, score = node.try_schedule(requests)

        .... do something with the score ....

    NOTE:
        One could also extend the scheduling behavior by overriding `try_schedule`
    	node_typestatusc                  *    t          t                    S N)r   r7   r8   r9   r:   <lambda>zSchedulingNode.<lambda>   s    D 1 1 r9   r%   sched_requestsavailable_resources_for_schedtotal_resourceslabelsNlaunch_reasontermination_requestim_instance_idim_instance_statusray_node_idr   idle_duration_mslaunch_config_hash	node_kind available_resourcesc                 P   || _         || _        t          j        t	          |          t          j        t	          |          i| _        t          j        g t          j        g i| _        || _        || _	        || _
        || _        || _        |	| _        |
| _        || _        || _        d S rX   )rU   r\   rP   rQ   r4   rR   r[   rZ   r]   rV   r`   ra   rb   rc   rd   re   r_   )rG   rU   r\   rg   r]   rV   r`   ra   rb   rc   rd   re   r_   s                r:   __init__zSchedulingNode.__init__   s     #.!0$7J2K2K!=tO?T?T.
*
 "0"!=r
 ,"4& 0"4"#6   r9   resource_request_sourcec                     | j         |         S )zBGet the available resources for the given resource request source.)r[   rG   rj   s     r:   get_available_resourcesz&SchedulingNode.get_available_resources   s    12IJJr9   c                     | j         |         S )z@Get the resource requests for the given resource request source.)rZ   rl   s     r:   get_sched_requestsz!SchedulingNode.get_sched_requests   s    "#:;;r9   rD   c                 F    | j         |                             |           dS )z
        Add the resource requests to the node.

        Args:
            request: The resource request to be added.
            resource_request_source: The source of the resource request.
        N)rZ   append)rG   rD   rj   s      r:   add_sched_requestz SchedulingNode.add_sched_request   s'     	34;;GDDDDDr9   instancer'   r$   rE   c                 v   t                               |           sdS | j        j        t          j        k    r| j        J d|              t          | j        j        t          | j        j	                  t          | j        j
                  i | j        j        pi | j        j        pi t          j        | j        j        | j        j        | j        j        | j        j        | j        j        | j        j                  S |                    | j        j        d          }||r*t*                              d| j        j         d           dS t          | j        j        i i i t          j        | j        j        | j        j        t1          t3          t5          j                              | j        j        | j        j        t0          j        j        | j        j                  t<          j        	  	        S t                                |t          j        | j        j        | j        j        | j        j                  S )	a  
        Create a new scheduling node from an autoscaler instance.

        It creates:
            - None if the instance is not schedulable by IM.
            - A schedulable node if the instance is running ray or pending to run ray,
              so it should be considered in the scheduling process.

        Args:
            instance: The instance.
            node_type_configs: The node type configs.
            disable_launch_config_check: If outdated node check through launch config is
                disabled.

        NzGray node should not be None when the instance is running ray: instance=)rU   r\   rg   r]   rV   r`   ra   rb   rc   rd   re   zNode config for z is missing, but we are not terminating the outdated node because `disable_launch_config_check` is True in the autoscaler's provider config.)idinstance_idinstance_statuscauseinstance_type)	rU   r\   rg   r]   rV   r`   ra   r_   re   )re   r`   ra   )!rT   is_schedulableim_instancerV   r   RAY_RUNNINGray_nodery   r4   r\   rg   r]   dynamic_labelsrK   rM   rv   node_idrc   rd   re   getloggerinforN   r!   struuiduuid4CauseOUTDATEDr    WORKERfrom_node_config)rs   r'   r$   node_configs       r:   newzSchedulingNode.new   sI   * ,,X66 	4&(*>>>$00I>FI I 100
 "".< $X%6%F G G %)):)N$O$O(/52
  (7=2 ,7'3?#+#7#>$08!)!2!C#+#7#J".8'   0 (++H,@,NPTUU* 8x';'I 8 8 8   t "".< "$&+8'3?#+#7#>$64:<<(( ( 4 @$,$8$?,2;"*"6"D% % % #/   $ .. ,*4#/;'3: / 
 
 	
r9   c                 X    | j         dS t          j        | j         j                  rdS dS )z
        Check if the instance is schedulable by IM.

        Args:
            instance: The instance.

        Returns:
            True if the instance is schedulable by IM.
        NFT)r{   r   is_ray_running_reachablerV   )rs   s    r:   rz   zSchedulingNode.is_schedulableV  s9     ' 5 01E1LMM 	4ur9   r   c           
          t          | j        t          | j                  t          | j                  t          | j                  ||||          S )aw  
        Create a scheduling node from a node config.

        Args:
            node_config: The node config.
            status: The status of the node.
            node_kind: The node kind.
            im_instance_id: The instance id of the im instance.
            im_instance_status: The instance status of the im instance.
            node_kind: The node kind.
        )rU   r\   rg   r]   rV   r`   ra   re   )rT   namer4   	resourcesr]   )r   rV   re   r`   ra   s        r:   r   zSchedulingNode.from_node_configu  sW    & !& !677 $[%: ; ;*++)1	
 	
 	
 		
r9   c                 (    | j         s
J d            d S )Nznode_type should be set)rU   rG   s    r:   __post_init__zSchedulingNode.__post_init__  s    ~88888~88r9   requestsc                     g }|D ]-}|                      ||          s|                    |           .|                     |          }||fS )a  
        Try to schedule the resource requests on this node.

        This modifies the node's available resources if the requests are schedulable.
        The requests are scheduled one by one in the sorted order, and no
        backtracking is done.

        Args:
            requests: The resource requests to be scheduled.
            resource_request_source: The source of the resource request, i.e.
                pending demands from ray actors/tasks or cluster resource constraints.

        Returns:
            A tuple of:
                - list of remaining requests that cannot be scheduled on this node.
                - the utilization score for this node with respect to the current
                resource requests being scheduled.
        )_try_schedule_onerq   _compute_score)rG   r   rj   unschedulable_requestsrscores         r:   try_schedulezSchedulingNode.try_schedule  sh    0 "$  	1 	1A))!-DEE 1&--a000##$;<<%u,,r9   c           	      V   |                      |          }|                     |          }d}t                      }|D ]<}|j                                        D ] \  }}|dk    r|                    |           !=|D ]}	|	| j        v r|dz  }g }
| j                                        D ]`\  }}|dk    r||v rP||                    |d          z
  |z  }|dk    r|dk    sJ d|             |
                    ||dz  z             ad}t          r>| j                            dd          dk    }t          d |D                       }|r|sd}|                     |          }||||
rt          |
          nd|
r,t          t          |
                    t          |
          z  ndfS )	a4  
        Compute the utilization score for this node with respect to the current resource
        request being scheduled.

        A "higher" score means that this node is more suitable for scheduling the
        current scheduled resource requests.

        The score is a tuple of 5 values:
            1. Whether this node has labels matching the current resource request's
                label_selector requirements:
                    0: if this node does not satisfy any label selector requirements or
                       no label selectors are provided.
                    len(label_selectors)-i: a score based on the priority of the label
                        selector in the resource request that this node satisfies.
            2. Whether this node is a GPU node and the current resource request has
                GPU requirements:
                    0: if this node is a GPU node and the current resource request
                    placed onto the node has no GPU requirements.
                    1: if this node is not a GPU node or the current resource request
                    placed onto the node has GPU requirements.
            3. The number of resource types being scheduled.
            4. The minimum utilization rate across all resource types.
            5. The average utilization rate across all resource types.

        NOTE:
            This function is adapted from  _resource_based_utilization_scorer from
            autoscaler v1.

        TODO(rickyx,jjyao):  We should also consider node labels for
            scoring. For example, if a node has a label that matches the affinity
            label of the resource request, we should give it a higher score.

        TODO(rickyx): add pluggable scoring functions here.

        Returns:
            A utilization score for this node.
        r      zInvalid utilization:    TGPUc              3   (   K   | ]}d |j         v V  dS )r   N)resources_bundle.0r   s     r:   	<genexpr>z0SchedulingNode._compute_score.<locals>.<genexpr>  s*      "W"W15A,>#>"W"W"W"W"W"Wr9   F)ro   rm   setr   itemsaddr\   r   rq   r   any_satisfies_label_constraintsminr6   sumlen)rG   rj   rZ   rg   num_matching_resource_typessched_resource_typesreqresource_namevsched_resource_typeutil_by_resourceskutilgpu_okis_gpu_nodeany_gpu_requestsmatches_labelss                    r:   r   zSchedulingNode._compute_score  s4   R 001HII"::;RSS '(#"uu! 	< 	<C$'$8$>$>$@$@ < < qq55(,,];;;< $8 	1 	1"d&:::+q0+ (..00 	8 	8DAqAvv'''/33Aq999Q>qyyTQYYY0N0N0NYY.!((dAg777 ( 	 .225!<<q@K""W"W"W"W"WWW #3  ::>JJ '&7>C!"""Q E#'(())C0A,B,BBB
 	
r9   c                 z   |D ]}t          |j                  }t          |j                  D ]\  }}d}|j        D ]r}|j        }t          |j                  }	|j        }
| j        	                    |          }|
t          j        k    r	||	vrd} nW|
t          j        k    r	||	v rd} npd} |r	||z
  c c S dS )zReturns a higher value based on the priority of the label selector this node
        satisfies (first returns highest score, decreasing sequentially for fallback), 0 otherwise.TFr   )r   label_selectors	enumeratelabel_constraints	label_keyr   label_valuesoperatorr]   r   r   LABEL_OPERATOR_INLABEL_OPERATOR_NOT_IN)rG   rZ   r   num_selectorsiselectorall_constraints_pass
constraintkeyvaluesopnode_vals               r:   r   z+SchedulingNode._satisfies_label_constraints  s   
 " 	- 	-C 344M()<== - -8'+$"*"<  J$.C !899F#,B#{s33H2DDD#611380!E 2 4JJJ#v--380!E . 05,' -(1,,,,,,-)-, qr9   c                 ^   |j         r|                     |g          dk    rdS |j        D ]J}|                    d          r3|j        }|j        | j        v r|j        | j        |j                 k    r dS K|                     |          }t          |t          |j                            sdS t          |t          |j                             |                     ||           |j        D ]>}|                    d          r'|j        }|                     |j        |j                   ?dS )a  
        Try to schedule one resource request on this node. The request could be from
        various sources, specified by `resource_request_source`.

        Args:
            request: The resource request to be scheduled.
            resource_request_source: The source of the resource request, i.e.
                pending demands from ray actors/tasks or cluster resource constraints.

        Returns:
            True if the resource request is scheduled on this node.
        r   Fanti_affinityT)r   r   placement_constraintsHasFieldr   
label_namer]   label_valuerm   r   r4   r   r   rr   
_add_label)rG   rD   rj   r   r   available_resources_dicts         r:   r   z SchedulingNode._try_schedule_one7  sd   " " 	00';;q@@u "7 	 	J""?33 ! * 8!,;;%1{=#;<= = !55
 #'#?#?@W#X#X  -tG4L/M/MNN 	5 	2D9Q4R4RSSS 	w(?@@@ "7 	U 	UJ
 ""?33 U * 8 8-:STTTtr9   r   r   c                     | j                             |          /| j         |         |k    sJ d| d| j         |          d|             || j         |<   dS )zd
        Add a label to the node.
        This assumes a label key can only have one value.
        NzLabel z already exists with value z, cannot set to )r]   r   )rG   r   r   s      r:   r   zSchedulingNode._add_labelu  s     KOOJ''/{:&+555Z  {:&   656 #.Jr9   c                    d                     | j        | j        | j        | j        | j        | j        | j        r!t          t          | j                            nd | j
        | j        | j        t          j                 | j        t          j                 | j        | j        d                    d | j        t          j                 D                       d                    d | j        t          j                 D                                 S )Na  SchedulingNode(node_type={node_type}, node_kind={node_kind}, instance_id={instance_id},instance_status={instance_status},ray_node_id={ray_node_id},idle_duration_ms={idle_duration_ms},termination_request={termination_request},status={status}, total_resources={total_resources}, available_resources_for_demand={available_resources_for_demand}, available_resources_for_cluster_resource_constraints={available_resources_for_cluster_resource_constraints},labels={labels}, launch_reason={launch_reason}), sched_requests_for_demand={sched_requests_for_demand}), sched_requests_for_cluster_resource_constraints={sched_requests_for_cluster_resources_constraint})|c              3   N   K   | ] }t          t          |                    V  !d S rX   r   r   r   s     r:   r   z*SchedulingNode.__repr__.<locals>.<genexpr>  sF       / / OA&&''/ / / / / /r9   c              3   N   K   | ] }t          t          |                    V  !d S rX   r   r   s     r:   r   z*SchedulingNode.__repr__.<locals>.<genexpr>  sN       E E OA&&''E E E E E Er9   )rU   re   rv   rw   rb   rc   r_   rV   r\   available_resources_for_demand4available_resources_for_cluster_resource_constraintsr]   r^   sched_requests_for_demand/sched_requests_for_cluster_resources_constraint)formatrU   re   r`   ra   rb   rc   r_   r   r   rV   r\   r[   rP   rQ   rR   r]   r^   joinrZ   r   s    r:   __repr__zSchedulingNode.__repr__  s&   A  &nn+ 3(!2'!OD4L$M$M N N N; 0+/+M%4, BFAc%AB ;,&)hh / /,-B-QR/ / / ' ' =@HH E E,)EE E E = =1  
 
#/	
r9   NN)7r/   r0   r1   rI   r   r3   rK   r   rZ   r	   rP   r
   r   r4   r[   r   r6   r\   r]   r^   r   r_   r!   r`   ra   r   InstanceStatus	ValueTyperb   rc   r5   rd   r    r   re   ri   rm   ro   rr   staticmethodr   r   r2   r   rz   r   r   r   r   r   r   r   r   r   r   r8   r9   r:   rT   rT      s              IN11J J JND._0EEF    	d### "4tCJ//$ $ $ $ ).d(C(C(COT#u*%CCC #U4888FDcN888 $(M8C='''8<"45<<< %)NHSM((( GK!8!BCJJJ "&K#%%%c(,,,,"/Ix))) !JN !"$&o<@!7 !7!7 c5j)!7 "#u*-	!7
 S#X!7 %!7 !7 %X%<%FG!7 !7 !7  !7 !7 &&89!7 !7 !7 !7FK?T K K K K<:O < < < <E E "7E E E E [
$[
. 89[
 &*[
 
"	#	[
 [
 [
 \[
z !3     \< 
 )-,0
 
#
$
 
 !	

 %SM
 

 
 
 \
:9 9 9!-'!- "7!- 
tO$&66	7	!- !- !- !-F]
'<]
	]
 ]
 ]
 ]
~"?3	   ><&<AV<	< < < <|.S .s . . . .0
# 0
 0
 0
 0
 0
 0
r9   rT   c                   @   e Zd ZdZd$dee         fdZe G d d                      Zde	de
fd	Ze	 	 	 	 d%d            Ze	 	 	 	 d%d            Ze	 	 d&dee         dedej        dee         dee         deee         ee         f         fd            Zededefd            Ze	 	 	 	 d%d            Zed
ddee         dee         fd            Zed
ddee         dee         fd            Zed
ddee         dee         fd            Zed
ddee         dedeee         ee         f         fd            Zedee         dee         ded e e!e"f         deeee         ee         f         f
d!            Z#e	 	 	 	 d%d"            Z$e	 	 	 	 d%d#            Z%dS )'ResourceDemandScheduleraL  
    A resource demand scheduler that schedules resource requests based on the
    following rules:
        1. Enforce the minimal count of nodes for each worker node type.
        2. Enforce the cluster resource constraints.
        3. Schedule the gang resource requests.
        4. Schedule the tasks/actor resource requests
    Nevent_loggerc                     || _         d S rX   )_event_logger)rG   r   s     r:   ri   z ResourceDemandScheduler.__init__  s    )r9   c                      e Zd ZU dZeeef         ed<   eed<   dZ	e
e         ed<   dZe
e         ed<    ee          Zee         ed<    ee          Zeeef         ed	<    ee          Zeeef         ed
<   	 	 d#dee         deeef         deeef         dede
e         de
e         fdZededd fd            Zedee         deeef         deeef         fd            Zdee         fdZdeeef         fdZdeeef         fdZdeeef         fdZ de
e         fdZ!deeef         fdZ"dee         ddfdZ#de
e         fdZ$deeef         fdZ%defd Z&dee'         fd!Z(dee)         fd"Z*dS )$'ResourceDemandScheduler.ScheduleContextz
        Encapsulates the context for processing one scheduling request.

        This exposes functions to read and write the scheduling nodes, to prevent
        accidental modification of the internal state.
        _node_type_configs_disable_launch_config_checkN_max_num_nodes_idle_timeout_sr%   _nodes_node_type_available_cloud_resource_availabilitiesnodesr'   r.   r$   r(   r)   c                     || _         || _        |                     ||          | _        || _        || _        || _        || _        d S rX   )r   r   _compute_available_node_typesr   r   r   r   r   )rG   r   r'   r.   r$   r(   r)   s          r:   ri   z0ResourceDemandScheduler.ScheduleContext.__init__  sY      DK&7D#(,(J(J() )D% #0D#1D 0KD-2OD///r9   r   rE   c                     g }|j         }|j        D ]:}t                              |||j                  }|r|                    |           ; | |||j        |j        |j        |j                  S )aM  
            Create a schedule context from a schedule request.
            It will populate the context with the existing nodes and the available node
            types from the config.

            Args:
                req: The scheduling request. The caller should make sure the
                    request is valid.
            )r   r'   r.   r$   r(   r)   )	r'   r-   rT   r   r$   rq   r.   r(   r)   )clsr   r   r'   rs   nodes         r:   from_schedule_requestz=ResourceDemandScheduler.ScheduleContext.from_schedule_request  s     E # 5  1 ' '%))/1P   'LL&&&3"3.1.O,/,K!/"1   r9   c                     t          t                    }t          t                    }| D ]}||j        xx         dz  cc<   |                                D ]&\  }}|j        |                    |d          z
  ||<   '|S )a  
            Compute the number of nodes by node types available for launching based on
            the max number of workers in the config.
            Args:
                nodes: The current existing nodes.
                node_type_configs: The node type configs.
            Returns:
                A dict of node types and the number of nodes available for launching.
            r   r   )r   r5   rU   r   max_worker_nodesr   )r   r'   node_type_availablenode_type_existingr   rU   node_type_configs          r:   r   zEResourceDemandScheduler.ScheduleContext._compute_available_node_types  s     8C37G7G6A#6F6F 8 8"4>222a72222
 #((**    %58J8N8Nq9 9  $  '&r9   c                 8    t          j        | j                  }|S )zs
            Get the current nodes with filter.

            Returns:
                A list of nodes.
            )copydeepcopyr   )rG   r   s     r:   	get_nodesz1ResourceDemandScheduler.ScheduleContext.get_nodes6  s     M$+..ELr9   c                 4    t          j        | j                  S rX   )r   r  r   r   s    r:   get_node_type_availablez?ResourceDemandScheduler.ScheduleContext.get_node_type_available@  s    =!:;;;r9   c                     t          t                    }| j        D ]-}|j        t          j        k    r||j        xx         dz  cc<   .|S Nr   )r   r5   r   rV   rK   rN   rU   )rG   cluster_shaper   s      r:   get_cluster_shapez9ResourceDemandScheduler.ScheduleContext.get_cluster_shapeC  s[    ',,M 3 3;"6"CCCdn---2----  r9   c                     t          t                    }| j        D ]G}|j        t          j        k    r|j                                        D ]\  }}||xx         |z  cc<   H|S )a  
            Aggregate total cluster resources.

            Sums each node's `total_resources` across the current context,
            excluding nodes marked `TO_TERMINATE`.

            Returns:
                A dict mapping resource names to their summed resources.
            )r   r6   r   rV   rK   rN   r\   r   )rG   cluster_resourcesr   r   values        r:   get_cluster_resourcesz=ResourceDemandScheduler.ScheduleContext.get_cluster_resourcesM  s     !,E 2 2 4 4;"6"CCC"&"6"<"<">"> 4 4JC%c***e3****4$$r9   c                     | j         S rX   )r   r   s    r:   get_idle_timeout_sz:ResourceDemandScheduler.ScheduleContext.get_idle_timeout_sa  s    ''r9   c                 4    t          j        | j                  S rX   )r   r  r   r   s    r:   !get_cloud_resource_availabilitieszIResourceDemandScheduler.ScheduleContext.get_cloud_resource_availabilitiesd  s    =!DEEEr9   	new_nodesc                 ^    || _         |                     | j         | j                  | _        dS )z@
            Update the context with the new nodes.
            N)r   r   r   r   )rG   r  s     r:   updatez.ResourceDemandScheduler.ScheduleContext.updateg  s5     $DK )-(J(JT4) )D%%%r9   c                     | j         S )zM
            Get the max number of nodes for the entire cluster.
            )r   r   s    r:   get_max_num_nodesz9ResourceDemandScheduler.ScheduleContext.get_max_num_nodesr  s     &&r9   c                     | j         S rX   )r   r   s    r:   get_node_type_configsz=ResourceDemandScheduler.ScheduleContext.get_node_type_configsx  s    **r9   c                 v    d                     t          | j                  t          | j                            S )Nz1ScheduleContext({} nodes, node_type_available={}))r   r   r   r4   r   r   s    r:   __str__z/ResourceDemandScheduler.ScheduleContext.__str__{  s4    FMMDK  $t'@"A"A  r9   c                 ~   t          t                    }| j        D ],}|j        t          j        k    r||j        xx         dz  cc<   -g }|                                D ]]\  }}|                    t          ||t          t          j                              t          j                    dz                       ^|S )zX
            Get the launch requests for the nodes that are to be launched.
            r     )ry   countru   request_ts_ms)r   r5   r   rV   rK   rL   rU   r   rq   r   r   r   r   timetime_ns)rG   launch_by_typer   launch_requestsry   r  s         r:   get_launch_requestsz;ResourceDemandScheduler.ScheduleContext.get_launch_requests  s     )--N 8 8;"6"@@@"4>222a7222 O(6(<(<(>(>  $u&&!&3#tz||,,&*lnn&<	      #"r9   c                 $    d | j         D             S )z]
            Get the terminate requests for the nodes that are to be terminated.
            c                 *    g | ]}|j         	|j         S rX   )r_   )r   r   s     r:   
<listcomp>zRResourceDemandScheduler.ScheduleContext.get_terminate_requests.<locals>.<listcomp>  s.       +7 (777r9   )r   r   s    r:   get_terminate_requestsz>ResourceDemandScheduler.ScheduleContext.get_terminate_requests  s%      K   r9   r   )+r/   r0   r1   rI   r	   r   r   r3   r2   r   r   r5   r   r6   r   r7   r   r
   rT   r4   r   r   ri   classmethodr#   r   r   r   r  r  r  r   r  r  r  r  r  r  r  r   r"  r!   r&  r8   r9   r:   ScheduleContextr     s        	 	 !>!9::::&****(,,,,+/%///',uT'B'B'B^$BBB 5:E$4O4O4Od8S=1OOO AF A
 A
 A
&Xu_(= 	
 	
 	
 ,0.2	P 	P'	P  $Hn$<=	P ,0%+@		P
 *.	P $C=	P %UO	P 	P 	P 	P& 
	'	6	 	 	 
	B 
	''	'#Hn$<=	' (C- 	' 	' 	' 
	'<	tN3 	 	 	 		<T(C--@ 	< 	< 	< 	<	!tHcM': 	! 	! 	! 	!	%4U
+; 	% 	% 	% 	%(	( 	( 	( 	( 	(	FtHeO7L 	F 	F 	F 	F		D$8 		T 		 		 		 			'x} 	' 	' 	' 	'	+4.0H+I 	+ 	+ 	+ 	+	S 	 	 	 	
	#m)< 	# 	# 	# 	#*
	$%
	 
	 
	 
	 
	 
	r9   r(  rD   rE   c           	      b   t                               d                    t          j        |j                  t          j        |j                  t          j        |j                                       t          j
                            |          }t                              |           t                              |           t                              |           t                              |           t                              ||j                  }t                              ||j                  }t                              |t          j        |j                            }t                              |           t+          ||||                                |                                          }| j        h	 | j                            |j        |j        ||||                                           n*# t:          $ r t                               d           Y nw xY w|S )Nz\Scheduling for request: resource_request={}, gang_resource_request={}, cluster_constraint={})r?   r@   rA   r=   r>   )r!  terminate_requestsinfeasible_requestsinfeasible_gang_requestsrA   r
  zFailed to emit event logs.)r   debugr   r   to_dict_listr*   r   r+   r,   r   r(  r   _terminate_outdated_nodes_enforce_min_workers_per_type_enforce_max_workers_per_type_enforce_max_workers_global_enforce_resource_constraints_sched_gang_resource_requests_sched_resource_requestsungroup_by_count_enforce_idle_terminationr<   r"  r&  r   log_cluster_scheduling_updater=   r>   r  	Exception	exception)rG   rD   ctxinfeasible_constraintsr,  r+  replys          r:   rH   z ResourceDemandScheduler.schedule  s(   $$*F#01JKK)'*HII)'*NOO% %	
 	
 	
 &5KKGTT 	 99#>>> 	 ==cBBB 	 ==cBBB 	 ;;C@@@ "9!V!V5"
 "
 $AAW3  	! 6NN01JKK
 
 	 99#>>>  )<.F4J--//3355
 
 
 )
?"@@$)O','9(;-E<R&)&?&?&A&A A      ? ? ?  !=>>>>>? s   <H $H,+H,r;  r   c                    |                                  }t          t                    }g }|D ]M}|j        t          j        k    r|                    |           -||j                                     |           N|                                D ]}||         }| 	                                |         }|j
        }t          |          |z
  }	|	dk    rDt                              ||	t          j        j        |          \  }
}|||<   |                    |
           g }|                                D ]}|                    |           t          |          t          ||z             k    s
J d            |                     ||z              |r-t(                              dt          |           d           dS dS )zG
        Enforce the max number of workers for each node type.
        r   )max_num_nodes_per_typezJThe number of nodes should be the same after enforcing max nodes per type.zTerminating z4 nodes for per node type max num node's constraints.N)r  r   r7   rV   rK   rN   rq   rU   keysr  r   r   r   _select_nodes_to_terminater!   r   MAX_NUM_NODE_PER_TYPEextendr   r  r   r-  )r;  	all_nodesnon_terminating_nodes_by_typeterminating_nodesr   rU   non_terminate_nodes_of_typer   num_max_nodes_per_typenum_extra_nodesr>   remained_nodesnon_terminating_nodesr   s                 r:   r1  z5ResourceDemandScheduler._enforce_max_workers_per_type  s+    MMOO	(3D(9(9% 	K 	KD{2???!((....-dn=DDTJJJJ 7;;== 	3 	3I*G	*R'3355i@K%0%A"!"=>>AWWO!## (BB+"(>'=	 C   8F))4$$\2222 "299;; 	0 	0E!((//// 9~~ 55"
 "
 
 
 
W
 
 
 	

$'<<=== 	LLFs#455 F F F    	 	r9   c                    |                                  }g }g }|D ]B}|j        t          j        k    r|                    |           -|                    |           C|                                 }|r t          t          |          |z
  d          nd}|dk    rdS t          	                    ||t          j        j        |          \  }}t          |          |k    s<J d                    t          |          |t          |          z
  |                      |                    |           t          |          t          ||z             k    s
J d            ||z   }|                     |           dS )zK
        Enforce the max number of workers for the entire cluster.
        r   N)r(   zNTerminating {} nodes, failed to terminate {} nodes to satisfy max_num_nodes={}zAThe number of nodes should be the same after enforcing max nodes.)r  rV   rK   rN   rq   r  maxr   r   rA  r!   r   MAX_NUM_NODESr   rC  r  )r;  rD  rF  rK  r   num_max_nodesnum_to_terminateto_terminate_nodess           r:   r2  z3ResourceDemandScheduler._enforce_max_workers_global$  s    MMOO	 " 	3 	3D{2???!((....%,,T2222--// CPVC)**]:A>>>UV 	 q  F $>>!$2'	 ? 
 
	
! %&&*::::''-v&'' 3'9#:#::( ( ;:: 	  !34449~~ 55"
 "
 
 
 
N
 
 
 &(==	

9r9   r   rP  rx   r(   r?  c                 @   |                      t          j                   d}t          |           D ]1\  }}|j        t
          j        k    r|                     |          } n2| d|         | |d         |r|gng z   }	}|t          j	        j
        t          j	        j        fv s
J d            |D ]}t          j        |_        t          t          t!          j                              |j        |j        ||j        |j        dt          j	                            |           d| d|           |_        |t          j	        j
        k    r||j        _        |t          j	        j        k    r||j        _        t5          d                    |                    ||	fS )	a  
        Select 'num_to_terminate' of nodes to be terminated
        from the 'nodes' list. It should never select a head node.

        Args:
            nodes: The nodes to be terminated.
            num_to_terminate: The number of nodes to be terminated.
            cause: The cause of the termination. Should be one of
                TerminationRequest.Cause.MAX_NUM_NODES or
                TerminationRequest.Cause.MAX_NUM_NODE_PER_TYPE.

            max_num_nodes: The max number of nodes for the entire cluster only
                used when the cause is TerminationRequest.Cause.MAX_NUM_NODES.
            max_num_nodes_per_type: The max number of nodes for each node type.
                Only used when the cause is
                TerminationRequest.Cause.MAX_NUM_NODE_PER_TYPE.

        Returns:
            A tuple of:
                - The terminated nodes.
                - The remained nodes.
        )r   NzDOther termination causes don't have to select nodes for termination.zTerminating node due to z: max_num_nodes=z, max_num_nodes_per_type=)ru   rv   rb   rx   ry   rw   detailszUnknown termination cause: {})sortr   _sort_nodes_for_terminationr   re   r    HEADpopr!   r   rN  rB  rK   rN   rV   r   r   r   r`   rb   rU   ra   Namer_   r(   r?  
ValueErrorr   )
r   rP  rx   r(   r?  	head_noder   r   terminated_nodesrJ  s
             r:   rA  z2ResourceDemandScheduler._select_nodes_to_terminate]  s   @ 	

.J
KKK 	 '' 	 	GAt~..!IIaLL	 / ###$ "##$y(HbI	 ) $2$:
 
 
 
 R
 
 

 % 	P 	PD.;DK'9tz||$$ / ,"n $ 7G/A/G/L/LU/S/S G G%2G G.DG G( ( (D$ *0>>>9F(66,2HHHBX(?? !@!G!G!N!NOOO//r9   r   c                    t          | j                  dk    }d| j        z  }|                     t          j                  }i }| j                                        D ]+\  }}|dk    r||                    |d          z
  |z  ||<   ,|r1t          |
                                          t          |          z  nd}|||fS )a  
        Sort the nodes for termination increasingly by:

            1. First if ray hasn't been started yet
            2. Then if the nodes are idle
            3. Then with lower resources util nodes first.

        Such that nodes sorted earlier will be terminated first.
        r   )r   rb   rc   rm   rP   rQ   r\   r   r   r   r   )r   running_rayidle_durrg   utils_per_resourcesresourcetotalavg_utils           r:   rU  z3ResourceDemandScheduler._sort_nodes_for_termination  s     $*++a/ --"::!0
 
 !#399;; 	 	OHezz+//!<<<-)) #C#**,,--4G0H0HHH 	 Xx00r9   c                 &   |                                  }g }|                                                                 D ]\  }}|                    |d          }|j        }||k     rt
                              d||z
   d| d           |                    t          	                    t          j        |          t          j        t          j                  g||z
  z             |                     ||                                 z              dS )zO
        Enforce the minimal count of nodes for each worker node type.
        r   zAdding z+ nodes to satisfy min count for node type: .rV   re   N)r  r  r   r   min_worker_nodesr   r   rC  rT   r   r   r  rK   rL   r    r   r  r  )r;  count_by_node_typer  rU   r   	cur_count	min_counts          r:   r0  z5ResourceDemandScheduler._enforce_min_workers_per_type  s:    !2244	
 &&((..00	 	 
*..y!<<I(9I9$$/i)3 / /"+/ / /     &77 M*:;;#7#A&.o 8   !9,.	 	 	 	

9s}}./////r9   constraintsc                 4   t          |          dk    s
J d            t          |          dk    rg S |d         }t          j        |j                  }t                              | |t          j                  \  }}|r|gS |                     |           g S )a  
        Enforce the cluster resource constraints.

        Args:
            ctx: The schedule context.
            constraints: The cluster resource constraints.

        Returns:
            A list of infeasible constraints.

        Notes:
            It's different from the other scheduling functions since it doesn't actually
        schedule any resource requests. Instead, it asks if the cluster could be
        upscale to a certain shape to fulfill the constraints.
        r   z/Max 1 cluster resource constraint is supported.r   rj   )	r   r   r6  r*   r   _try_schedulerP   rR   r  )r;  rk  r   r   scheduled_nodes
infeasibles         r:   r3  z5ResourceDemandScheduler._enforce_resource_constraints  s    . ;1$$$&W$$${q  I ^
&7
8TUU '>&K&K$9$U 'L '
 '
#  	 <

?###	r9   r   c                     t                               | |t          j                  \  }}|                     |           |S )z
        Schedule the resource requests.

        Args:
            ctx: The schedule context.
            requests_by_count: The resource requests.

        Returns:
            A list of infeasible resource requests.
        rm  )r   rn  rP   rQ   r  )r;  r   r   rp  s       r:   r5  z0ResourceDemandScheduler._sched_resource_requests%  sI     4AA3H3W B 
 
z
 	

5r9   gang_requestsc                 n   dt           dt          fd}g }t          ||d          D ]}|j        r|j        d         j        }n|j        }t          j        |          }t          	                    | |t          j                  \  }}|r|                    |           x|                     |           |S )a@  
        Schedule the gang resource requests.

        These requests should be scheduled atomically, i.e. either all of the resources
        requests in a gang request are scheduled or none of them are scheduled.

        For now, the gang resource requests represent Ray's placement groups, while it
        could be more general in the future:
        - For STRICT_PACK placement group requests, we combine them into a single
            request and try to schedule them together.
        - For STRICT_SPREAD placement groups requests, they should be scheduled on
            different nodes by leveraging on the node labels that are associated with
            the placement group.
            If there are requests from rescheduling placement groups due to node
            failures, these requests should not be scheduled on nodes with requests
            from the same placement group.


        Args:
            ctx: The schedule context.
            gang_requests: The gang resource requests.

        Returns:
            A list of infeasible gang resource requests.
        r   rE   c                 t    d}| j         D ]}|t          |j                  z  }|t          | j                   fS )z
            Key function for sorting the gang resource request by:
                1. the number of placement constraints in the gang request.
                2. the number of resource requests in the gang request.
            r   )r   r   r   )r   total_placement_constraintsresource_requests      r:   _sort_gang_resource_requestsz[ResourceDemandScheduler._sched_gang_resource_requests.<locals>._sort_gang_resource_requests\  sS     +,'$'L   +s$:0 0 ++ 0S\1B1BCCr9   Tr   reverser   )r   r   sortedbundle_selectorsr*   r   r   combine_requests_with_affinityr   rn  rP   rQ   rq   r  )r;  rr  rw  r,  gang_reqr   r   rp  s           r:   r4  z5ResourceDemandScheduler._sched_gang_resource_requests=  s    >	D.A 	De 	D 	D 	D 	D $& ;T
 
 
 	 	H ( - $4Q7I $,*I(SSH 7 E EX4C! !E:   )//999 JJu''r9   requests_to_schedrj   c                 0    dt           dt          fd}t          ||d          }                                 }                                 }g }t          |          dk    rt          |          dk    rqt                              |||                                           \  }}}|n;|	                    |           t          |          dk    rt          |          dk    q|
                    |            fd|                                D             }t          |          dk    rdt          |          dk    rP                                 }	|	At          |          |	k    r.t                              d	                    |	                     nt                              |||                                           \  }}}|n|	                    |           ||j        xx         d
z  cc<   ||j                 dk    ra|	                    t"                                                               |j                 t(          j        t,          j                             t          |          dk    rt          |          dk    P||fS )a  
        Try to schedule the resource requests on the current context.

        It tries to schedule the requests on the existing nodes first, and
        then try to schedule the requests on new nodes if possible.

        Args:
            requests_to_sched: The resource requests to be scheduled.
            ctx: The current scheduling context.
            resource_request_source: The source of the resource request, i.e.
                pending demands from ray actors/tasks or cluster resource
                constraints.

        Returns:
            - List of scheduled nodes to that have part or all of the requests
                scheduled.
            - List of infeasible requests remained that cannot be scheduled.
        r   rE   c                 \   | j         rt          | j         d         j                  nd}t          | j                  |t          | j                                                  t          | j                                                  t          | j                                                  fS )aR  
            Sort the resource requests by:
                1. The length of its placement constraints.
                2. The length of its first label selector constraints (if any).
                3. The number of resources it requests.
                4. The values of resources it requests.
                5. lexicographically for each resource (for stable ordering)

            This is a legacy sorting function for the autoscaler's binpacking
            algo - we do this so that we could have a deterministic scheduling
            results with reasonable fragmentation.
            r   )	r   r   r   r   r   r   r   rz  r   )r   label_constraint_lens     r:   _sort_resource_requestzEResourceDemandScheduler._try_schedule.<locals>._sort_resource_request  s     &C'*<=== ! C-..$C(//1122C(//1122s+113344 r9   Trx  r   Nc                     g | ]T\  }}|d k    t                                                               |         t          j        t
          j                  US )r   rf  )rT   r   r  rK   rL   r    r   )r   rU   num_availabler;  s      r:   r%  z9ResourceDemandScheduler._try_schedule.<locals>.<listcomp>  sj     
 
 
 )	=q   ++))++I6+5"/ ,   !  r9   z:Max number of nodes reached: {}, cannot launch more nodes.r   rf  )r   r   rz  r  r  r   r   _sched_best_noder  rq   rC  r   r  r   r-  r   rU   rT   r   r  rK   rL   r    r   )
r;  r~  rj   r  existing_nodesr   target_nodes	best_node
node_poolsr(   s
   `         r:   rn  z%ResourceDemandScheduler._try_schedule  s   2	 	E 	 	 	 	4 ##94
 
 
 !99;;
  #$$q((S-@-@1-D-D
 (88!'5577	 	!  	*** #$$q((S-@-@1-D-D$ 	N+++
 
 
 
 -@,E,E,G,G
 
 

 #$$q((S__q-@-@1133M(S->->--O-O006}0E0E    (88!'5577	 	!  	***  	 34449444"9#67!;;!!"331133I4GH3="*/ 4    9 #$$q((S__q-@-@H ...r9   r.   c           	          g }t            G d d                      }t          j        |          }t          |          D ]`\  }}|                    | |          \  }	}
t          |	          t          |           k    r?|                     |||	||
                     at          |          dk    rEt                              d	                    t          j        |           |                     d| |fS t          |fdd          }|d         }|                    |j                   t                              d		                    |j        |j        t          j        |j                                       |j        |j        |fS )
a"  
        Schedule the requests on the best node.
        A simple greedy algorithm is used to schedule the requests:
            1. Try to schedule the requests on each node.
            2. Sort the nodes by a score. The sorting includes:
                2.1. UtilizationScore: to maximize resource utilization.
                2.2. Cloud resource availabilities: prioritize node types with
                the most available cloud resources, in order to minimize allocation
                failures.
            3. Return the node with the highest score.

        The highest score node is updated with the scheduled requests, and the node is
        removed from the node list.

        Args:
            requests: The resource requests to be scheduled.
            nodes: The node candidates to be scheduled on. The nodes will be updated
                after the scheduling attempt, i.e. the node that is scheduled will be
                removed from the list.
            resource_request_source: The source of the resource request, i.e.
                pending demands from ray actors/tasks or cluster resource constraints.
            cloud_resource_availabilities: The cloud resource availability score. A low
                score indicates that allocation for this node type has recently failed.

        Returns:
            best_node: The best node to schedule the requests.
            infeasible: The infeasible requests that cannot be scheduled on the best
                node.
            nodes: Remaining nodes after the best node is removed.
        c                   D    e Zd ZU eed<   ee         ed<   eed<   eed<   dS )@ResourceDemandScheduler._sched_best_node.<locals>.ScheduleResultr   r+  idxr   N)	r/   r0   r1   rT   r3   r
   r   r5   r   r8   r9   r:   ScheduleResultr  4  sF          !   !%o!6666HHH######r9   r  r   z5No nodes can schedule the requests: {}, for nodes: {}Nc                 R    | j                             | j        j        d          fS r  )r   r   r   rU   )r   r.   s    r:   rY   z:ResourceDemandScheduler._sched_best_node.<locals>.<lambda>X  s'    -11!&2BAFF r9   Trx  z0Best node: {}, score: {}, remaining requests: {})r   r   r  r   r   r   rq   r   r-  r   r   r.  rz  rW  r  r   r   r+  )r   r   rj   r.   resultsr  
nodes_copyr  r   	remainingr   best_results      `        r:   r  z(ResourceDemandScheduler._sched_best_node  s   J  
	$ 	$ 	$ 	$ 	$ 	$ 	$ 
	$ ]5))
 #:.. 	H 	HIC#00;RSSIu9~~X..NN>>$	3FFGGGG w<<1LLGNN'4X>>   
 5((     
 
 
 aj		+/""">EE !#01PQQ 	
 	
 	
 !@%GGr9   c                    |                                  }| j        rdS |D ]}|j        t          j        k    r|j        t          j        k    r,t          	                    d|j
         d|j         d           Z|j        }|                                                     |          }||j        r|j        |j        k    rvt          j        |_        t#          t%          t'          j                              |j
        |j        |j        |j        t"          j        j        d|j         d          |_        |                     |           dS )z
        Terminate the nodes that are outdated, i.e. the node type config has been
        updated or the node's launch config hash is outdated.

        Args:
            ctx: The schedule context.
        Nz
Head node z(ray=z) is outdated with node config changes. Please check the node's config or restart the cluster or restart the head node. Autoscaler is not able to shutdown the outdated head nodez
node from z has outdated config)ru   rv   rb   ry   rw   rx   rS  )r  r   rV   rK   rM   re   r    rV  r   warningr`   rb   rU   r  r   rd   rN   r!   r   r  r  ra   r   r   r_   r  )r;  r   r   rU   r   s        r:   r/  z1ResourceDemandScheduler._terminate_outdated_nodesk  s^    + 	F  	  	D{2>>>~.. !4    4;K         I"88::>>yII' 3 ($74;RRR 3?+=4<>>** $ 3 $ 0"&.$($;,2;MMMM, , ,( 	

5r9   c                 4   |                                  }|                                 }t          t                    }|                                 }d}|D ],}|j        t          j        k    r|j        t          j
        k    r/|                                 }|j        }||v r||         j        ||         j        }|k|j        ||z  k    rz|j        t           j                 r<t$                              d                    |j        |j        |z                       |j        t           j                 r=t$                              d                    |j        |j        |z                       !d}	||v r||         j        }	|                    |d          ||         z
  |	k    r>t$                              d                    |j        |j        |z  |                     ||j        xx         dz  cc<   t          j        |_        t7          t9          t;          j                              |j        |j        t6          j         j!        |j        |j"        |j        d|j        |z   d	| d
          |_#        .| $                    |           dS )z
        Enforce the idle termination for the nodes that are not needed by the cluster
        resource constraints and idle for too long.

        Args:
            ctx: The schedule context.
        r  NzTNode {} (idle for {} secs) is needed by the pending requests, skip idle termination.z`Node {} (idle for {} secs) is needed by the cluster resource constraints, skip idle termination.r   zrNode {} (idle for {} secs) belongs to node_type {} and is required by min_worker_nodes, skipping idle termination.r   z	idle for z secs > timeout=z secs)ru   rv   rb   rx   ry   rw   rc   rS  )%r  r  r   r5   r  rV   rK   rM   re   r    rV  r  rU   r)   rc   rZ   rP   rQ   r   r-  r   rb   rR   rg  r   r   rN   r!   r   r   r   r`   r   IDLEra   r_   r  )
r;  rh  r'   terminate_nodes_by_typer   s_to_msr   r)   rU   rj  s
             r:   r7  z1ResourceDemandScheduler._enforce_idle_termination  s    !224455777B37G7G K	 K	D{2>>>~.. 3355NI---$Y/>J%6y%A%PN%$(@@@"#8#GH 	 --3V($*?'*I. .   "#8#TU 	 ::@&($*?'*I; ;    I----i8I	"&&y!44))45  OOUv($*?'*I9P P   #DN333q8333.;DK'9tz||$$ / ,(.3"n $ 7!%!61D$9'$A 1 1)1 1 1
( 
( 
(D$$ 	

5r9   rX   )r;  r   rE   Nr   )&r/   r0   r1   rI   r   r   ri   r   r(  r#   r<   rH   r   r1  r2  r
   rT   r5   r!   r   r   rA  rU  r0  r   r3  r   r5  r   r4  rP   rn  r	   r   r6   r  r/  r7  r8   r9   r:   r   r     s4        * *X.C%D * * * * Z Z Z Z Z Z Z YZxD 1 Do D D D DL :6:	: : : \:x 666	6 6 6 \6p 
 (,04K0 K0N#K0K0 "'K0  }	K0
 !)K0 
tN#T.%99	:K0 K0 K0 \K0Z !1. !1U !1 !1 !1 \!1F %06%0	%0 %0 %0 \%0N +6+34+ 
'	(+ + + \+Z 6' 
o	   \. H(6H(/0H( 
!	"H( H( H( \H(T A/6A/0A/ "7A/ 
tN#T/%::	;	A/ A/ A/ \A/F \H'\HN#\H "7\H (,HeO'<	\H
 
~tO4d>6JJ	K\H \H \H \\H| 262	2 2 2 \2h ]6]	] ] ] \] ] ]r9   r   )=r   loggingr  r   abcr   r   collectionsr   dataclassesr   r   enumr   typingr	   r
   r   r   ray._private.protobuf_compatr   !ray.autoscaler._private.constantsr   1ray.autoscaler._private.resource_demand_schedulerr   r   r   ray.autoscaler.v2.event_loggerr   )ray.autoscaler.v2.instance_manager.commonr   )ray.autoscaler.v2.instance_manager.configr   ray.autoscaler.v2.schemar   r   ray.autoscaler.v2.utilsr   r   !ray.core.generated.autoscaler_pb2r   r   r   r   ray.core.generated.common_pb2r   'ray.core.generated.instance_manager_pb2r   r   r    r!   	getLoggerr/   r   r#   r<   rC   rK   rP   rT   r   r8   r9   r:   <module>r     ss       # # # # # # # # # # # # # # ( ( ( ( ( ( ( (       . . . . . . . . . . . . 8 8 8 8 8 8 K K K K K K         
 A @ @ @ @ @ B B B B B B D D D D D D A A A A A A A A E E E E E E E E            @ ? ? ? ? ?            
	8	$	$ W W W W W W W W2        "       "" " " " "4 " " "
@ 
@ 
@ 
@ 
@D 
@ 
@ 
@ k
 k
 k
 k
 k
 k
 k
 k
\G G G G G0 G G G G Gr9   