
    &`iVb                       d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m	Z	 d dl
mZ d dlmZ d dlmZmZmZmZmZmZmZ d dlZd dlmZmZ d dlmZ d d	lmZ d d
lmZmZm Z m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZA d dlBmCZC d dlDmEZEmFZFmGZGmHZHmIZI d dlJmKZK d dlLmMZMmNZN d dlOmPZP d dlQmRZR d dlSmTZTmUZUmVZVmWZWmXZXmYZY d dlZm[Z[ d dl\m]Z] d dl^m_Z_m`Z`maZambZb d dlcmdZd  eje        e@          Zf G d de          Zg G d d e          Zhe G d! d"                      Zie G d# d$                      Zjd%Zk elejm        n                    d&d'                    Zo elejm        n                    d(d'                    Zp eqe3          Zrejm        n                    d)d*          d*k    Zsejm        n                    d+d*          d,k    Ztd- Zu G d. d/          Zv G d0 d1          Zw G d2 d3          Zx G d4 d5          Zy G d6 d7          Zz G d8 d9          Z{ G d: d;          Z|dS )<    N)defaultdict)copy)	dataclass)Enum)AnyCallableDictListOptionalSetTuple)	ObjectRefcloudpickle)ray_constants)ActorHandle)RayActorErrorRayErrorRayTaskErrorRuntimeEnvSetupError)metrics)default_impl)AutoscalingStateManager)ClusterNodeInfoCache)DeploymentIDDeploymentStatusDeploymentStatusInfoDeploymentStatusInternalTriggerDeploymentStatusTriggerDeploymentTargetInfoDuration	ReplicaIDReplicaStateRequestRoutingInfoRunningReplicaInfo)DeploymentConfig)	&MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNTMAX_PER_REPLICA_RETRY_COUNTRAY_SERVE_ENABLE_TASK_EVENTSRAY_SERVE_FAIL_ON_RANK_ERROR'RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS&RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY(REPLICA_HEALTH_CHECK_UNHEALTHY_THRESHOLDSERVE_LOGGER_NAMESERVE_NAMESPACE)DeploymentInfo)DeploymentDownscaleRequestDeploymentSchedulerReplicaSchedulingRequestReplicaSchedulingRequestStatus SpreadDeploymentSchedulingPolicy)DeploymentIsBeingDeletedError)LongPollHostLongPollNamespace)KVStoreBase)ServeUsageTag)JavaActorHandleProxycheck_obj_ref_ready_nowait"get_capacity_adjusted_num_replicasget_random_stringmsgpack_deserializemsgpack_serialize)DeploymentVersion)DeploymentLanguage)DeploymentDetailsReplicaDetailsReplicaRank_deployment_info_to_schema)PlacementGroupc                       e Zd ZdZdZdZdZdS )ReplicaStartupStatus            N)__name__
__module____qualname__PENDING_ALLOCATIONPENDING_INITIALIZATION	SUCCEEDEDFAILED     w/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/serve/_private/deployment_state.pyrH   rH   Q   s$        IFFFrU   rH   c                       e Zd ZdZdZdZdZdS )ReplicaHealthCheckResponserI   rJ   rK   rL   N)rM   rN   rO   NONErR   APP_FAILUREACTOR_CRASHEDrT   rU   rV   rX   rX   X   s"        DIKMMMrU   rX   c            
           e Zd ZU dZee         ed<   eed<   ee         ed<   e	ed<   e
dd            Ze
dd	dedede	dd fd
            Zdd de	fdZdS )DeploymentTargetStateaQ  The current goal state for a deployment.

    info: contains the information needed to initialize a replica.
    target_num_replicas: the number of replicas to run. This should already
        be adjusted by the target_capacity.
    version: the goal version of the deployment.
    deleting: whether the deployment is being deleted.
    infotarget_num_replicasversiondeletingreturnc                      | d dd d          S )NFrT   )clss    rV   defaultzDeploymentTargetState.defaulto   s    s4T5)))rU   Fra   c          	          |r|dk    rt          d| d          t          |j        |j        |j        j        |j        j        |j        j        |j        j        |j	                  } | ||||          S )Nr   zItarget_num_replicas must be 0 when setting target state to deleting. Got z	 instead.)deployment_configray_actor_optionsplacement_group_bundlesplacement_group_strategymax_replicas_per_noderoute_prefix)

ValueErrorr@   r`   ri   replica_configrj   rk   rl   rm   rn   )re   r^   r_   ra   r`   s        rV   createzDeploymentTargetState.creates   s      	"a'' G(;G G G  
 $L"4"1C$($7$O%)%8%Q"&"5"K*
 
 
 s4,gx@@@rU   other_target_statec           
         |j         dS t          | j         j        j        |j         j        j        k    | j         j        j        |j         j        j        k    | j         j        j        |j         j        j        k    | j         j        j        |j         j        j        k    | j         j                            dh          |j         j                            dh          k    | j	        | j	        |j	        k    g          S )ai  Checks if this target state is a scaled copy of another target state.

        A target state is a scaled copy of another target state if all
        configurable info is identical, other than target_num_replicas.

        Returns: True if this target state contains a non-None DeploymentInfo
            and is a scaled copy of the other target state.
        NFnum_replicas)exclude)
r^   allrp   rj   rk   rl   rm   ri   dictr`   )selfrr   s     rV   is_scaled_copy_ofz'DeploymentTargetState.is_scaled_copy_of   s     "*5	(:%*9KL	(@%*9QR	(A%*9RS	(>%*9OP	+00.9I0JJ%*<AA+, B    2 ::!
 
 	
rU   N)rb   r]   )rM   rN   rO   __doc__r   r/   __annotations__intr@   boolclassmethodrf   rq   ry   rT   rU   rV   r]   r]   _   s           >
""""'((((NNN* * * [*  A A AA !A
 A 
!A A A [A4 
4K  
PT  
  
  
  
  
  
rU   r]   c                   P    e Zd ZU eed<   eed<   ee         ed<   ee         ed<   dS )DeploymentStateUpdateResultdeletedany_replicas_recoveringupscale	downscaleN)	rM   rN   rO   r}   r{   r
   r2   r   r0   rT   rU   rV   r   r      sK         MMM!!!!*++++2333333rU   r   z!serve-deployment-state-checkpointSERVE_SLOW_STARTUP_WARNING_S   #SERVE_SLOW_STARTUP_WARNING_PERIOD_SSERVE_ENABLE_SCALING_LOG0;RAY_SERVE_DISABLE_SHUTTING_DOWN_INGRESS_REPLICAS_FORCEFULLY1c                     t           sJ d} d}g }t          j                            |           rDt	          |           5 }|                                | d          }d d d            n# 1 swxY w Y   t          j                    t          j                    t          j	                    |d}t                              dt          j        |d                      d S )Nz(/tmp/ray/session_latest/logs/monitor.log2   )nodesavailable_resourcestotal_resourcesautoscaler_logszScaling information
rJ   )indent)_SCALING_LOG_ENABLEDospathexistsopen	readlinesrayr   r   cluster_resourcesloggererrorjsondumps)log_pathlast_n_linesautoscaler_log_last_n_linesf
debug_infos        rV   print_verbose_scaling_logr      s    9HL"$	w~~h H(^^ 	Hq*+++--*G'	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H "6880226	 J LLKJq)I)I)IKKLLLLLs   A((A,/A,c                      e Zd ZdZdedefdZedefd            Z	edefd            Z
edee         fd            Zedefd	            Zedefd
            Zedee         fd            Zedeeeeef                           fd            Zedefd            Zedefd            Zedee         fd            Zedeee                  fd            Zedefd            Zedefd            Zedefd            Zedefd            Z edefd            Z!edee         fd            Z"edee         fd            Z#edefd            Z$edefd            Z%edee         fd            Z&edee         fd            Z'edee         fd            Z(edee         fd            Z)edee         fd            Z*edee         fd            Z+edee         fd             Z,edee         fd!            Z-d"e.d#e/egef         de0fd$Z1	 d<d&ed'ee2         fd(Z3d)e4fd*Z5ded+edefd,Z6defd-Z7de8e9ee         f         fd.Z:edeeeef                  fd/            Z;edeeef         fd0            Z<de=fd1Z>defd2Z?de@fd3ZAdefd4ZBdefd5ZCdefd6ZDdeee4f         fd7ZEd=d9efd:ZFdeeeG                  fd;ZHd%S )>ActorReplicaWrapperzWraps a Ray actor for a deployment replica.

    This is primarily defined so that we can mock out actual Ray operations
    for unit testing.

    *All Ray API calls should be made here, not in DeploymentState.*
    
replica_idr`   c                 ,   || _         |j        | _        |                                | _        d | _        d | _        d | _        || _        d| _	        d | _
        d| _        d| _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d | _        d| _        d| _         i | _!        d | _"        d| _#        d| _$        d | _%        d S )NTg        r   F)&_replica_iddeployment_id_deployment_idto_full_id_str_actor_name_allocated_obj_ref_ready_obj_ref_actor_resources_version_healthy_health_check_ref_last_health_check_time"_consecutive_health_check_failures_initialization_latency_s_internal_grpc_port
_docs_path_route_patterns_assign_rank_callback_rank_actor_handle_placement_group_pid	_actor_id
_worker_id_node_id_node_ip_node_instance_id_log_file_path
_http_port
_grpc_port_graceful_shutdown_ref_is_cross_language_deployment_is_cross_language_routing_stats_record_routing_stats_ref_last_record_routing_stats_time_ingress_outbound_deploymentsrx   r   r`   s      rV   __init__zActorReplicaWrapper.__init__   s;   
 &(6%4466 .2)-26 ,3"6:.1$23/:>&26 )-48SW",0
*.04 	"#!!&*#'## 26# #(-2*.0>B&69,# DH"""rU   rb   c                     | j         S Nr   rx   s    rV   r   zActorReplicaWrapper.replica_id       rU   c                     | j         j        S r   )r   namer   s    rV   deployment_namez#ActorReplicaWrapper.deployment_name$  s    "''rU   c                     | j         S r   )r   r   s    rV   rankzActorReplicaWrapper.rank(  s
    zrU   c                     | j         j        S r   )r   app_namer   s    rV   r   zActorReplicaWrapper.app_name,  s    "++rU   c                     | j         S r   )r   r   s    rV   is_cross_languagez%ActorReplicaWrapper.is_cross_language0  s    &&rU   c                     | j         s>	 t          j        | j        t                    | _         n# t
          $ r
 d | _         Y nw xY w| j        r(t          | j         t                    sJ | j         j	        S | j         S )N	namespace)
r   r   	get_actorr   r.   ro   r   
isinstancer:   handler   s    rV   actor_handlez ActorReplicaWrapper.actor_handle4  s    ! 	**%(]$& & &""  * * *%)"""* " 	-d02FGGGGG%,,!!s   %/ AAc                 ,    | j         sd S | j         j        S r   )r   bundle_specsr   s    rV   rk   z+ActorReplicaWrapper.placement_group_bundlesD  s    $ 	4$11rU   c                     | j         S )a  Replica version. This can be incorrect during state recovery.

        If the controller crashes and the deployment state is being
        recovered, this will temporarily be the deployment-wide target
        version, which may be inconsistent with the actual version
        running on the replica actor. If so, the actual version will be
        updated when the replica transitions from RECOVERING -> RUNNING
        )r   r   s    rV   r`   zActorReplicaWrapper.versionK  s     }rU   c                     | j         j        S )a  Deployment config. This can return an incorrect config during state recovery.

        If the controller hasn't yet recovered the up-to-date version
        from the running replica actor, this property will return the
        current target config for the deployment.
        )r   ri   r   s    rV   ri   z%ActorReplicaWrapper.deployment_configW  s     }..rU   c                     | j         S r   r   r   s    rV   	docs_pathzActorReplicaWrapper.docs_patha  
    rU   c                     | j         S r   r   r   s    rV   route_patternsz"ActorReplicaWrapper.route_patternse      ##rU   c                     | j         j        S r   )ri   max_ongoing_requestsr   s    rV   r   z(ActorReplicaWrapper.max_ongoing_requestsi  s    %::rU   c                     | j         j        S r   )ri   max_queued_requestsr   s    rV   r   z'ActorReplicaWrapper.max_queued_requestsm  s    %99rU   c                     | j         j        S r   )ri   graceful_shutdown_timeout_sr   s    rV   r   z/ActorReplicaWrapper.graceful_shutdown_timeout_sq  s    %AArU   c                     | j         j        S r   )ri   health_check_period_sr   s    rV   r   z)ActorReplicaWrapper.health_check_period_su  s    %;;rU   c                     | j         j        S r   )ri   health_check_timeout_sr   s    rV   r   z*ActorReplicaWrapper.health_check_timeout_sy  s    %<<rU   c                     | j         S r   )r   r   s    rV   	http_portzActorReplicaWrapper.http_port}  r   rU   c                     | j         S r   )r   r   s    rV   	grpc_portzActorReplicaWrapper.grpc_port  r   rU   c                 $    | j         j        j        S r   )ri   request_router_configrequest_routing_stats_period_sr   s    rV   r  z2ActorReplicaWrapper.request_routing_stats_period_s  s     "8W	
rU   c                 $    | j         j        j        S r   )ri   r  request_routing_stats_timeout_sr   s    rV   r  z3ActorReplicaWrapper.request_routing_stats_timeout_s  s     "8X	
rU   c                     | j         S )z2Returns the pid of the actor, None if not started.)r   r   s    rV   pidzActorReplicaWrapper.pid  s     yrU   c                     | j         S )z*Returns the actor id, None if not started.)r   r   s    rV   actor_idzActorReplicaWrapper.actor_id  s     ~rU   c                     | j         S )z+Returns the worker id, None if not started.)r   r   s    rV   	worker_idzActorReplicaWrapper.worker_id  s     rU   c                     | j         S z5Returns the node id of the actor, None if not placed.)r   r   s    rV   node_idzActorReplicaWrapper.node_id       }rU   c                     | j         S )z5Returns the node ip of the actor, None if not placed.)r   r   s    rV   node_ipzActorReplicaWrapper.node_ip  r  rU   c                     | j         S )z>Returns the node instance id of the actor, None if not placed.)r   r   s    rV   node_instance_idz$ActorReplicaWrapper.node_instance_id  s     %%rU   c                     | j         S )zDReturns the relative log file path of the actor, None if not placed.)r   r   s    rV   log_file_pathz!ActorReplicaWrapper.log_file_path  s     ""rU   c                     | j         S )zReturns the initialization latency for the replica actor.

        Returns None if the replica hasn't started yet.

        Note: this value isn't checkpointed, so if the controller restarts,
        this value goes back to None.
        )r   r   s    rV   initialization_latency_sz,ActorReplicaWrapper.initialization_latency_s  s     --rU   deployment_infoassign_rank_callbackc                    || _         |j        j        | _        |j        | _        |j        j        | _        t          
                    d| j         dddi           |j        }|j        j        t          j        k    r|j        j        t#          j        d          }n>| j        r+t#          j        t'          |j        j                            n|j        j        }| j        | j        rt#          j        |j        j                  n|j        j        ||j        j        r|j        j        nt#          j        i           |j                                        | j        |j        |j        f}n|j        j        t          j        k    rd| _        t8          j                            d	          }| j        | j                                         |j        j!        | j        r+tE          t#          j#        |j        j                            n|j        j        |j                                        | j        $                                %                                | j&        f}| j'        tP          d
tR          d}|*                    |j        j+                   |j        j,        tZ          j.        k    r|j        j,        |d<   t_          | j        || j        |||j        j0        |j        j1        |j        j2        | j3        	  	        S )zStart the current DeploymentReplica instance.

        The replica will be in the STARTING and PENDING_ALLOCATION states
        until the deployment scheduler schedules the underlying actor.
        z	Starting .log_to_stderrFextraNrT   Tz+io.ray.serve.replica.RayServeWrappedReplicadetached)r   r   lifetimeenable_task_eventsmax_concurrency)	r   	actor_defactor_resourcesactor_optionsactor_init_argsrk   rl   rm   on_scheduled)4r   rp   resource_dictr   ingressr   ri   r   r   r   r^   r   r"  deployment_languagerA   PYTHONserialized_init_argsr   r   r>   deployment_defserialized_deployment_defserialized_init_kwargsto_proto_bytesr   rn   JAVAr   r   cross_languagejava_actor_classr   r   deployment_def_namer?   loadsto_protoSerializeToStringr   r   r.   r(   updaterj   r   r   DEFAULT_MAX_CONCURRENCY_ASYNCr2   rk   rl   rm   r&  )rx   r  r  r"  r+  	init_argsr$  s          rV   startzActorReplicaWrapper.start  s"    &:" / > L'/ -? 	* 	****"E* 	 	
 	
 	

 $-	-A!() ) -BJ'2'8'<'<$$ 9MK%++:O    )7L % 5N!/"@"OPPP$3M$"1H+.EE &r**1@@BB',II  -A!&' ' '+D#*;;= I
 $..00.B 5I!%'6K    %3H1@@BB&&((::<< +I2 $("">	
 
 	_;MNNN -B9: :
  1F ! ( 1'%.F  .G  .D*
 
 
 	
rU   Nr   placement_groupc                     || _         || _        | j        r>t          | j                   | _         | j         j                                        | _        d S | j         j                                        | _        d S r   )r   r   r   r:   is_allocatedremoter   )rx   r   r;  s      rV   r&  z ActorReplicaWrapper.on_scheduled>  ss    
 * /" 	O!5d6H!I!ID&*&8&E&L&L&N&ND###&*&8&E&L&L&N&ND###rU   user_configc                     t          |          }|-| j        r&| j        rt          |          }nt	          |          }|S r   )r   r   r   r?   r>   )rx   r?  temps      rV   _format_user_configz'ActorReplicaWrapper._format_user_configL  sI    K  "t'I"& 1(..*400rU   r   c                 8   d}| j                             |          }| j        |k    }|s|r`d}t          |j                  }|                     |j                  |_        | j        j        	                    |||j
                  | _        || _         || _        |S )
        Update replica version. Also, updates the deployment config on the actor
        behind this DeploymentReplica instance if necessary.

        Returns: whether the actor is being updated.
        FT)r   requires_actor_reconfigurer   r   ri   rB  r?  r   reconfigurer>  rn   r   )rx   r`   r   updatingneeds_actor_reconfigurehas_rank_changesri   s          rV   rF  zActorReplicaWrapper.reconfigureU  s      #'-"J"J7"S"S:-" 	&6 	
 H $W%> ? ?,0,D,D!-- -) #'"4"@"G"G!$# #D  
rU   c                 h   t                               d| j         d           	 t          j        | j        t                    | _        n4# t          $ r' t           	                    d| j         d           Y dS w xY w	 t          j
                            | j                  | _        n# t          $ r
 d| _        Y nw xY w| j        j                                        | _        | j        r$| j        j                                        | _        n#| j        j                                        | _        dS )	a  Recover replica version from a live replica actor.

        When controller dies, the deployment state loses the info on the version that's
        running on each individual replica actor, so as part of the recovery process, we
        need to recover the version that is running on the replica actor.

        Also confirm that actor is allocated and initialized before marking as running.

        Returns: False if the replica actor is no longer alive; the
            actor could have been killed in the time between when the
            controller fetching all Serve actors in the cluster and when
            the controller tries to recover it. Otherwise, return True.
        zRecovering r  r   z Failed to get handle to replica z- during controller recovery. Marking as dead.FNT)r   r^   r   r   r   r   r.   r   ro   warningutilget_placement_groupr   r=  r>  r   r   check_healthr   initialize_and_get_metadatar   s    rV   recoverzActorReplicaWrapper.recoverw  sa    	4$/444555		!$ O" " "D  	 	 	NN?43C ? ? ?   55		)$'H$@$@ % %D!!  	) 	) 	)$(D!!!	)
 #'"4"A"H"H"J"J " 	"&"4"A"H"H"J"JD ">EEGG  ts#   %A -A<;A< )B* *B>=B>c           
      ~   | j         t          | j                   st          j        dfS | j        sT	 t          j        | j                   \  | _        | _        | _	        | _
        | _        | _        | _        n# t          $ rZ}t                              d| j         d           t          j        t'          |                                          fcY d}~S d}~wt*          $ rL}d| j         dt'          |           }t                              |           t          j        |fcY d}~S d}~wt,          $ rJ d| j         dt/          j                    z   }t                              |           t          j        |fcY S w xY w| j        t5          | j        j                  }|                     |j                  |_        | j        r7| j        j         !                    |"                                          | _        nV| j        j#        }| $                    | j        j%        | j
                  | _&        |!                    || j&                  | _        t          j'        dfS t          | j                  }|st          j'        dfS 	 | j        rt          j(        dfS | j)        sQt          j        | j                  \
  }| _        | _*        | _+        | _,        | _-        | _.        | _&        | _/        | _0        n# t          $ rZ}t                              d| j         d           t          j        t'          |                                          fcY d}~S d}~wt,          $ rH}t                              d| j         d           t          j        tc          |          fcY d}~S d}~ww xY wt          j(        dfS )a  
        Check if current replica has started by making ray API calls on
        relevant actor / object ref.

        Replica initialization calls __init__(), reconfigure(), and check_health().

        Returns:
            state (ReplicaStartupStatus):
                PENDING_ALLOCATION: replica is waiting for a worker to start
                PENDING_INITIALIZATION: replica initialization hasn't finished.
                FAILED: replica initialization failed.
                SUCCEEDED: replica initialization succeeded.
            error_msg:
                None: for PENDING_ALLOCATION, PENDING_INITIALIZATION or SUCCEEDED states
                str: for FAILED state
        NzException in z, the replica will be stopped.zException when allocating : z:
)2r   r;   rH   rP   r   r   getr   r   r   r   r   r   r   r   r   	exceptionr   rS   stras_instanceof_causer   	Exception	traceback
format_excr   r   r   ri   rB  r?  r   is_initializedr>  r/  rO  r   	unique_idr   rQ   rR   r   r   r   r   r   r   r   r   repr)rx   emsgri   replica_ready_check_funcreplica_ready_s          rV   check_readyzActorReplicaWrapper.check_ready  s?   & "*2L#3
 3
* (:D@@& 	88 GD344INOMM*'' Q Q Q  TD$4TTT   ,2C8M8M8O8O4P4PPPPPPPP' 8 8 8O43COOs1vvOO  %%%+2C7777777 8 8 8F1AFFF*,,-    %%%+2C77778 & !%T]%D E E,0,D,D!-- -) & &*&8&G&N&N%4466' '##
 &B ) "77$. 
 '?&E&E%tz' '# (>DD 343FGG  &	<'>DD#<* @/94?? 9 5  34460
,2 Q Q Q  TD$4TTT   ,2C8M8M8O8O4P4PPPPPPPP < < <  TD$4TTT   ,2DGG;;;;;;;	< $-t33sq   AA9 9
FACFF%AD,&F,AFFK: !AK: :
N.AMN.N.&=N)#N.)N.c                     | j         S r   )r   r   s    rV   r#  z#ActorReplicaWrapper.actor_resources"  s    $$rU   c                 (    t          j                    S r   )r   r   r   s    rV   r   z'ActorReplicaWrapper.available_resources&  s    &(((rU   c                     	 t          j        | j        t                    }| j        rt          |          }|j                                        | _        n# t          $ r Y nw xY w| j
        S )zjRequest the actor to exit gracefully.

        Returns the timeout after which to kill the actor.
        r   )r   r   r   r.   r   r:   perform_graceful_shutdownr>  r   ro   r   )rx   r   s     rV   graceful_stopz!ActorReplicaWrapper.graceful_stop*  s{    
	]4#3OOOF& 6-f55*0*J*Q*Q*S*SD'' 	 	 	D	 //s   AA 
A$#A$c                 <   	 t          j        | j        t                    }t	          | j                  }|ro	 t          j        | j                   n># t          $ r1 t          	                    dt          j                    z              Y nw xY wt          j        |d           n# t          $ r d}Y nw xY w|r+| j        $t           j                            | j                   n4# |r,| j        &t           j                            | j                   w w w xY w|S )zCheck if the actor has exited.r   z6Exception when trying to gracefully shutdown replica:
T)
no_restart)r   r   r   r.   r;   r   rS  rW  r   rT  rX  rY  killro   r   rL  remove_placement_group)rx   r   stoppeds      rV   check_stoppedz!ActorReplicaWrapper.check_stopped:  sO   	G]4#3OOOF01LMMG 	2GD78888    $$Q#.001     D1111 	 	 	GGG	  G40<//0EFFF  G40<//0EFFFFG< sL   6B( A B( 8BB( BB( 'C( (B74C( 6B77C( (1Dc                 ^   | j         t          j        }nt          | j                   r	 t	          j        | j                    t          j        }n# t          $ r t          j        }Y nt          $ r;}t                              d| j         d|            t          j        }Y d}~npd}~ww xY wt          j                    | j        z
  | j        k    r8t                              d| j         d| j         d           t          j        }nt          j        }|t          j        urd| _         |S )a}  Check the active health check (if any).

        self._health_check_ref will be reset to `None` when the active health
        check is deemed to have succeeded or failed. This method *does not*
        start a new health check, that's up to the caller.

        Returns:
            - NONE if there's no active health check, or it hasn't returned
              yet and the timeout is not up.
            - SUCCEEDED if the active health check succeeded.
            - APP_FAILURE if the active health check failed (or didn't return
              before the timeout).
            - ACTOR_CRASHED if the underlying actor crashed.
        NzHealth check for z	 failed: z1Didn't receive health check response for replica  after zs, marking it unhealthy.)r   rX   rY   r;   r   rS  rR   r   r[   r   r   rK  r   rZ   timer   r   )rx   responser]  s      rV   _check_active_health_checkz.ActorReplicaWrapper._check_active_health_checkT  s_    !)16HH'(>?? 	7
B.///5?  D D D5C B B BQ43CQQaQQRRR5AB Y[[477$:UUUNNI#I I.I I I  
 2=HH 26H5:::%)D"s   %A B-)	B-21B((B-c                     | j         dS t          j                    | j        z
  }| j        t	          j        dd          z  }||k    S )a  Determines if a new health check should be kicked off.

        A health check will be started if:
            1) There is not already an active health check.
            2) It has been more than health_check_period_s since the
               previous health check was *started*.

        This assumes that self._health_check_ref is reset to `None` when an
        active health check succeeds or fails (due to returning or timeout).
        NF?皙?)r   rp  r   r   randomuniformrx   time_since_lastrandomized_periods      rV   _should_start_new_health_checkz2ActorReplicaWrapper._should_start_new_health_check  sM     !-5 )++(DD 6S9Q9QQ!222rU   c                     | j         dS t          j                    | j        z
  }| j        t	          j        dd          z  }||k    S )a  Determines if a new record routing stats should be kicked off.

        A record routing stats will be started if:
            1) There is not already an active record routing stats.
            2) It has been more than request_routing_stats_period_s since
               the previous record routing stats was *started*.

        This assumes that self._record_routing_stats_ref is reset to `None`
        when an active record routing stats succeeds or fails (due to
        returning or timeout).
        NFrt  ru  )r   rp  r   r  rv  rw  rx  s      rV   _should_record_routing_statsz0ActorReplicaWrapper._should_record_routing_stats  sX     )55 )++(LL ?&.C
 C
 
 !222rU   c                    |                                  }|t          j        u rn|t          j        u rD| j        dk    r*t
                              | j         d| j         d           d| _        d| _        n|t          j	        u rS| xj        dz  c_        | j        t          k    r2t
                              d| j         d| j         d           d	| _        nG|t          j        u r+t
                              d
| j         d           d	| _        nJ d| d            |                                 r;t          j                    | _        | j        j                                        | _        | j        S )aS  Check if the actor is healthy.

        self._healthy should *only* be modified in this method.

        This is responsible for:
            1) Checking the outstanding health check (if any).
            2) Determining the replica health based on the health check results.
            3) Kicking off a new health check if needed.
        r   z passed the health check after z consecutive failures.TrI   Replica z failed the health check z& times in a row, marking it unhealthy.Fz
Actor for z+ crashed, marking it unhealthy immediately.zUnknown response type: r  )rr  rX   rY   rR   r   r   r^   r   r   rZ   r,   rK  r[   r{  rp  r   r   rN  r>  r   )rx   rq  s     rV   rN  z ActorReplicaWrapper.check_health  s    04/N/N/P/P16663=== 6::' W W>W W W   78D3 DMM3??? 33q8337;< < <t/ < <!D< < <  
 !&3AAANN,T- , , ,   "DMM??H?????5..00 	N+/9;;D(%)%7%D%K%K%M%MD"}rU   c                 ^   | j         nt          | j                   rf	 t          j        | j                   | _        n># t
          $ r1 t                              dt          j	                    z              Y nw xY wd| _         nVt          j
                    | j        z
  | j        k    r2t                              d| j         d| j         d           d| _         |                                 r;t          j
                    | _        | j        j                                        | _         | j        S )z&Get the routing stats for the replica.Nz,Exception when trying to get routing stats:
z2Didn't receive routing stats response for replica ro  zs, retrying.)r   r;   r   rS  r   rW  r   rT  rX  rY  rp  r   r  rK  r   r}  r   record_routing_statsr>  r   s    rV   get_routing_statsz%ActorReplicaWrapper.get_routing_stats  sY   )1'(FGG 	2&)gd.L&M&M##     C*,,-    
 .2D**IKK$>>23 3 NNF#F F7F F F  
 .2D*,,.. 	379;;D0"7>>@@ * ""s   = 8A87A8Flog_shutdown_messagec                     | j         r-t          r&|r"t                              | j         d           dS 	 t          j        t          j        | j        t                               dS # t          $ r Y dS w xY w)z9Force the actor to exit without shutting down gracefully.z did not shut down because it had not finished draining requests. Going to wait until the draining is complete. You can force-stop the replica by setting RAY_SERVE_DISABLE_SHUTTING_DOWN_INGRESS_REPLICAS_FORCEFULLY to 0.Nr   )r   r   r   r^   r   r   rj  r   r   r.   ro   )rx   r  s     rV   
force_stopzActorReplicaWrapper.force_stop  s     M
	K
	 $  ` ` `  
 F	HS]4#3OOOPPPPP 	 	 	DD	s   2A* *
A87A8c                     | j         S r   )r   r   s    rV   get_outbound_deploymentsz,ActorReplicaWrapper.get_outbound_deployments!  s    ))rU   r   F)IrM   rN   rO   rz   r!   r@   r   propertyrU  r   r   r   rD   r   r   r}   r   r   r   r
   r	   floatrk   r`   r%   ri   r   r   r|   r   r   r   r   r   r   r   r  r  r  r  r
  r  r  r  r  r  r/   r   r2   r:  rF   r&  r   rB  rF  rP  r   rH   rb  r#  r   r    rg  rm  rX   rr  r{  r}  rN  r  r  r   r  rT   rU   rV   r   r      sm        :H:H #:H :H :H :Hx  C       X  ( ( ( ( X( h{+    X ,# , , , X, '4 ' ' ' X' "h{3 " " " X" 2$tCJ7G2H)I 2 2 2 X2 	* 	 	 	 X	 /#3 / / / X/ 8C=    X $c 3 $ $ $ X$ ;c ; ; ; X; :S : : : X: BU B B B XB <u < < < X< = = = = X= 8C=    X 8C=    X 
 
 
 
 X

 
 
 
 
 X

 Xc]    X (3-    X 8C=    X #    X #    X &(3- & & & X& #x} # # # X# 	.(5/ 	. 	. 	. X	.|
'|
 '	{K'?@|
 
"	|
 |
 |
 |
B 59O O!O ".1O O O Os     #4  K  D        D. . . . .`y4U#7##FG y4 y4 y4 y4v %$sEz*:!; % % % X% )T#u*%5 ) ) ) X)0x 0 0 0 0 t    4.,F . . . .`3 3 3 3 3.3d 3 3 3 344d 4 4 4 4l!#4S> !# !# !# !#F t    &*(43E*F * * * * * *rU   r   c                      e Zd ZdZdedefdZdedefdZ	de
e         fd	Zd
eeeef                  fdZede
e         fd            Zedeeef         fd            Zedefd            Zedefd            Zedefd            Zedefd            Zed             Zedee         fd            Zedee
e                  fd            Zedefd            Zedefd            Zedee         fd            Zedee          fd            Z!edee          fd            Z"edee          fd            Z#edee$         fd            Z%de&de'ege(f         de)fdZ*dede(de+fd Z,de+fd!Z-edee(         fd"            Z.de/e0ee         ee$         f         fd#Z1d0d%e+dd&fd'Z2de+fd(Z3de+fd)Z4deeeef                  fd*Z5d+e6dd&fd,Z7d1d-Z8de/eef         fd.Z9dee
e:                  fd/Z;d&S )2DeploymentReplicazzManages state transitions for deployment replicas.

    This is basically a checkpointable lightweight state machine.
    r   r`   c                     || _         t          ||          | _        d | _        t	          |                                | j         j        t          j        d          | _	        g | _
        i | _        d| _        d S )Nr   )
actor_namer   statestart_time_sF)r   r   _actor_start_timerC   r   r[  r"   STARTING_actor_details_multiplexed_model_idsr   _logged_shutdown_messager   s      rV   r   zDeploymentReplica.__init__+  s}    
 &)*g>>,!0022'1'	
 
 
 24#.0(-%%%rU   cluster_node_info_cacherb   c                     t          | j        | j        | j        j        |                    | j                  | j        j        | j        j        | j        j        | j	        | j
        | j        j        
  
        S )N)
r   r  r  availability_zoner  r   r   multiplexed_model_idsrouting_statsport)r$   r   actor_node_idr  r  get_node_azr   r   r   r  r  r   )rx   r  s     rV   get_running_replica_infoz*DeploymentReplica.get_running_replica_info=  so     "'&K'5AA$BTUU{.!%!A"k;"&"<,0
 
 
 	
rU   r  c                     || _         dS )z2Record the multiplexed model ids for this replica.Nr  )rx   r  s     rV   record_multiplexed_model_idsz.DeploymentReplica.record_multiplexed_model_idsM  s    &;###rU   r  c                     |	|| _         dS dS )zRecord the routing stats for this replica.

        Recording routing_stats as an empty dictionary is valid. But skip
        update if the routing_stats is None.
        Nr   )rx   r  s     rV   r  z&DeploymentReplica.record_routing_statsQ  s      $"/D %$rU   c                     | j         S r   r  r   s    rV   r  z'DeploymentReplica.multiplexed_model_idsZ  s    **rU   c                     | j         S r   r  r   s    rV   r  zDeploymentReplica.routing_stats^      ""rU   c                     | j         S r   )r  r   s    rV   actor_detailszDeploymentReplica.actor_detailsb  r  rU   c                     | j         S r   r   r   s    rV   r   zDeploymentReplica.replica_idf  r   rU   c                 $    | j         j        j        S r   )r   r   r   r   s    rV   r   z!DeploymentReplica.deployment_namej  s    -22rU   c                 $    | j         j        j        S r   )r   r   r   r   s    rV   r   zDeploymentReplica.app_namen  s    -66rU   c                     | j         j        S r   )r  r`   r   s    rV   r`   zDeploymentReplica.versionr  s    {""rU   c                     | j         j        S r   )r  r   r   s    rV   r   zDeploymentReplica.docs_pathv      {$$rU   c                     | j         j        S r   )r  r   r   s    rV   r   z DeploymentReplica.route_patternsz  s    {))rU   c                     | j         j        S r   )r  r  r   s    rV   r  zDeploymentReplica.actor_id~  s    {##rU   c                     | j         j        S r   )r  r   r   s    rV   r   zDeploymentReplica.actor_handle  s    {''rU   c                     | j         j        S r  )r  r  r   s    rV   r  zDeploymentReplica.actor_node_id  s     {""rU   c                     | j         j        S r   )r  r   r   s    rV   actor_http_portz!DeploymentReplica.actor_http_port  r  rU   c                     | j         j        S r   )r  r   r   s    rV   actor_grpc_portz!DeploymentReplica.actor_grpc_port  r  rU   c                     | j         j        S r  )r  r  r   s    rV   	actor_pidzDeploymentReplica.actor_pid  s     {rU   c                     | j         j        S )z0Returns how long the replica took to initialize.)r  r  r   s    rV   r  z*DeploymentReplica.initialization_latency_s  s     {33rU   r  r  c                     | j                             ||          }t          j                    | _        d| _        |                     | j                   |S )zK
        Start a new actor for current DeploymentReplica instance.
        r  Fr  )r  r:  rp  r  r  update_actor_details)rx   r  r  replica_scheduling_requests       rV   r:  zDeploymentReplica.start  s_     &*[%6%62F &7 &
 &
"  9;;(-%!!t/?!@@@))rU   r   c                 :    | j                             ||          S )rD  r   )r  rF  )rx   r`   r   s      rV   rF  zDeploymentReplica.reconfigure  s     {&&wT&:::rU   c                     | j                                         sdS t          j                    | _        |                     | j                   dS )z
        Recover states in DeploymentReplica instance by fetching running actor
        status

        Returns: False if the replica is no longer alive at the time
            when this method is called.
        Fr  T)r  rP  rp  r  r  r   s    rV   rP  zDeploymentReplica.recover  sL     {""$$ 	59;;!!t/?!@@@trU   c                     | j         j        S )z%Get the rank assigned to the replica.)r  r   r   s    rV   r   zDeploymentReplica.rank  s     {rU   c           	          | j                                         }|                     | j         j        | j         j        | j         j        | j         j        | j         j        | j         j        | j         j	                   |S )zCheck if the replica has started. If so, transition to RUNNING.

        Should handle the case where the replica has already stopped.

        Returns:
            status: Most recent state of replica by
                querying actor obj ref
        )r  r  r  r  r  r
  r  )
r  rb  r  r  r  r  r  r  r
  r  )rx   is_readys     rV   check_startedzDeploymentReplica.check_started  sr     ;**,,!!K'K'![9[)k++3 	" 	
 	
 	
 rU   TgracefulNc                     | j         j        }t                              d| j         d| dddi           | j                                        }|sd}t          j                    |z   | _        dS )	z`Stop the replica.

        Should handle the case where the replica is already stopped.
        	Stopping z (currently ).r  Fr  r   N)	r  r  r   r^   r   r  rg  rp  _shutdown_deadline)rx   r  r  	timeout_ss       rV   stopzDeploymentReplica.stop  s    
 #)>>>U>>>"E* 	 	
 	
 	
 K--//	 	I"&)++	"9rU   c                 (   | j                                         rdS t          j                    | j        k    }|rX| j        s)t
          s"t                              | j         d           | j         	                    | j                    d| _        dS )z+Check if the replica has finished stopping.Tz9 did not shut down after grace period, force-killing it. )r  F)
r  rm  rp  r  r  r   r   r^   r   r  )rx   timeout_passeds     rV   rm  zDeploymentReplica.check_stopped  s    ;$$&& 	4(?? 	11S  1 1 1  
 K"")-)F%F #    -1D)urU   c                 4    | j                                         S )zjCheck if the replica is healthy.

        Returns `True` if the replica is healthy, else `False`.
        )r  rN  r   s    rV   rN  zDeploymentReplica.check_health  s    
 {'')))rU   c                 4    | j                                         S )zGet the latest response from the routing stats on the replica.

        Returns None if the replica is still calculating the stats.
        )r  r  r   s    rV   pull_routing_statsz$DeploymentReplica.pull_routing_stats  s    
 {,,...rU   r  c                 2    |                      |           dS )zUpdates state in actor details.)r  N)r  )rx   r  s     rV   update_statezDeploymentReplica.update_state  s    !!!.....rU   c                     | j                                         }|                    |           t          di || _         d S )NrT   )r  rw   r7  rC   )rx   kwargsdetails_kwargss      rV   r  z&DeploymentReplica.update_actor_details  sF    ,1133f%%%,>>~>>rU   c                 F   | j         j        dS | j         j        | j         j        n(d | j         j                                        D             fd| j         j                                        D             }t          j                  t          j        |          fS )a  Returns required and currently available resources.

        Only resources with nonzero requirements will be included in the
        required dict and only resources in the required dict will be
        included in the available dict (filtered for relevance).
        N)UNKNOWNr  c                 *    i | ]\  }}||dk    ||S Nr   rT   ).0kvs      rV   
<dictcomp>z;DeploymentReplica.resource_requirements.<locals>.<dictcomp>1  s3       Aq=QUU 1%*UUrU   c                 $    i | ]\  }}|v 	||S rT   rT   )r  r  r  requireds      rV   r  z;DeploymentReplica.resource_requirements.<locals>.<dictcomp>7  s+     
 
 
QXAqrU   )r  r#  rk   itemsr   r   r   )rx   	availabler  s     @rV   resource_requirementsz'DeploymentReplica.resource_requirements$  s     ;&.'';.:{:HH  K7==??  H
 
 
 
![<BBDD
 
 
	 z(##TZ	%:%:::rU   c                 4    | j                                         S r   )r  r  r   s    rV   r  z*DeploymentReplica.get_outbound_deployments@  s    {33555rU   Trb   N)<rM   rN   rO   rz   r!   r@   r   r   r$   r  r
   rU  r  r   r	   r   r  r  r  r  rC   r  r   r   r   r`   r   r   r  r   r   r  r|   r  r  r  r  r  r/   r   rD   r2   r:  r}   rF  rP  r   r   rH   r  r  rm  rN  r  r"   r  r  r  r   r  rT   rU   rV   r  r  %  s'        
.. #. . . .$
';
	
 
 
 
 <$s) < < < <0(4S>2J 0 0 0 0 +tCy + + + X+ #tCH~ # # # X# #~ # # # X#  I       X  3 3 3 3 X3 7# 7 7 7 X7 # # X# %8C= % % % X% *c 3 * * * X* $# $ $ $ X$ (k ( ( ( X( #x} # # # X# %# % % % X% %# % % % X% 8C=    X 4(5/ 4 4 4 X4
*'* '	{K'?@* 
"	* * * * ;"; ; 
	; ; ; ;       h{+       X 	#Xc]HUOC	D   0: :T :T : : : :t    ,*d * * * */HT#s(^$< / / / //, /4 / / / /? ? ? ?
;uS#X ; ; ; ;86(43E*F 6 6 6 6 6 6rU   r  c            
          e Zd ZdZd ZdedefdZ	 ddee	e                  de	e         fd	Z
ddej        fd
ee         dee	e                  dee         de	e         fdZ	 	 	 dd
ee         dee         dee	e                  fdZd Zd ZdS )ReplicaStateContainerzCContainer for mapping ReplicaStates to lists of DeploymentReplicas.c                 8    t          t                    | _        d S r   )r   list	_replicasr   s    rV   r   zReplicaStateContainer.__init__G  s    FQRVFWFWrU   r  replicac                     t          |t                    sJ dt          |                       |                    |           | j        |                             |           dS )zAdd the provided replica under the provided state.

        Args:
            state: state to add the replica under.
            replica: replica to add.
        zType: N)r   r"   typer  r  append)rx   r  r  s      rV   addzReplicaStateContainer.addJ  sf     %..FF0Fe0F0FFF.U###u$$W-----rU   Nstatesrb   c                 |     |t           }t          |t                    sJ t           fd|D             g           S )a  Get all replicas of the given states.

        This does not remove them from the container. Replicas are returned
        in order of state as passed in.

        Args:
            states: states to consider. If not specified, all replicas
                are considered.
        Nc              3   2   K   | ]}j         |         V  d S r   )r  r  r  rx   s     rV   	<genexpr>z,ReplicaStateContainer.get.<locals>.<genexpr>f  s*      >>eDN5)>>>>>>rU   )ALL_REPLICA_STATESr   r  sum)rx   r  s   ` rV   rS  zReplicaStateContainer.getU  sJ     >'F&$'''''>>>>v>>>CCCrU   exclude_versionmax_replicasc                    |t           }|t          |t                    sJ t          |t                    sJ g }|D ]}g }g }| j        |         D ]s}t          |          t          |          z   |k    r|                    |           ;|!|j        |k    r|                    |           ^|                    |           t|| j        |<   |                    |           |S )a"  Get and remove all replicas of the given states.

        This removes the replicas from the container. Replicas are returned
        in order of state as passed in.

        Args:
            exclude_version: if specified, replicas of the
                provided version will *not* be removed.
            states: states to consider. If not specified, all replicas
                are considered.
            max_replicas: max number of replicas to return. If not
                specified, will pop all replicas matching the criteria.
        )	r  r   r@   r  r  lenr  r`   extend)	rx   r  r  r  replicasr  popped	remainingr  s	            rV   popzReplicaStateContainer.poph  s   & >'F&*_FW*X*X&&X&$''''' 	$ 	$EFI>%0 + +x==3v;;.,>>$$W----$0W_5W5W$$W----MM'****$-DN5!OOF####rU   r`   c                     |t           }t          |t                    sJ t          t                    sJ t          t                    sJ t	           fd|D                       S t	           fd|D                       S t	           fd|D                       S t          d          )a  Get the total count of replicas of the given states.

        Args:
            exclude_version: version to exclude. If not
                specified, all versions are considered.
            version: version to filter to. If not specified,
                all versions are considered.
            states: states to consider. If not specified, all replicas
                are considered.
        Nc              3   L   K   | ]}t          j        |                   V  d S r   )r  r  r  s     rV   r  z.ReplicaStateContainer.count.<locals>.<genexpr>  s2      FFes4>%011FFFFFFrU   c           
   3      K   | ]<}t          t          t          fd j        |                                       V  =dS )c                     | j         k    S r   r`   )rr`   s    rV   <lambda>z7ReplicaStateContainer.count.<locals>.<genexpr>.<lambda>  s    !)w*> rU   Nr  r  filterr  )r  r  rx   r`   s     rV   r  z.ReplicaStateContainer.count.<locals>.<genexpr>  sb         D > > > >u@UVVWWXX     rU   c           
   3      K   | ]<}t          t          t          fd j        |                                       V  =dS )c                     | j         k    S r   r  )r  r  s    rV   r  z7ReplicaStateContainer.count.<locals>.<genexpr>.<lambda>  s    ai?&B rU   Nr	  )r  r  r  rx   s     rV   r  z.ReplicaStateContainer.count.<locals>.<genexpr>  ss       
 
  BBBB N51   
 
 
 
 
 
rU   z;Only one of `version` or `exclude_version` may be provided.)r  r   r  r@   r  ro   )rx   r  r`   r  s   ``` rV   countzReplicaStateContainer.count  s3     >'F&$'''''&*_FW*X*X&&X*W6G"H"HH"wFFFFvFFFFFF$)<     #      (W_ 
 
 
 
 
 $
 
 
 
 
 
 M  rU   c                 *    t          | j                  S r   )rU  r  r   s    rV   __str__zReplicaStateContainer.__str__  s    4>"""rU   c                 *    t          | j                  S r   )r\  r  r   s    rV   __repr__zReplicaStateContainer.__repr__  s    DN###rU   r   )NNN)rM   rN   rO   rz   r   r"   r  r  r   r
   rS  mathinfr@   r|   r  r  r  r  rT   rU   rV   r  r  D  sp       MMX X X	. 	.0A 	. 	. 	. 	. 6:D DtL12D		 D D D D* 8</3&*h	) )!"34) l+,) sm	)
 
	 ) ) ) )Z 8</3/3	+ +!"34+ +,+ l+,	+ + + +Z# # #$ $ $ $ $rU   r  c                       e Zd ZdZd ZdedefdZdeddfdZdededdfd	Z	dedefd
Z
dedefdZdeeef         fdZddZdee         dee         fdZdee         dee         fdZdS )RankManagerz Manages ranks for a single node.c                 H    i | _         t                      | _        d| _        d S r  )_ranksset_released_ranks
_next_rankr   s    rV   r   zRankManager.__init__  s     &(), rU   keyrb   c                    || j         v r t          d| d| j         |                    | j        r/t          | j                  }| j                            |           n| j        }| xj        dz  c_        || j         |<   |S N	Rank for  already assigned: rI   )r  RuntimeErrorr  minremover  rx   r  r   s      rV   assign_rankzRankManager.assign_rank  s    $+U3UU4;sCSUUVVV 	!t+,,D ''---- ?DOOq OOCrU   Nc                     || j         vrt          d| d          | j                             |          }| j                            |           d S Nr   not assigned)r  r   r  r  r  r#  s      rV   release_rankzRankManager.release_rank  s[    dk!!=3===>>>{s## 	  &&&&&rU   r   c                     || j         v r t          d| d| j         |                    || j         |<   | j                            |           || j        k    r|dz   | _        d S d S r  )r  r   r  discardr  r#  s      rV   recover_rankzRankManager.recover_rank  s|    $+U3UU4;sCSUUVVVC$$T***4?"""QhDOOO #"rU   c                 T    || j         vrt          d| d          | j         |         S r&  )r  r   rx   r  s     rV   get_rankzRankManager.get_rank  s6    dk!!=3===>>>{3rU   c                     || j         v S r   )r  r-  s     rV   has_rankzRankManager.has_rank  s    dk!!rU   c                 4    | j                                         S r   )r  r   r   s    rV   get_ranks_mappingzRankManager.get_ranks_mapping  s    {!!!rU   c                 x    | j                                          | j                                         d| _        d S r  )r  clearr  r  r   s    rV   r4  zRankManager.clear  s6    ""$$$rU   active_keysc                    |sg S t          |          }t          | j                                                  |z
  }|r-t                              d| d           t          d          |t          | j                                                  z
  }|r-t                              d| d           t          d          i }| j                                                                        D ]^\  }}||v rU|                    |d          dz   ||<   ||         dk    r-t                              d| d           t          d          _t          | j        
                                          }t          t          t          |                              }	g }
||	k    r6t                              d	| d
|	 d           |                     |          }
|
S )a  Verify rank system invariants and reassign ranks when needed.

        This method ensures:
        1. All active keys have ranks
        2. No duplicate ranks exist
        3. Ranks are contiguous when at target count

        Args:
            active_keys: List of currently active keys

        Returns:
            List of keys that need to be reconfigured with new ranks

        Raises:
            RuntimeError: If rank system invariants are violated and fail_on_error=True
        zFound stale ranks for keys: z8. This should never happen. Please report this as a bug.z#Rank system is in an invalid state.z!Found active keys without ranks: r   rI   zFound duplicate rank zR assigned to multiple keys. This should never happen. Please report this as a bug.z7At target count but ranks are not contiguous. Current: z, Expected: z". Performing minimal reassignment.)r  r  keysr   r   r   r   r  rS  sortedvaluesr  ranger  debug"_perform_minimal_rank_reassignment)rx   r5  active_keys_set
stale_keysunranked_keysrank_countsr  r   current_ranksexpected_ranks.keys_needing_reconfiguration_from_reassignments              rV   -check_rank_consistency_and_reassign_minimallyz9RankManager.check_rank_consistency_and_reassign_minimally   sK   (  	Ik** ))++,,>
 	FLLIz I I I   DEEE (#dk.>.>.@.@*A*AA 	FLLIM I I I   DEEE ))++1133 	N 	NICo%%$/OOD!$<$<q$@D!t$q((LLQ Q Q Q   ''LMMM t{113344eC$4$455669;6N**LL3)3 37E3 3 3   77DD ; >=rU   c           	         t          t          t          |                              }g }g }|D ][}|                     |          }||v r+|                    |           |                    |           F|                    |           \t          |          }t          |          D ]{\  }}||         }	| j        |         }
t          
                    d| d|
 d|	            |	| j        |<   | j                            |	           | j                            |
           |t          
                    dt          |           dt          |           d           |S )a  Perform minimal rank reassignment to achieve contiguity.

        This method reassigns ranks while minimizing the number of keys that need
        to be reconfigured. It prioritizes keeping existing ranks when possible.

        Args:
            active_keys: List of currently active keys

        Returns:
            List of keys that need to be reconfigured with new ranks
        zReassigning key z: rank z -> zMinimal reassignment complete: z keys kept ranks, z keys reassigned)r  r:  r  r.  r"  r  r8  	enumerater  r   r;  r  r*  r  )rx   r5  target_ranks_setkeys_needing_rankskeys_keeping_ranksr  current_rankavailable_ranksinew_rankold_ranks              rV   r<  z.RankManager._perform_minimal_rank_reassignmentJ  s    uS%5%56677   		/ 		/C==--L/// ''555"))#.... #))#.... !!122   233 	/ 	/FAs&q)H {3'HLLPCPPPPhPPQQQ  (DK ((222 $$X.... 	9c2D.E.E 9 9%&&9 9 9	
 	
 	

 "!rU   r  )rM   rN   rO   rz   r   rU  r|   r$  r(  r+  r.  r}   r0  r	   r2  r4  r
   rD  r<  rT   rU   rV   r  r    sq       **! ! !
s s    "' ' ' ' ' '' '3 '4 ' ' ' ' C  C        
"C "D " " " ""4S> " " " "   
H>#YH> 
cH> H> H> H>T6"d3i 6"DQTI 6" 6" 6" 6" 6" 6"rU   r  c                       e Zd ZdZddefdZd Zdededefd	Z	dedd
fdZ
dedededd
fdZdedefdZdedefdZded         ded         fdZddZdeeef         fdZd
S )DeploymentRankManagera  Manages replica ranks for a deployment.
    This class handles rank assignment, release, consistency checking, and reassignment.
    It maintains the rank system invariants and provides a clean interface for rank operations.

    Maintains three levels of rank tracking:
    - Global rank: Replica-level rank across all nodes (0, 1, 2, ...)
    - Local rank: Replica's rank within its node (0, 1, 2, ... per node)
    - Node rank ID: Index assigned to each node (0, 1, 2, ...)
    Tfail_on_rank_errorc                 |    t                      | _        || _        t                      | _        i | _        i | _        d S r   )r  _replica_rank_manager_fail_on_rank_error_node_rank_manager_local_rank_managers_replica_to_node)rx   rQ  s     rV   r   zDeploymentRankManager.__init__  s>    %0]]"#5  #.-- =?! 13rU   c                     | j         r ||i |S 	  ||i |S # t          $ r1}t                              d|j         d|            |cY d }~S d }~ww xY w)NzError executing function rR  )rT  rW  r   r   rM   )rx   funcsafe_defaultargsr  r]  s         rV   _execute_with_error_handlingz2DeploymentRankManager._execute_with_error_handling  s    # 		$4(((($tT,V,,, $ $ $MMM!MMNNN#######$s    
A&A	AAr   r  rb   c                 b      fd}                      |t          ddd                    S )a9  Assign a rank to a new replica.

        Args:
            replica_id: The unique ID of the replica
            node_id: The unique ID of the node

        Returns:
            ReplicaRank object with the assigned rank

        Raises:
            RuntimeError: If the replica already has a rank assigned
        c                                                    r-t          d dj                                                 j        <   j                                      } j        vr0j                                       t                      j        <   j                                      }j                                               }t          | ||          S )Nr  r  r   	node_rank
local_rank)
has_replica_rankr   rS  r.  rW  r$  rV  rU  r  rD   )r   r`  ra  r  r   rx   s      rV   _assign_rank_implz<DeploymentRankManager.assign_rank.<locals>._assign_rank_impl  s    $$Z00 "p
ppt?Y?b?bcm?n?npp  
 18D!*- -99*EED d777'33G<<<5@]])'2/88AAI27;GG
SSJDI*UUUUrU   r   r_  r\  rD   )rx   r   r  rc  s   ``` rV   r$  z!DeploymentRankManager.assign_rank  s]    	V 	V 	V 	V 	V 	V 	V. 00{Q1MMM
 
 	
rU   Nc                 >      fd}                      |d          S )zRelease rank for a replica.

        Args:
            replica_id: ID of the replica

        Raises:
            RuntimeError: If replica doesn't have ranks
        c                                                    st          d d          j                 } j                                       j        |                                         t          j        |                                                    dk    r"j                            |            j        | = j        = d S )Nr  r'  r   )	rb  r   rW  rS  r(  rV  r  r2  rU  )r  r   rx   s    rV   _release_rank_implz>DeploymentRankManager.release_rank.<locals>._release_rank_impl  s    ((44 J"#Hz#H#H#HIII +J7G &33J??? %g.;;JGGG 4,W5GGIIJJaOO'44W===-g6 %j111rU   Nr\  )rx   r   rg  s   `` rV   r(  z"DeploymentRankManager.release_rank  s;    	2 	2 	2 	2 	2 	2* 001CTJJJrU   r   c                 F      fd}                      |d          S )a  Recover rank for a replica (e.g., after controller restart).

        Args:
            replica_id: ID of the replica
            node_id: ID of the node
            rank: The rank to recover

        Raises:
            RuntimeError: If replica already has ranks assigned
        c                                                    r-t          d dj                                                 j                            j                   j                                       s j                             j                    j	        vrt                      j	         <   j	                                      j                    j        <   d S )Nr  r  )rb  r   rS  r.  r+  r   rU  r0  r`  rV  r  ra  rW  )r  r   r   rx   s   rV   _recover_rank_implz>DeploymentRankManager.recover_rank.<locals>._recover_rank_impl  s    $$Z00 "p
ppt?Y?b?bcm?n?npp  
 &33J	JJJ *33G<< N'44WdnMMM d7775@]])'2%g.;;JXXX 18D!*---rU   Nrh  )rx   r   r  r   rk  s   ```` rV   r+  z"DeploymentRankManager.recover_rank  sG    "	8 	8 	8 	8 	8 	8 	8 	8* 001CTJJJrU   c                     || j         vrdS | j         |         }| j                            |          oB|| j        v o9| j                            |          o| j        |                             |          S )a  Check if replica has a rank assigned.

        Args:
            replica_id: The unique ID of the replica

        Returns:
            True if the replica has a rank assigned, False otherwise

        Raises:
            RuntimeError: If the replica doesn't have ranks assigned
        F)rW  rS  r0  rV  rU  )rx   r   r  s      rV   rb  z&DeploymentRankManager.has_replica_rank  s     T2225'
3&//
;; H444H'0099H )'2;;JGG		
rU   c                 ^      fd}                      |t          ddd                    S )zGet the rank for a replica.

        Args:
            replica_id: ID of the replica

        Returns:
            ReplicaRank object

        Raises:
            RuntimeError: If replica doesn't have ranks assigned
        c                  :                                  st          d d          j                                      } j                 }j                            |          }j        |                                       }t          | ||          S )Nr  r'  r_  )rb  r   rS  r.  rW  rU  rV  rD   )global_rankr  r`  ra  r   rx   s       rV   _get_replica_rank_implzFDeploymentRankManager.get_replica_rank.<locals>._get_replica_rank_impl>  s    ((44 J"#Hz#H#H#HIII4==jIIK+J7G/88AAI27;DDZPPJ I*   rU   r   r_  rd  )rx   r   rp  s   `` rV   get_replica_rankz&DeploymentRankManager.get_replica_rank1  sQ    
	 
	 
	 
	 
	 
	 00"KQ!PQ$R$R$R
 
 	
rU   active_replicasr  c                 >      fd}                      |g           S )a  Verify rank system invariants and reassign ranks when needed across all three levels.

        This method ensures:
        1. Global ranks are contiguous [0, N-1] for N replicas
        2. Node ranks are contiguous [0, M-1] for M nodes
        3. Local ranks are contiguous [0, K-1] for K replicas on each node

        Args:
            active_replicas: List of currently active replicas

        Returns:
            List of replicas that need to be reconfigured with new ranks
        c                     sg S d D             } d D             t                      }j                            |           }|                    |           i }| D ]P}j                            |          }|J d| d            ||vrg ||<   ||                             |           Q|                                D ]:\  }}j        |                             |          }|                    |           ;t          |
                                          }|r:j                            |          }	|	D ]}|                    ||                    fd|D             }
|
S )Nc                 &    g | ]}|j         j        S rT   r   r[  r  r  s     rV   
<listcomp>z}DeploymentRankManager.check_rank_consistency_and_reassign_minimally.<locals>._check_rank_consistency_impl.<locals>.<listcomp>e  s+     " " "18"," " "rU   c                 (    i | ]}|j         j        |S rT   rv  rw  s     rV   r  z}DeploymentRankManager.check_rank_consistency_and_reassign_minimally.<locals>._check_rank_consistency_impl.<locals>.<dictcomp>j  s-     % % %:A",g% % %rU   r  z not assigned to any nodec                 (    g | ]}|v |         S rT   rT   )r  r   replica_id_to_replicas     rV   rx  z}DeploymentRankManager.check_rank_consistency_and_reassign_minimally.<locals>._check_rank_consistency_impl.<locals>.<listcomp>  s4     0 0 0!666 &j1666rU   )r  rS  rD  r7  rW  rS  r  r  rV  r  r7  rU  )active_replica_ids'all_replica_ids_needing_reconfigurationreplica_ids_from_globalreplicas_by_noder   r  replica_ids_on_nodereplica_ids_from_localactive_node_idsnode_ids_needing_reassignment replicas_needing_reconfigurationr{  rr  rx   s              @rV   _check_rank_consistency_implziDeploymentRankManager.check_rank_consistency_and_reassign_minimally.<locals>._check_rank_consistency_impl`  s   " 	" "<K" " "
% %ET% % %!
 7:ee3 '+&@&n&n"' '# 4::;RSSS 680 = =
/33J??''CjCCC (''"22202$W- )00<<<<0@0F0F0H0H W W,,)-)B*??@STT ' 8>>?UVVVV ##3#8#8#:#:;;O 040G0u0u#1 1-  =  G;BB(1   0 0 0 0"I0 0 0, 43rU   rh  )rx   rr  r  s   `` rV   rD  zCDeploymentRankManager.check_rank_consistency_and_reassign_minimallyN  s<    $<	4 <	4 <	4 <	4 <	4 <	4| 001MrRRRrU   c                     | j                                          | j                                         | j                                         | j                                         d S r   )rS  r4  rU  rV  rW  r   s    rV   r4  zDeploymentRankManager.clear  s\    "((***%%'''!'')))##%%%%%rU   c                     i }| j                                                                         D ]}|                     |          ||<   |S )zGet the current mapping of replica IDs to ReplicaRank objects.

        Returns:
            Dict mapping replica_id to ReplicaRank object
        )rS  r2  r7  rq  )rx   resultr   s      rV   get_replica_ranks_mappingz/DeploymentRankManager.get_replica_ranks_mapping  sU     4FFHHMMOO 	C 	CJ!%!6!6z!B!BF:rU   r  r  )rM   rN   rO   rz   r}   r   r\  rU  rD   r$  r(  r+  rb  rq  r
   rD  r4  r	   r  rT   rU   rV   rP  rP    s        3 34 3 3 3 3
$ 
$ 
$'
c '
C '
K '
 '
 '
 '
RKs Kt K K K KB&K&K &K 	&K
 
&K &K &K &KP
3 
4 
 
 
 
.
3 
; 
 
 
 
:PS12PS 
!	"PS PS PS PSd& & & &	4[0@+A 	 	 	 	 	 	rU   rP  c                   X   e Zd ZdZeZdZdedede	de
def
dZd	efd
Zd	efdZdefdZdee         fdZed	efd            Zed	efd            Zed	efd            Zed	efd            Zed	efd            Zed	efd            Zed	e e         fd            Z!ed	e ee                  fd            Z"ed	efd            Z#d	efdZ$d	efdZ%d	e&e         fdZ'd	ee(         fdZ)d	ee*         fdZ+dLded	efd Z,d	e&e         fd!Z-d	ee.         fd"Z/dMd#Z0dMd$Z1dMd%Z2	 dNd&ed'ed(ed	dfd)Z3d*ed	efd+Z4d,ed	efd-Z5d	efd.Z6d'ed	dfd/Z7e8j9        fd	efd0Z:d	efd1Z;d	e<ee=         e>f         fd2Z?d	e<eef         fd3Z@	 dNd4eAd	ee<eBeCf                  fd5ZDd6efd7ZEdMd8ZFdOd:eBfd;ZGd< ZHd=ed>         fd?ZId	eJeeKf         fd@ZLdAeeB         dBeJeef         dCed	e<eeB         eeB         f         fdDZMdEeJeef         fdFZNdGeOd	dfdHZPdI ZQd	efdJZRd	e ee                  fdKZSdS )PDeploymentStatez>Manages the target state and replicas for a single deployment.Fidlong_poll_hostdeployment_schedulerr  autoscaling_state_managerc                 v   || _         || _        || _        || _        || _        t
                                          | _        t          j                    | _	        d | _
        d| _        d| _        t                      | _        t          | j         j        t"          j        t&          j                  | _        t-          t.                    | _        i | _        t5          j        ddd          | _        t5          j        dd	d
          | _        d| _        g | _        d| _         d | _!        d | _"        d | _#        d S )Nr   F)rQ   serve_deployment_replica_healthyzVTracks whether this deployment replica is healthy. 1 means healthy, 0 means unhealthy.
deploymentr  application)descriptiontag_keys!serve_autoscaling_target_replicaszhThe target number of replicas for this deployment. This is the number the autoscaler is trying to reach.r  r  T)$_id_long_poll_host_deployment_scheduler_cluster_node_info_cache_autoscaling_state_managerr]   rf   _target_staterp  _prev_startup_warning_replica_constructor_error_msg"_replica_constructor_retry_counter_replica_has_startedr  r  r   r   r   UPDATINGr   CONFIG_UPDATE_STARTED_curr_status_inforP  r)   _rank_manager replica_average_ongoing_requestsr   Gaugehealth_check_gaugetarget_replicas_gauge_request_routing_info_updated'_last_broadcasted_running_replica_infos_last_broadcasted_availability#_last_broadcasted_deployment_configr   r   )rx   r  r  r  r  r  s         rV   r   zDeploymentState.__init__  sL    -;%9"(?%*C' 5J4Q4Q4S4S,0IKK"=A+ 89/ +0!0E0G0G7KHM%#98
 8
 3;
 
 
 CE-")-.. >#
 #
 #
 &-]/H 3&
 &
 &
" .3*QS448+370)-48rU   rb   c                 @    | j                             | j                  S )z>
        Check if the deployment is under autoscaling
        )r  should_autoscale_deploymentr  r   s    rV   should_autoscalez DeploymentState.should_autoscale  s     .JJ48TTTrU   c                     | j         S )z
        Return deployment's target state submitted by user's deployment call.
        Should be persisted and outlive current ray cluster.
        )r  r   s    rV   get_checkpoint_dataz#DeploymentState.get_checkpoint_data  s    
 !!rU   target_state_checkpointc                 R   t                               d| j         d           || _        | j                            | j        | j        j        j                   | j        j        j        j        r7| j	        
                    | j        | j        j        | j        j                   d S d S )NzRecovering target state for z from checkpoint.)r   r^   r  r  r  on_deployment_deployedrp   ri   autoscaling_configr  register_deploymentr_   )rx   r  s     rV   $recover_target_state_from_checkpointz4DeploymentState.recover_target_state_from_checkpoint  s     	N48NNNOOO4"99Hd(-<	
 	
 	
 "4G 	+??"'"6    	 	rU   replica_actor_namesc                    | j         
J d            t                              d| j         dt	          |           d           |D ]}t          j        |          }t          || j         j                  }|	                                st          
                    | d           b| j                            t          j        |           | j                            |           t                              d| d           dS )	zGRecover deployment state from live replica actors found in the cluster.NzmTarget state should be recovered successfully first before recovering current state from replica actor names.zRecovering current state for  from z live actors.z) died before controller could recover it.zRECOVERING r  )r  r   r^   r  r  r!   from_full_id_strr  r`   rP  rK  r  r  r"   
RECOVERINGr  on_replica_recoveringr;  )rx   r  replica_actor_namer   new_deployment_replicas        rV   .recover_current_state_from_replica_actor_namesz>DeploymentState.recover_current_state_from_replica_actor_names  s8   
 !--A .-- 	<DH < <+,,< < <	
 	
 	

 #6 	6 	6"34FGGJ%6"*& &" *1133 *WWWXXXN|68NOOO&<<ZHHHLL4z4445555	6 	6rU   c                     | j         j        S r   )r  r^   r   s    rV   target_infozDeploymentState.target_info=  s    !&&rU   c                     | j         j        S r   )r  r`   r   s    rV   target_versionzDeploymentState.target_versionA  s    !))rU   c                     | j         j        S r   )r  r_   r   s    rV   r_   z#DeploymentState.target_num_replicasE  s    !55rU   c                     | j         S r   )r  r   s    rV   curr_status_infoz DeploymentState.curr_status_infoI  s    %%rU   c                     | j         j        S r   )r  r   r   s    rV   r   zDeploymentState.deployment_nameM  s    x}rU   c                     | j         j        S r   )r  r   r   s    rV   r   zDeploymentState.app_nameQ  s    x  rU   c                     | j         S r   r   r   s    rV   r   zDeploymentState.docs_pathU  r   rU   c                     | j         S r   r   r   s    rV   r   zDeploymentState.route_patternsY  r   rU   c                     t           }|(| j        s!t                              d           d| _        ||n| j        j        j        j        }t          || j        j	        t          z            S )NzMAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT is deprecated and will be removed in the future. Please use 'max_constructor_retry_count' instead in configurations.T)r&   *MAX_CONSTRUCTOR_RETRY_COUNT_WARNING_LOGGEDr   rK  r  r^   ri   max_constructor_retry_countr!  r_   r'   )rx   valuebase_retry_counts      rV   _failed_to_start_thresholdz*DeploymentState._failed_to_start_threshold]  s     7T%TNNV   ?CD;   E#(:V 	 25PP
 
 	
rU   c                 B    | j         j        dk    o| j        | j        k    S )zrCheck whether replicas are currently failing and the number of
        failures has exceeded a threshold.
        r   )r  r_   r  r  r   s    rV   _replica_startup_failingz(DeploymentState._replica_startup_failingr  s-    
 2Q6 /7./	
rU   c                 :    | j          o|                                 S )a  Check whether the current version is terminally errored.

        The version is considered terminally errored if the number of
        replica failures has exceeded a threshold, and there hasn't been
        any replicas of the target version that has successfully started.
        )r  r  r   s    rV   _terminally_failedz"DeploymentState._terminally_failed|  s!     ,,P1N1N1P1PPrU   c                 H    d | j                                         D             S )Nc                     h | ]	}|j         
S rT   )r  rw  s     rV   	<setcomp>z>DeploymentState.get_alive_replica_actor_ids.<locals>.<setcomp>  s    EEEW EEErU   r  rS  r   s    rV   get_alive_replica_actor_idsz+DeploymentState.get_alive_replica_actor_ids  s$    EE0B0B0D0DEEEErU   c                 v    d | j                             t          j        t          j        g          D             S )Nc                     g | ]	}|j         
S rT   )r   rw  s     rV   rx  z;DeploymentState.get_running_replica_ids.<locals>.<listcomp>  s+     
 
 
 
 
 
rU   r  rS  r"   RUNNINGPENDING_MIGRATIONr   s    rV   get_running_replica_idsz'DeploymentState.get_running_replica_ids  sC    
 
>--%|'EF 
 
 
 	
rU   c                 |      fd j                             t          j        t          j        g          D             S )Nc                 D    g | ]}|                     j                  S rT   )r  r  )r  r  rx   s     rV   rx  z=DeploymentState.get_running_replica_infos.<locals>.<listcomp>  s:     
 
 
 ,,T-JKK
 
 
rU   r  r   s   `rV   get_running_replica_infosz)DeploymentState.get_running_replica_infos  sN    
 
 
 
>--%|'EF 
 
 
 	
rU   Nr`   c                 P    | j                             t          j        g|          S )Nr  r`   )r  r  r"   r  )rx   r`   s     rV   get_num_running_replicasz(DeploymentState.get_num_running_replicas  s#    ~##L,@+A7#SSSrU   c                     t           j        t           j        t           j        t           j        t           j        g}d | j                            |          D             S )zGet the node ids of all running replicas in this deployment.

        This is used to determine which node has replicas. Only nodes with replicas and
        head node should have active proxies.
        c                 *    h | ]}|j         	|j         S r   r  rw  s     rV   r  z6DeploymentState.get_active_node_ids.<locals>.<setcomp>  s.     
 
 
$0 !000rU   )r"   r  r  r  r  r  r  rS  )rx   active_statess     rV   get_active_node_idsz#DeploymentState.get_active_node_ids  s\     !!#  *

 
>--m<<
 
 
 	
rU   c                 H    d | j                                         D             S )Nc                     g | ]	}|j         
S rT   )r  rw  s     rV   rx  z8DeploymentState.list_replica_details.<locals>.<listcomp>  s    JJJ'%JJJrU   r  r   s    rV   list_replica_detailsz$DeploymentState.list_replica_details  s$    JJT^5G5G5I5IJJJJrU   c                    |                                  }|                                  }t          | j                  t          |          k    p| j        }|| j        k    }|s|sdS t          ||          }| j                            t          j
        | j        f|t          j
        | j        j        f|i           || _        || _        d| _        dS )aA  Broadcasts the set of running replicas over long poll if it has changed.

        Keeps an in-memory record of the last set of running replicas that was broadcast
        to determine if it has changed.

        The set will also be broadcast if any replicas have an updated set of
        multiplexed model IDs.
        N)is_availablerunning_replicasF)r  r  r  r  r  r  r   r  notify_changedr7   DEPLOYMENT_TARGETSr  r   )rx   running_replica_infosr  running_replicas_changedavailability_changeddeployment_metadatas         rV   %broadcast_running_replicas_if_changedz5DeploymentState.broadcast_running_replicas_if_changed  s    !% > > @ @22444 <==())* 21 	!
  ,t/RR' 	0D 	F2%2
 
 
 	++ &8H ' &8HM '	
 	
 	
  8M4.:+-2***rU   c                     | j         j        j        }| j        |k    rdS | j                            t          j        | j        f|i           || _        dS )zBroadcasts the deployment config over long poll if it has changed.

        Keeps an in-memory record of the last config that was broadcast to determine
        if it has changed.
        N)	r  r^   ri   r  r  r  r7   DEPLOYMENT_CONFIGr  )rx   current_deployment_configs     rV   &broadcast_deployment_config_if_changedz6DeploymentState.broadcast_deployment_config_if_changed  sf     %)$6$;$M!37PPPF++148<>WX	
 	
 	
 4M000rU   c                    t                               | j        j        dd          }|| _        | j                            t          j                  | _        t                              d| j	         ddi           d	S )
z6Set the target state for the deployment to be deleted.r   T)r^   r_   ra   triggerz	Deleting r  Fr  N)
r]   rq   r  r^   r  handle_transitionr   DELETEr   r  )rx   target_states     rV   _set_target_state_deletingz*DeploymentState._set_target_state_deleting  s    ,33#( ! 4 
 
 *!%!7!I!I3: "J "
 "
 	""""E* 	 	
 	
 	
 	
 	
rU   r  r_   updated_via_apic                 0   t                               ||d          }| j        j        |j        k    r| j        j        j        j        |j        j        j        k    r t          j                            d           nj|r t          j	                            d           nH| j        j        j        j
        |j        j        j
        k    rt          j                            d           || _        | j                            || j        | j        d           dS )a  Set the target state for the deployment to the provided info.

        Args:
            target_info: The info with which to set the target state.
            target_num_replicas: The number of replicas that this deployment
                should attempt to run.
            status_trigger: The driver that triggered this change of state.
            updated_via_api: Whether the target state update was triggered via API.
        Frg   Truer  tagsN)r]   rq   r  r`   ri   r  r9   &AUTOSCALING_CONFIG_LIGHTWEIGHT_UPDATEDrecord!NUM_REPLICAS_VIA_API_CALL_UPDATEDrt    NUM_REPLICAS_LIGHTWEIGHT_UPDATEDr  r  r   r   )rx   r  r_   r  new_target_states        rV   _set_target_statez!DeploymentState._set_target_state	  s2    177,u 8 
 
 %)9)AAA "*<O#+=PQ Q DKKFSSSS  N?FFvNNNN"*<I#+=JK K >EEfMMM- 	"&&"2#}  	' 	
 	
 	
 	
 	
rU   r  c                    | j         j        }|| j         j        s|j        |_        | j         j        pR|j        |j        k    pB|j        j        |j        j        k    p(|j        |j        k    p|j        du p|j        |j        k    }|j	        |j	        k    p|j
        |j
        k    }nd}d}|s|sdS |j        j        r,| j                            | j        || j         j                  }n>| j                            | j                   t#          |j        j        |j	                  }| j         }|                     ||           | j                            | j        |j                   | j                             |          r|j        }| j         j        }||k    r3| j                            t2          j        d| d| d          | _        nc||k     r2| j                            t2          j        d	| d| d          | _        n*| j                            t2          j        
          | _        t:                              d| j         d| d           d| _        d| _        dS )a  Deploy the deployment.

        If the deployment already exists with the same version, config,
        target_capacity, and target_capacity_direction,
        this method returns False.

        Returns:
            bool: Whether the target state has changed.
        NTF)r_   Upscaling from  to 
 replicas.r  messageDownscaling from r   zDeploying new version of z (initial target replicas: r  r   ) r  r^   ra   start_time_msri   rp   rj   rn   r`   target_capacitytarget_capacity_directionr  r  r  r  r_   deregister_deploymentr<   rt   r  r  r  ry   r  r  r   MANUALLY_INCREASE_NUM_REPLICASMANUALLY_DECREASE_NUM_REPLICASCONFIG_UPDATEr   r  r  )	rx   r  curr_deployment_infodeployment_settings_changedtarget_capacity_changedr_   old_target_stateold_numnew_nums	            rV   deployzDeploymentState.deploy.	  s     $16+%. S0D0R- "+ K'9"45K (6H"1CDK
 (48TTK #*d2K (/?3JJ ( %48WW ='A"<= $# +/'&*# + 	3J 	5,? 		"&"A"U"U/4+=+Q# # +AA$(KKK"D1>/# #
  -DWXXX"99Ho4	
 	
 	
 //0@AA 	&:G(<G  )-)?)Q)Q;ZNgNN7NNN *R * *&& 7"")-)?)Q)Q;ZPPPWPPP *R * *& &*%;%M%M7E &N & &D" 	A A A)<A A A	
 	
 	
 34/$)!trU   decision_num_replicasc           
      B   | j         j        rdS || j         j        k    rdS t          | j         j                  }| j         j        j        |_        | j         j        }|                     ||           | j        	                    | j
        | j                            t          j        g| j         j                            sdS d| j                            | j
                  dd| j                            t          j        g           d}| j         j        }||k    r}t                               d	| j
         d
| d| d|            | j                            t&          j        d| d| d          | _        | j                            | j
                   n||k     r|t                               d| j
         d
| d| d|            | j                            t&          j        d| d| d          | _        | j                            | j
                   dS )a  
        Apply the given scaling decision by updating the target replica count.

        Skips if deleting, if `decision_num_replicas` is None, or matches the
        current target. Otherwise updates the state and logs an up/down scaling.

        Args:
            decision_num_replicas: target replica count to apply.

        Returns:
            bool: True if the target state was updated, False if no change occurred.
        Fr  TzCurrent ongoing requests: z.2fz, current running replicas: r  r  z
Upscaling r  r  z replicas. r  r  r  zDownscaling r  )r  ra   r_   r   r^   r`   code_versionr  r  is_within_boundsr  r  r  r"   r  %get_total_num_requests_for_deploymentr   r  r  r   AUTOSCALE_UPrecord_scale_upAUTOSCALE_DOWNrecord_scale_down)rx   r&  new_infor#  curr_stats_strr$  s         rV   	autoscalezDeploymentState.autoscale	  s    & 	5 D$6$JJJ5*/00-5B$8x)>??? .??HN  $,-t7I7Q !  
 
 	 4F.TTUYU]^^fF F ~##L,@+A#BBF F F 	 $8WKK$TX $ $W $ $' $ $!$ $   &*%;%M%M7DJ'JJwJJJ &N & &D" +;;DHEEEEwKK$tx $ $w $ $G $ $!$ $   &*%;%M%M7FLGLLLLL &N & &D" +==dhGGGtrU   c                 J    | j         j        s|                                  dS dS )NTF)r  ra   r  r   s    rV   deletezDeploymentState.delete	  s,    !* 	++---4urU   c                 J    |                      | j        j        |d           dS )z=Set the target state for the deployment to the provided info.T)r  N)r  r  r^   )rx   r_   s     rV   set_target_num_replicasz'DeploymentState.set_target_num_replicas	  s9    
 	#%8$ 	 	
 	
 	
 	
 	
rU   c                    | j                             | j        j        t          j        t          j        t          j        g          }d}d}d}|D ]}||z   |k    r&| j                             |j	        j
        |           2|j                            | j        j                  r9|dz  }|j	        j
        t          j        k    }|                     ||           d}|j	        j
        t          j        k    r|dz  }|j                            | j        j                  rd}| j                            |j        j                  }|                    | j        j        |j                  }	|	r'| j                             t          j        |           G| j                             t          j        |           n| j                             |j	        j
        |           |dk    r&t*                              d| d	| j         d
           |dk    rEt*                              d| d	| j         d           t0          j                            d           |S )au  Stop or update replicas with outdated versions.

        Stop replicas with versions that require the actor to be restarted, and
        reconfigure replicas that require refreshing deployment config values.

        Args:
            max_to_stop: max number of replicas to stop, by default,
                         it stops all replicas with an outdated version.
        r  r  Fr   rI   rg  Tr  r  z replicas of z with outdated versions.z	Updating z" with outdated deployment configs.r  )r  r  r  r`   r"   r  r  r  r  r  r  requires_actor_restart_stop_replicarequires_long_poll_broadcastr  rq  r   r[  rF  r   r  r   r^   r  r9   USER_CONFIG_LIGHTWEIGHT_UPDATEDr  )
rx   max_to_stopreplicas_to_updatereplicas_changedcode_version_changesreconfigure_changesr  rg  rJ  actor_updatings
             rV   )_stop_or_update_outdated_version_replicasz9DeploymentState._stop_or_update_outdated_version_replicas	  s    "^// .6%.$ 0 
 
 ! ) "	I "	IG$'::{JJ""7#8#>HHHH 778J8RSS I$)$ !( 5 ;|?S S""7-"HHH#'   &,0DDD#q(#???&.  , (,$#1BB&0    ")!4!4&.\5F "5 " " " FN&&|'<gFFFFN&&|';WEEEE ""7#8#>HHHH!##KK*0 * *tx * * *  
 ""KK4/ 4 4dh 4 4 4  
 9@@HHHrU   c                 j   | j         j        dk    rdS | j                            | j         j        t
          j        t
          j        t
          j        g          }| j                            | j         j        t
          j	        g          }| j                            | j         j        t
          j        g          }| j         j        ||z   k     rdS | j         j        |z
  |z
  }t          t          d| j         j        z            d          }t          ||z
  d          }|                     |          S )zStops replicas with outdated versions to implement rolling updates.

        This includes both explicit code version updates and changes to the
        user_config.

        Returns whether any replicas were stopped.
        r   Fr8  )r`   r  g?rI   )r  r_   r  r  r`   r"   r  r  r  STOPPINGmaxr|   rD  )rx   old_running_replicasold_stopping_replicasnew_running_replicaspending_replicasrollout_sizer>  s          rV   )_check_and_stop_outdated_version_replicasz9DeploymentState._check_and_stop_outdated_version_replicas$
  sK    1Q665
  $~33 .6%%$  4  
  
 !% 4 4 .6@U?V !5 !
 !
  $~33&.8L7M  4  
  
 2"%::; ; 5 2"#"# 	 3sT%7%KKLLaPP,)991====kJJJrU   c           	          | j         j        dk    s
J d            g }d}|                                  | j                            t
          j        t
          j        t
          j        g          }| j                            t
          j	        g          }| j         j        |z
  |z
  }|dk    r||fS |dk    r |}|dk    r| 
                                st                              d| dd|dk    z   d	| j         d
           t          |          D ]}t          t!                      | j                  }t#          || j         j                  }	|	                    | j         j        | j        j                  }
|                    |
           | j                            t
          j        |	           nT|dk     rN| }| d|dk    rdnd }t                              d| d| j         d
           t1          | j        |          }||fS )z5Scale the given deployment to the number of replicas.r   z=Target number of replicas must be greater than or equal to 0.Nr(  zAdding z replicasrI   r  r  )r   r   z	Removing r  )r   num_to_stop)r  r_   rM  r  r  r"   r  r  r  r  r  r   r^   r  r:  r!   r=   r  r`   r:  r  r$  r  r  r0   )rx   r   r   current_replicasrecovering_replicasdelta_replicasto_addra  r   r  scheduling_request	to_removeremoved_replicass                rV   scale_deployment_replicasz)DeploymentState.scale_deployment_replicas]
  sf    2a777J 877 	66888>// )<+@,BVW 0 
 
 #n22<;R:S2TT 2!" 	
 QY''a#Fzz$"9"9";";zWfWWcVAX6FWWDHWWWXXXv V VA!*+<+>+>dh!W!W!WJ->"*2. .* *@)E)E*/-1-?-K *F * *&
 NN#5666N&&|'<>TUUUUa'I"+SSIMMSSrSSKKG$4GGDHGGGHHH2"hI  I 	!!rU   c                 *   | j         j        }| j                            t          j        g          dk    }| j                            t          j        g          }| j                            t          j        g|          }|dk    rd| _        nS|                                 r?| j	        
                    t          j        d| j         d| j                   | _	        d|fS | j                            t          j        t          j        t          j        t          j        g          dk    ra| j         j        r
|dk    rd|fS | j         j        |k    r;||k    r5| j	        
                    t          j        	          | _	        d| _        d|fS d|fS )
a6  Check the current deployment status.

        Checks the difference between the target vs. running replica count for
        the target version.

        This will update the current deployment status depending on the state
        of the replicas.

        Returns (deleted, any_replicas_recovering).
        r(  r   r  TzThe deployment failed to start z times in a row. This may be due to a problem with its constructor or initial health check failing. See controller logs for details. Error:
r  Fr   )r  r`   r  r  r"   r  r  r  r  r  r  r   REPLICA_STARTUP_FAILEDr  r  r  r  rF  ra   r_   HEALTHY)rx   r  r   all_running_replica_cnt%running_at_target_version_replica_cnts        rV   check_curr_statusz!DeploymentState.check_curr_status
  s    +3 N  )@(A BBQF 	  #'."6"6|?S>T"6"U"U040D0D ()> 1E 1
 1
- 1144 )-D%%**,, 	2%)%;%M%M7N=>= =
 := = &N 
& 
&D" 111 N   ) ) + )	 !     !* 5/F!/K/K444 "689 99=TTT)-)?)Q)Q;C *R * *& ;<7555---rU   original_statec           	         g }| j                             |g          D ]}|                                \  }}|t          j        k    r(|t
          j        k    r2|j        j        }| j	        
                    ||j        |j                   | j                             t
          j        |           | j                            |j        |j                   |j        | j        j        k    r|j        | _        |j        | _        t/          j                    |j        z
  }|j         d|j         d|dd|j         d}	|j        |	d|j        dd	z  }	t6                              |	d
di           S|t          j        k    r,|                     |           |                     |           |t          j         t          j!        fv rut/          j                    |j        z
  tD          k    }
|
r|#                    ||f           |
r|r|                     |d           | j                             ||            |S )a  
        Common helper function for startup actions tracking and status
        transition: STARTING, UPDATING and RECOVERING.

        Args:
            stop_on_slow: If we consider a replica failed upon observing it's
                slow to reach running state.
        r(  z started successfully on node 'z' after z.1fzs (PID: r  NzH Replica constructor, reconfigure method, and initial health check took zs.r  Fr  r9  )$r  r  r  rH   rR   r"   r  r   r[  r  r+  r  r   r  r  r  on_replica_runningr`   r  r   r   r   r   rp  r  r  r  r   r^   rS   record_replica_startup_failurer;  rP   rQ   SLOW_STARTUP_WARNING_Sr  )rx   r`  stop_on_slowslow_replicasr  start_status	error_msgr   e2e_replica_start_latencyreplica_startup_messageis_slows              rV   _check_startup_replicasz'DeploymentState._check_startup_replicas
  s    ~)).1A)BB =	@ =	@G&-&;&;&=&=#L)3===!\%<<< ")!3!=J&33"G$97<  
 ""<#7AAA*==&(=   ?d&8&@@@&-&7DO+2+AD( -1IKK':M,M)) T T ' 5T T0ST T>E>OT T T (
 3? ,D";CD D D+
 3OU;STTTT!5!<<<33I>>>""7++++$7$;"   )++(;;>TT B!(('<)@AAA  @| @&&we&DDDDN&&~w???rU   rh  c                     | j         j        dk    rdS | xj        dz  c_        || _        d}| j        s#t          | j        | j        z
  d          }d| d}d| d| }| j                            |          | _        dS )	z&Record that a replica failed to start.r   NrI   rP   z more time(s)z2A replica failed to start with exception. Retryingz	. Error:
)	r  r_   r  r  r  rG  r  r  update_message)rx   rh  retrying_msgremaining_retriesr  s        rV   rc  z.DeploymentState.record_replica_startup_failure1  s     1Q66F 	//14//.7+
 ( 	@ #/9:! !
 @0???L# # # # # 	 "&!7!F!Fw!O!OrU   c                     | j                                         D ]F}|j        |v r|                     |           !| j                             |j        j        |           Gd S r   )r  r  r   r;  r  r  r  )rx   replicas_to_stopr  s      rV   stop_replicaszDeploymentState.stop_replicasN  st    ~))++ 	I 	IG!%555""7++++""7#8#>HHHH		I 	IrU   Tr  c                 h   t                               d|j         d           |                    |           | j                            t          j        |           | j        	                    |j                   | j
                            d| j        |j        j        | j        d           dS )zStop replica
        1. Stop the replica.
        2. Change the replica into stopping state.
        3. Set the health replica stats to 0.
        zAdding STOPPING to replica: r  r  r   r  r	  N)r   r;  r   r  r  r  r"   rF  r  on_replica_stoppingr  r  r   r[  r   )rx   r  rg  s      rV   r;  zDeploymentState._stop_replicaU  s     	IG4FIIIJJJm,,,<0':::"66w7IJJJ##"2"-7#}  	$ 	
 	
 	
 	
 	
rU   c                 
   | j                             t          j        t          j        g          D ]N}|                                r| j                             |j        j        |           | j	        
                    d| j        |j        j        | j        d           |                                }|                    |           t"                              d|j         d           | j	        
                    d| j        |j        j        | j        d           |                     || j                    |j        | j        j        k    r+| j                            t2          j        d	
          | _        Pg }|                     t          j                  }|                     t          j                  }|                     t          j        d          }||z   |z   }t?          |          rtA          j                     | j!        z
  tD          k    rg }g }|D ]O\  }}	|	tF          j$        k    r|%                    |           |	tF          j&        k    r|%                    |           Pt?          |          dk    r|d         '                                \  }
}d| j         d| j         dt?          |           dtP           d|
 d| d}t"                              |           tR          rtU                       | j        j+        tX          j-        tX          j.        fvr| j        /                    |          | _        t?          |          dk    rd| j         d| j         dt?          |           dtP           d	}t"                              |           | j        j+        tX          j-        tX          j.        fvr| j        /                    |          | _        tA          j                     | _!        | j                             t          j0        g          D ]}|1                                }|s&| j                             t          j0        |           >t"          2                    |j         d           |j        j        }| j3        4                    |          r?| j3        5                    |           t"          6                    d| d| j7                    | j8        9                    |j                   | j         :                                }|rK| j        j+        tX          j;        k    r3| j3        <                    |          }| =                    |           dS dS dS )z
        Check current state of all DeploymentReplica being tracked, and compare
        with state container from previous update() cycle to see if any state
        transition happened.
        r(  rI   r  r	  r  z" failed health check, stopping it.r   r9  zvA replica's health check failed. This deployment will be UNHEALTHY until the replica recovers or a new deploy happens.r  T)re  Deployment 'z' in application 'z' has z$ replicas that have taken more than zs to be scheduled. This may be due to waiting for the cluster to auto-scale or for a runtime environment to be installed. Resources required for each replica: z, total resources available: z$. Use `ray status` for more details.zMs to initialize.
This may be caused by a slow __init__ or reconfigure method.z is stopped.zReleased rank from replica z in deployment N)>r  r  r"   r  r  rN  r  r  r  r  r  r   r   r[  r   r  r  r   rK  r;  FORCE_STOP_UNHEALTHY_REPLICASr`   r  r  r  r   HEALTH_CHECK_FAILEDrl  r  r  r  r  rp  r  SLOW_STARTUP_WARNING_PERIOD_SrH   rP   r  rQ   r  rd  r   r   statusr   	UNHEALTHYDEPLOY_FAILEDro  rF  rm  r^   r  rb  r(  r;  r  r  on_replica_stoppedrS  r\  rD  $_reconfigure_replicas_with_new_ranks)rx   r  r  slow_start_replicas
slow_startslow_updateslow_recoverpending_allocationpending_initializationstartup_statusr  r  r  rl  r   rr  replicas_to_reconfigures                    rV   check_and_update_replicasz)DeploymentState.check_and_update_replicash  s    ~)) (,*HI * 
 
 '	 '	G ##%% $""7#8#>HHH'++&*&:#*#5#?'+}  ,    !( : : < <,,];;;;Uw1UUU   '++&*&:#*#5#?'+}  ,    ""t/Q+Q #    ?d&8&@@@-1-C-U-U ? S!< .V . .D* !11,2GHH
22<3HII33#$ 4 
 
 );6E #$$;	5	d88;XXX!#%'"+> ; ;'!%9%LLL&--g666!%9%PPP*11':::%&&**&8&;&Q&Q&S&S#)94#7 9 99 9-01C-D-D9 9,B9 9
 =E9 9 3<9 9 9  w'''' 0-/// )0$.$29   .2-C-R-R. .D* )**Q..S4#7 S SS S-01G-H-HS S1GS S S  w''' )0$.$29   .2-C-R-R. .D* *.D&~)),2G1H)II 	W 	WG++--G W""<#8'BBBBw1???@@@ %/9
%66zBB  &33J???LL[j[[QUQY[[   /BB7CUVVVV .,,..	O&-1A1III "PP#  $ 556MNNNNN	O 	OIIrU   r  r  c                 b   |sdS t                               dt          |           d| j                    d}|D ]N}|j        j        }| j                            |          }|                    | j	        j
        |          }|dz  }Ot                               d| d| j                    dS )	zReconfigure replicas with their new ranks after reassignment.
        This uses the reconfigure() mechanism to update replicas with their new ranks.
        NzReconfiguring z* replicas with rank changes in deployment r   r  rI   zSuccessfully reconfigured z' replicas with new ranks in deployment )r   r;  r  r  r   r[  r  rq  rF  r  r`   )rx   r  updated_countr  r   rM  ra  s          rV   r  z4DeploymentState._reconfigure_replicas_with_new_ranks  s     ' 	FoS!899ooeiemoo	
 	
 	
 . 
	 
	G +5J):::FFH ##"* $  A QMMiii_c_gii	
 	
 	
 	
 	
rU   c                 4    | j                                         S )zGet the current mapping of replica IDs to ReplicaRank objects.

        Returns:
            Dictionary mapping replica_id to ReplicaRank object (with rank, node_rank, local_rank).
        )r  r  r   s    rV   _get_replica_ranks_mappingz*DeploymentState._get_replica_ranks_mapping#  s     !;;===rU   r  	deadlinesmin_replicas_to_stopc                    g }g }|D ]q}|j         v sJ t          j                    dz  }|j        j        dz  }||j                  |z
  k    r|                    |           \|                    |           r|                    fd           |t          |          z
  }	|	dk    r'|                    |d|	                    ||	d         }||fS )a  Returns a partition of replicas to stop and to keep.

        Args:
            replicas: The current list of replicas pending migration.
            deadlines: The current draining node deadlines.
            min_replicas_to_stop: The minimum number of replicas to stop.
        i  c                     | j                  S r   r  )r  r  s    rV   r  zLDeploymentState._choose_pending_migration_replicas_to_stop.<locals>.<lambda>I  s    Yq%? rU   r  r   N)r  rp  r  r   r  sortr  r  )
rx   r  r  r  to_stopr   r  curr_timestamp_ms
timeout_ms
num_excesss
     `       rV   *_choose_pending_migration_replicas_to_stopz:DeploymentState._choose_pending_migration_replicas_to_stop+  s    	   	* 	*G(I5555 $	d 2 CdJJ Ig.C$Dz$QQQw''''  ))))
 	????@@@)CLL8
>>NN9[j[1222!*++.I	!!rU   draining_nodesc                    | j                             t          j        g          D ]V}|j        |vr&| j                             t          j        |           1| j                             t          j        |           W| j                             t          j        t          j        t          j        g          D ]}|j        |v r|j	        j
        t          j        k    rQt                              d|j         d|j         d           | j                             t          j        |           v|                     |d           | j                             |j	        j
        |           | j                             t          j        g          }| j                             t          j        g          }||z   | j        j        z
  }|                     | j                             t          j        g          ||          \  }}|D ]D}t                              d|j         d|j         d	           |                     |d           E|D ]'}| j                             t          j        |           (d S )
Nr(  z
Migrating z from draining node 'z1'. A new replica will be created on another node.Tr9  r  z on draining node r  )r  r  r"   r  r  r  r  r  r  r  r  r   r^   r   r;  r  r  r_   r  )rx   r  r  num_runningnum_draining&num_pending_migration_replicas_to_stoprs  replicas_to_keeps           rV   "migrate_replicas_on_draining_nodesz2DeploymentState.migrate_replicas_on_draining_nodesR  s   
 ~)),2P1Q)RR 	L 	LG$N::""<#7AAAA""<#A7KKKK ~)) )<+?AVW * 
 
 	I 	IG $66 (.,2FFFKK(W%7 ( (#1( ( (  
 N&&|'EwOOOO &&wd&CCCC""7#8#>HHHHn**<3G2H*II~++L4R3S+TT,&);)OO 	/ ;;N|'E&FGG2
 
	
 ( 	< 	<GKK=G. = =$+$9= = =   wd;;;;' 	H 	HGN|=wGGGG	H 	HrU   r^   c                 :   | j                                         D ]^}|j        |j        k    rL|j        |                    |j                   |j        |                    |j                   d| _         dS _t          	                    |j         d           dS )zRecords the multiplexed model IDs of a replica.

        Args:
            info: RequestRoutingInfo including deployment name, replica tag,
                multiplex model ids, and routing stats.
        NTz not found.)
r  rS  r   r  r  r  r  r  r   rK  )rx   r^   r  s      rV   record_request_routing_infoz+DeploymentState.record_request_routing_info  s     ~))++ 	 	G!T_44-9889STTT%1001CDDD592 5 	$/66677777rU   c                 D   | j                             t          j        g          }|                                }|                    d           | j                             t          j        |           |D ]'}| j                             t          j        |           (d S )Nr(  Frv  )r  r  r"   r  r  r  rF  )rx   r  replica_to_stopr  s       rV   %_stop_one_running_replica_for_testingz5DeploymentState._stop_one_running_replica_for_testing  s    >--l6J5K-LL*..00e,,,<0/BBB' 	> 	>GN|3W====	> 	>rU   c                 $    | j         j        j        S r   )r  r^   r(  r   s    rV   
is_ingresszDeploymentState.is_ingress  s    !&..rU   c                 $   t                      }d}| j                            t          j        g          D ]E}|j        | j        j        k    r|                                }||                    |           d}F|sdS t          |d           S )zGet the outbound deployments.

        Returns:
            Sorted list of deployment IDs that this deployment calls. None if
            outbound deployments are not yet polled.
        FNTc                     | j         S r   )r   )ds    rV   r  z:DeploymentState.get_outbound_deployments.<locals>.<lambda>  s    QV rU   r  )
r  r  rS  r"   r  r`   r  r  r7  r8  )rx   r  has_outbound_deploymentsr  outbound_deploymentss        rV   r  z(DeploymentState.get_outbound_deployments  s     %(EE#( ~))<+?*@AA 	0 	0G$"4"<<<#*#C#C#E#E #/2333+/(' 	4f"4"45555rU   r   r  r  r  )TrM   rN   rO   rz   r*   rz  r  r   r6   r1   r   r   r   r}   r  r]   r  r  r
   rU  r  r  r/   r  r@   r  r|   r_   r   r  r   r   r   r   r   r  r  r  r   r  r!   r  r$   r  r  r  rC   r  r  r  r  r  r%  r2  r4  r6  r  r  rD  rM  r   r2   r0   rY  r_  r"   r  rH   rl  rc  rt  r;  r  r  r	   rD   r  r  r  r#   r  r  r  r  rT   rU   rV   r  r    s'       HH$K!16.D9D9 %D9 2	D9
 "6D9 $;D9 D9 D9 D9LU$ U U U U"%: " " " "'<   6#'96 6 6 6F '^ ' ' ' X' * 1 * * * X* 6S 6 6 6 X6 &"6 & & & X&     X !# ! ! ! X! 8C=    X $c 3 $ $ $ X$ 
C 
 
 
 X
(
$ 
 
 
 
QD Q Q Q QFSX F F F F
i 
 
 
 

40B+C 
 
 
 
T T0A TS T T T T
SX 
 
 
 
*Kd>&: K K K K+3 +3 +3 +3ZM M M M 
 
 
 
* !&	+
 +
#+
 !+
 	+

 
+
 +
 +
 +
ZYn Y Y Y Y Yv@s @t @ @ @ @D    
 
 

 
 
 
 EIH G  G QU G  G  G  G R7K4 7K 7K 7K 7Kr7"	t,-/II	J7" 7" 7" 7"rL.5t#4 L. L. L. L.^ :?K K*K	e%';;<	=K K K KZP P P P P:I I I I
 
%6 
 
 
 
&[O [O [Oz
'+,?'@
 
 
 
<>Dk1A,B > > > >%"()%" S>%" "	%"
 
t%&->(??	@%" %" %" %"N8Hc3h 8H 8H 8H 8Ht80B 8t 8 8 8 8&> > >/D / / / /6(43E*F 6 6 6 6 6 6rU   r  c                   z   e Zd ZdZ	 	 d3dededee         dee         dede	d	e
e         d
e
e         fdZd Zdee         deeee         f         fdZdee         dee         fdZdee         dee         fdZd ZdefdZd4dZdeeee         f         fdZdeeef         fdZdede
e         fdZdede
e         fdZdede
ee                  fdZdede
e         fdZdede
e          fdZ!	 d5de
ee                  dee"         fdZ#de$e         fdZ%ded edefd!Z&d"edee         fd#Z'defd$Z(defd%Z)ded&efd'Z*defd(Z+ded&edefd)Z,ded*ee-         fd+Z.d, Z/d-e0ddfd.Z1de$e         fd/Z2dee3eeeef                  fd0Z4dedeee5f         fd1Z6dede
ee                  fd2Z7dS )6DeploymentStateManagerzManages all state for deployments in the system.

    This class is *not* thread safe, so any state-modifying methods should be
    called with a lock held.
    Nkv_storer  all_current_actor_names!all_current_placement_group_namesr  r  head_node_id_override"create_placement_group_fn_overridec	                     || _         || _        || _        t          j        |||          | _        || _        d| _        i | _        t          t                    | _        |                     ||           d S NF)	_kv_storer  r  r   create_deployment_schedulerr  r  _shutting_down_deployment_statesr   r  _app_deployment_mapping_recover_from_checkpoint)	rx   r  r  r  r  r  r  r  r  s	            rV   r   zDeploymentStateManager.__init__  s     "-(?%%1%M#!.&
 &
"
 +D'#GI<G<L<L$%%#%F	
 	
 	
 	
 	
rU   c                     | j                             |t                                 t          || j        | j         | j        | j                  S r   )r  on_deployment_createdr4   r  r  r  r  rx   r   s     rV   _create_deployment_statez/DeploymentStateManager._create_deployment_state  sX    "88;==	
 	
 	
  &)+
 
 	
rU   rb   c                     d |D             }t          t                    }t          |          dk    r9|D ]6}t          j        |          }||j                                     |           7|S )a\  
        Given a list of all actor names queried from current ray cluster,
        map them to corresponding deployments.

        Example:
            Args:
                [A#zxc123, B#xcv234, A#qwe234]
            Returns:
                {
                    A: [A#zxc123, A#qwe234]
                    B: [B#xcv234]
                }
        c                 :    g | ]}t          j        |          |S rT   )r!   is_full_id_str)r  r  s     rV   rx  zIDeploymentStateManager._map_actor_names_to_deployment.<locals>.<listcomp>  s9     
 
 
'
33

 
 
rU   r   )r   r  r  r!   r  r   r  )rx   r  all_replica_namesdeployment_to_current_replicasreplica_namer   s         rV   _map_actor_names_to_deploymentz5DeploymentStateManager._map_actor_names_to_deployment  s     
 
5
 
 

 *5T):):& !!A%% 1  &7EE
.z/GHOO     .-rU   c                    g }|D ]/}t          j        |          r||vr|                    |           0t          |          dk    rt                              d| d           |D ]o}	 t          j                            |          }t          j        	                    |           B# t          $ r! t                              d| d           Y lw xY wdS )a  Detect and remove any placement groups not associated with a replica.

        This can happen under certain rare circumstances:
            - The controller creates a placement group then crashes before creating
            the associated replica actor.
            - While the controller is down, a replica actor crashes but its placement
            group still exists.

        In both of these (or any other unknown cases), we simply need to remove the
        leaked placement groups.
        r   z"Detected leaked placement groups: z. The placement groups will be removed. This can happen in rare circumstances when the controller crashes and should not cause any issues. If this happens repeatedly, please file an issue on GitHub.z(Failed to remove leaked placement group r  N)r!   r  r  r  r   rK  r   rL  rM  rk  rW  rT  )rx   r  r  leaked_pg_namespg_nameleaked_pg_namepgs          rV   *_detect_and_remove_leaked_placement_groupszADeploymentStateManager._detect_and_remove_leaked_placement_groups  s(     8 	0 	0G(110#:::&&w///!##NNV_ V V V   . 	 	NX11.AA//3333     P~PPP    		 	s   +>B**(CCc                 "   |                      ||           |                     |          }| j                            t                    }|t          j        |          }|                                D ]\  }}|                     |          }|	                    |           t          ||                   dk    r|                    ||                    || j        |<   | j        |j                                     |j                   dS dS )aS  
        Recover from checkpoint upon controller failure with all actor names
        found in current cluster.

        Each deployment resumes target state from checkpoint if available.

        For current state it will prioritize reconstructing from current
        actor names found that matches deployment tag if applicable.
        Nr   )r  r  r  rS  CHECKPOINT_KEYr   r4  r  r  r  r  r  r  r  r   r  r   )	rx   r  r  r  
checkpointdeployment_state_infor   checkpoint_datadeployment_states	            rV   r  z/DeploymentStateManager._recover_from_checkpoint:  s<    	77#-	
 	
 	

 *.)L)L#*
 *
& ^''77
!$/$5j$A$A!2G2M2M2O2O 
 
.#'#@#@#O#O  EEoVVV5mDEEII$SS6}E   :J'6,]-CDHH!&    "!
 
rU   c                     d| _         | j                                        D ]}|                                 | j                            t
                     dS )a  
        Shutdown all running replicas by notifying the controller, and leave
        it to the controller event loop to take actions afterwards.

        Once shutdown signal is received, it will also prevent any new
        deployments or replicas from being created.

        One can send multiple shutdown signals but won't effectively make any
        difference compare to calling it once.
        TN)r  r  r9  r4  r  r  )rx   r  s     rV   shutdownzDeploymentStateManager.shutdown`  s^     # $ 7 > > @ @ 	& 	&##%%%%
 	n-----rU   c                     | j         o8t          | j                  dk    o | j                            t
                    du S )zwReturn whether all deployments are shutdown.

        Check there are no deployment states and no checkpoints.
        r   N)r  r  r  r  rS  r  r   s    rV   is_ready_for_shutdownz,DeploymentStateManager.is_ready_for_shutdownx  sD      ;D+,,1;"">22d:	
rU   c                     | j         rdS d | j                                        D             }| j                            t
          t          j        |                     dS )z,Write a checkpoint of all deployment states.Nc                 >    i | ]\  }}||                                 S rT   )r  )r  r   r  s      rV   r  z:DeploymentStateManager.save_checkpoint.<locals>.<dictcomp>  s<     !
 !
 !
// +??AA!
 !
 !
rU   )r  r  r  r  putr  r   r   )rx   r  s     rV   save_checkpointz&DeploymentStateManager.save_checkpoint  sz     	 F!
 !
373J3P3P3R3R!
 !
 !

 	344	
 	
 	
 	
 	
rU   c                 H    d | j                                         D             S )Nc                 >    i | ]\  }}||                                 S rT   )r  )r  r  r  s      rV   r  zDDeploymentStateManager.get_running_replica_infos.<locals>.<dictcomp>  s<     
 
 
$$  ::<<
 
 
rU   )r  r  r   s    rV   r  z0DeploymentStateManager.get_running_replica_infos  s2    
 
(,(?(E(E(G(G
 
 
 	
rU   c                 \    i }| j                                         D ]\  }}|j        ||<   |S r   )r  r  r  )rx   infosr   r  s       rV   get_deployment_infosz+DeploymentStateManager.get_deployment_infos  sB    46/3/F/L/L/N/N 	@ 	@+M+#3#?E-  rU   r   c                 <    || j         v r| j         |         j        S d S r   )r  r  r  s     rV   get_deploymentz%DeploymentStateManager.get_deployment  s&    D333*=9EE4rU   c                 <    || j         v r| j         |         j        S d S r   )r  r   r  s     rV   get_deployment_docs_pathz/DeploymentStateManager.get_deployment_docs_path  s)    D333*=9CC 43rU   c                 <    || j         v r| j         |         j        S dS )z1Get route patterns for a deployment if available.N)r  r   r  s     rV   get_deployment_route_patternsz4DeploymentStateManager.get_deployment_route_patterns  s(     D333*=9HHtrU   c                 <    || j         vrd S | j         |         j        S r   )r  r_   r  s     rV   "get_deployment_target_num_replicasz9DeploymentStateManager.get_deployment_target_num_replicas  s(      7774&}5IIrU   r  c                 z   |                      |g          }t          |          dk    rdS |d         }| j        |         }t          |j        |j        |j        |j        t          |j        | 	                    |                    |j
        j        |j        j        j        |                                          S )zGets detailed info on a deployment.

        Returns:
            DeploymentDetails: if the deployment is live.
            None: if the deployment is deleted.
        r   N)r   r}  status_triggerr  ri   r_   required_resourcesr  )get_deployment_statusesr  r  rB   r   r}  r  r  rE   r  r  r_   r  rp   r'  r  )rx   r  statusesstatus_infor  s        rV   get_deployment_detailsz-DeploymentStateManager.get_deployment_details  s     //55x==A4"1+K#6r:$W")*9#+"<GT0044# # %5$B$V#3#?#N#\)>>@@   rU   idsc                     |#d | j                                         D             S g }|D ]8}| j                             |          }||                    |j                   9|S )z
        Return the statuses of the deployments with the given `ids`.
        If `ids` is `None`, returns the status of all deployments.
        Nc                     g | ]	}|j         
S rT   )r  )r  r  s     rV   rx  zBDeploymentStateManager.get_deployment_statuses.<locals>.<listcomp>  s(       +0&  rU   )r  r9  rS  r  r  )rx   r  r  r  r  s        rV   r  z.DeploymentStateManager.get_deployment_statuses  s     ; 484K4R4R4T4T    H < </33B77$OOE$:;;;OrU   c                     t                      }| j                                        D ]}||                                z  }|S r   )r  r  r9  r  )rx   alive_replica_actor_idsdss      rV   r  z2DeploymentStateManager.get_alive_replica_actor_ids  sL    "%%%)0022 	H 	HB#r'E'E'G'GG##&&rU   r  c                 
   || j         vr[|                     |          | j         |<   | j        |j                                     |j                   |                                  | j         |                             |          S )zDeploy the deployment.

        If the deployment already exists with the same version and config,
        this is a no-op and returns False.

        Returns:
            bool: Whether the target state has changed.
        )r  r  r  r   r  r   _record_deployment_usager%  )rx   r   r  s      rV   r%  zDeploymentStateManager.deploy  s      777595R5R6 6D#M2 ()?@DD]EWXXX))+++&}5<<_MMMrU   r   c                 6    t          | j        |                   S )z/Return list of deployment names in application.)r  r  )rx   r   s     rV   get_deployments_in_applicationz5DeploymentStateManager.get_deployments_in_application  s    D0:;;;rU   c                 V    || j         v r| j         |                                         S dS r  )r  r4  )rx   r  s     rV   delete_deploymentz(DeploymentStateManager.delete_deployment  s1     (((*2.55777urU   c                     |                      |g          }|t          |          dk    rt          d| d          |d         j        t          j        k    rt          d| d          dS )z:Validate the state of a deployment for num replica update.Nr   zDeployment z
 not foundz6 is being deleted. Scaling operations are not allowed.)r  r  ro   r  r   DELETINGr5   )rx   r   r  s      rV   1_validate_deployment_state_for_num_replica_updatezHDeploymentStateManager._validate_deployment_state_for_num_replica_update  s     //@@s8}}11D=DDDEEEa['+B+KKK/cmccc   LKrU   r_   c                 @   |                      |           | j        |         }||j        k    rSt                              d|j         d| d|            |                    |           |                                  dS t                              d|            dS )z/Set target number of replicas for a deployment.z'Target number of replicas changed from r  z for deployment zPSkipping updating target number of replicas as it did not change for deployment N)r  r  r_   r   r^   r6  r  )rx   r   r_   r  s       rV   r6  z.DeploymentStateManager.set_target_num_replicas!  s     	>>}MMM2=A"2"FFFKK Y:J:^  Y  Ydw  Y  Y  JW  Y  Y   445HIII  """""KKrcprr    rU   c                 p   g }d}i }i }d}| j                                         D ]}|                                 | j                                         D ]}|                                 | j                                        }t          |          dk    o/t          d | j                                         D                       }t          r&| j	        
                    |          }	|	r	|	\  }
}|
|i}| j                                         D ]\  }}|                    |           | j                                         D ]*\  }}|                                \  }}|r|||<   |r|||<   +| j                                         D ]8\  }}|                                \  }}|r|                    |           ||z  }9| j	                            ||          }|                                D ]%\  }}| j         |                             |           &|                                D ]\  }}|                     ||           | j                                         D ]o\  }}|                                 |                                 |                                r.| j                            ||                                           p|D ]}| j	                            |           | j                            |           | j         |= |j        | j        v rb|j        | j        |j                 v rI| j        |j                                     |j                   | j        |j                 s| j        |j        = t          |          r|                                  |r|                                  |S )zUpdates the state of all deployments to match their goal state.

        Returns True if any of the deployments have replicas in the RECOVERING state.
        Fr   c              3      K   | ]p}|j         j        t          j        k    oQ|                    |j                  |j        k    o.t          |j        	                                          |j        k    V  qd S r   )
r  r}  r   r\  r  r  r_   r  r  rS  )r  r  s     rV   r  z0DeploymentStateManager.update.<locals>.<genexpr>I  s       
@
 
@
  &*:*BB B ++B,=>>"BXX	B BL$$&&''2+AA
@
 
@
 
@
 
@
 
@
 
@
rU   )allow_new_compaction)r   r  ) r  r9  r  r_  r  get_draining_nodesr  rv   r+   r  get_node_to_compactr  r  rY  r  schedulert  #_handle_scheduling_request_failuresr  r  r  r  update_running_replica_idsr  on_deployment_deletedr  r   r  r   r"  r  r  )rx   deleted_idsany_recoveringupscales
downscalestarget_state_changedr  r  r  	node_infotarget_node_iddeadliner   r   r   r   r   deployment_to_replicas_to_stoprs  scheduling_requestss                       rV   r7  zDeploymentStateManager.update3  s    GIEG
$ !% 7 > > @ @ 	9 	9668888 !% 7 > > @ @ 	1 	1..0000 6IIKK">22a7 
 
C 
@
 
@
 -4466
@
 
@
 
@
 
=
 
=
 2 		< *>>%9 ?   
  <+4("0(!;/3/F/L/L/N/N 	P 	P+M+??OOOO 04/F/L/L/N/N 	6 	6+M+!1!K!K!M!MGY 2*1' 6,5
=) 04/F/L/L/N/N 	6 	6+M+/?/Q/Q/S/S,G, 2""=11155NN *.)C)L)Lj*
 *
& 0N/S/S/U/U 	S 	S+M+#M2@@AQRRRR2:..2B2B 	Y 	Y.M.44]DWXXXX 04/F/L/L/N/N 	 	+M+BBDDDCCEEE0022 /JJ"/%5%M%M%O%O K    ) 	M 	MM&<<]KKK+AA-PPP'6&$*FFF!&/0FGH H ,]-CDKK!&   3M4JK M4]5KL{ 	,))+++ 	#  """rU   c                 X    || j         vrdS | j         |                             |          S )a  Autoscale the deployment to the target number of replicas.

        Args:
            deployment_id: The deployment ID.
            target_num_replicas: The target number of replicas.

        Returns:
            True if the deployment was autoscaled, False otherwise.
        F)r  r2  )rx   r   r_   s      rV   r2  z DeploymentStateManager.autoscale  s4      7775&}5??@STTTrU   r  c                    g }|D ]}|j         t          j        k    rD|                    |j                   | j        |                             d|j         d           [|j         t          j        k    rC|                    |j                   | j        |                             d|j         d           |r"| j        |                             |           dS dS )zCUpdates internal datastructures when replicas fail to be scheduled.zJReplica scheduling failed. Failed to create a placement group for replica z-. See Serve controller logs for more details.zAReplica scheduling failed. Failed to create an actor for replica N)	r}  r3   PLACEMENT_GROUP_CREATION_FAILEDr  r   r  rc  ACTOR_CREATION_FAILEDrt  )rx   r   r  failed_replicasrV  s        rV   r  z:DeploymentStateManager._handle_scheduling_request_failures  s:    ,."5 	 	")1QR R  &&'9'DEEE'6UUB);)FB B B    #)1GH H  &&'9'DEEE'6UUB#5#@B B B  
  	R#M2@@QQQQQ	R 	RrU   c                    t           j                            t          t	          | j                                       d}| j                                        D ]U}|j        L|j        j        @|j        j        j	        	 .|j        j        j	        
                    dd          dk    r|dz  }Vt           j                            t          |                     d S )Nr   num_gpusrI   )r9   NUM_DEPLOYMENTSr  rU  r  r  r9  r  rp   rj   rS  NUM_GPU_DEPLOYMENTS)rx   num_gpu_deploymentsr  s      rV   r  z/DeploymentStateManager._record_deployment_usage  s    %,,ST5L1M1M-N-NOOO $ 7 > > @ @ 	) 	) ,8$0?K$0?Q %0?QUU"A    $q(#)005H1I1IJJJJJrU   r^   c                     |j         j        }|| j        vr3d|j         d}t                              d|j         d| d           dS | j        |                             |           dS )z
        Record request routing information for a replica.

        Args:
            info: Request routing info including deployment name, replica tag,
                multiplex model ids, and routing stats.
        z in application ''ry  z not found in state manager.N)r   r   r  r   r   r   r   r  )rx   r^   r   app_msgs       rV   r  z2DeploymentStateManager.record_request_routing_info  s     5 777C-*@CCCGLL}1  G      F.JJ4PPPPPrU   c                     t                      }| j                                        D ])}|                    |                                           *|S )zReturn set of node ids with running replicas of any deployment.

        This is used to determine which node has replicas. Only nodes with replicas and
        head node should have active proxies.
        )r  r  r9  r7  r  )rx   node_idsr  s      rV   r  z*DeploymentStateManager.get_active_node_ids  sU     55 $ 7 > > @ @ 	D 	DOO,@@BBCCCCrU   c                     d | j                                         D             }g }|D ]9}|D ]4}|                    |j        |j        j        |j        |j        f           5:|S )z2Get all ingress replicas info for all deployments.c                 h    g | ]/}|                                 |j                                        0S rT   )r  r  rS  )r  r  s     rV   rx  zDDeploymentStateManager.get_ingress_replicas_info.<locals>.<listcomp>   sI     !
 !
 !
 **,,!
&**,,!
 !
 !
rU   )r  r9  r  r  r   r[  r  r  )rx   ingress_replicas_listingress_replicas_infor  r  s        rV   get_ingress_replicas_infoz0DeploymentStateManager.get_ingress_replicas_info  s    !
 !
$($;$B$B$D$D!
 !
 !
 !#- 		 		H#  %,,-*4//	    %$rU   c                 f    | j                             |          }|i S |                                S )a	  Get the current rank mapping for all replicas in a deployment.
        Args:
            deployment_id: The deployment ID to get ranks for.
        Returns:
            Dictionary mapping replica_id to ReplicaRank object (with rank, node_rank, local_rank).
        )r  rS  r  rx   r   r  s      rV   r  z1DeploymentStateManager._get_replica_ranks_mapping  s8      266}EE#I::<<<rU   c                 f    | j                             |          }|dS |                                S )aD  Get the cached outbound deployments for a specific deployment.

        Args:
            deployment_id: The deployment ID to get outbound deployments for.

        Returns:
            List of deployment IDs that this deployment calls, or None if
            the deployment doesn't exist or hasn't been polled yet.
        N)r  rS  r  r'  s      rV   #get_deployment_outbound_deploymentsz:DeploymentStateManager.get_deployment_outbound_deployments"  s8      266}EE#488:::rU   )NNr  r   )8rM   rN   rO   rz   r8   r6   r
   rU  r   r   r   r   r   r  r	   r  r  r  r  r}   r  r  r   r$   r  r/   r  r  r  r  r|   r  rB   r  r   r  r   r  r%  r  r  r  r6  r7  r2  r2   r  r  r#   r  r  r   r%  rD   r  r)  rT   rU   rV   r  r    s#         04AE
 

 %
 "&c	

 ,09
 "6
 $;
  (}
 -5X,>
 
 
 
<
 
 
.'+Cy.	c49n	. . . .>'!%c' ,09' ' ' 'R$!%c$ ,09$ $ $ $L. . .0	
t 	
 	
 	
 	

 
 
 
"
	lD!344	5
 
 
 
d<+G&H    L Xn=U    Dl DxPS} D D D D)	$s)	   J)J	#J J J J (CT:U    6 37 D./	"	#   *'SX ' ' ' 'N#N (N 
	N N N N,<s <tCy < < < <L    )   )@C   $j j j j jXU| U# URV U U U UR#R "":;R R R R>K K K(Q0B Qt Q Q Q Q$	SX 	 	 	 	%4c3S6H0I+J % % % %*=)=	c;	= = = =;);	$|$	%; ; ; ; ; ;rU   r  )}r   loggingr  r   rv  rp  rX  collectionsr   r   dataclassesr   enumr   typingr   r   r	   r
   r   r   r   r   r   r   ray._commonr   	ray.actorr   ray.exceptionsr   r   r   r   	ray.server   ray.serve._privater   $ray.serve._private.autoscaling_stater   *ray.serve._private.cluster_node_info_cacher   ray.serve._private.commonr   r   r   r   r   r   r    r!   r"   r#   r$   ray.serve._private.configr%   ray.serve._private.constantsr&   r'   r(   r)   r*   r+   r,   r-   r.   "ray.serve._private.deployment_infor/   'ray.serve._private.deployment_schedulerr0   r1   r2   r3   r4   ray.serve._private.exceptionsr5   ray.serve._private.long_pollr6   r7   #ray.serve._private.storage.kv_storer8   ray.serve._private.usager9   ray.serve._private.utilsr:   r;   r<   r=   r>   r?   ray.serve._private.versionr@   ray.serve.generated.serve_pb2rA   ray.serve.schemarB   rC   rD   rE   ray.util.placement_grouprF   	getLoggerr   rH   rX   r]   r   r  r|   environrS  rd  r|  r  r  r   r   r   r   r  r  r  rP  r  r  rT   rU   rV   <module>rF     s      				       # # # # # #       ! ! ! ! ! !       B B B B B B B B B B B B B B B B B B 



 & & & & & & & & % % % % % % ! ! ! ! ! ! V V V V V V V V V V V V       + + + + + + H H H H H H K K K K K K                          7 6 6 6 6 6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 > = = = = =              H G G G G G H H H H H H H H ; ; ; ; ; ; 2 2 2 2 2 2                9 8 8 8 8 8 < < < < < <            4 3 3 3 3 3		,	-	-    4           N
 N
 N
 N
 N
 N
 N
 N
b 4 4 4 4 4 4 4 4 5RZ^^,JBOOPP  #JNN8"==! !  T,'' z~~&@#FF#M  JNNPRUVV
 <M M M&G* G* G* G* G* G* G* G*T\6 \6 \6 \6 \6 \6 \6 \6~@$ @$ @$ @$ @$ @$ @$ @$Fy" y" y" y" y" y" y" y"xl l l l l l l l^	K6 K6 K6 K6 K6 K6 K6 K6\(r	; r	; r	; r	; r	; r	; r	; r	; r	; r	;rU   