&`i~ddlZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z ddl mZddlmZmZmZmZmZmZmZddlZddlmZdd lmZmZmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)ej*e$Z+Gd dZ,eGdde-Z.Gdde/e Z0e GddZ1e GddZ2e GddZ3e GddZ4deeeeeffdeeeffdZ5GddeZ6Gd d!e6Z7dS)"N)ABCabstractmethod) defaultdict) dataclass)Enum)total_ordering)AnyCallableDictListOptionalSetTuple)ClusterNodeInfoCache)CreatePlacementGroupRequest DeploymentID ReplicaID) ReplicaConfig)(RAY_SERVE_HIGH_PRIORITY_CUSTOM_RESOURCES)RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY&RAY_SERVE_USE_PACK_SCHEDULING_STRATEGYSERVE_LOGGER_NAME)LabelMatchExpressionsTNodeAffinitySchedulingStrategyNodeLabelSchedulingStrategy PlacementGroupSchedulingStrategyceZdZdZdS) SpreadDeploymentSchedulingPolicyz;A scheduling policy that spreads replicas with best effort.N)__name__ __module__ __qualname____doc__{/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/serve/_private/deployment_scheduler.pyrr$sEEDr$rcbeZdZUeZeeed<dZdeffd Z dZ dZ dZ dZ d ZxZS) ResourcesCUSTOM_PRIORITYg& .>keyct|}||S|tjjrdSdS)Nr)superget startswithray_rayletIMPLICIT_RESOURCE_PREFIX)selfr)val __class__s r%r-z Resources.get0sIggkk# ?J >>#+> ? ? 1qr$cttz}tfd|DS)Nc3K|]8}|jz|kV9dSN)r-EPSILON.0kotherr2s r% z$Resources.can_fit..?sCLL!488A;;-1=LLLLLLr$setkeysallr2r<r@s`` r%can_fitzResources.can_fit<sT499;;#ejjll"3"33LLLLLtLLLLLLr$cttz}tfd|DS)Nchg|].}||k/Sr#r-r9s r% z$Resources.__eq__..Cs2>>>ADHHQKK599Q<</>>>r$r>rBs`` r%__eq__zResources.__eq__AsT499;;#ejjll"3"33>>>>>>>>???r$ct|t|z}t}|D]}|tjjr=td||||z||<c||||z||<t|S)N?) r?r@dictr.r/r0r1minr-r')r2r<r@kwargsr)s r%__add__zResources.__add__Es499;;#ejjll"3"33 = =C~~ckBCC =!#txx}}uyy~~'EFFs "hhsmmeiinn<s    r$cttz}fd|D}t|S)Nchi|].}|||z /Sr#rF)r:r)r<r2s r% z%Resources.__sub__..Ss4FFF##txx}}uyy~~5FFFr$)r?r@r')r2r<r@rMs`` r%__sub__zResources.__sub__QsX499;;#ejjll"3"33FFFFFFFF   r$ct|t|z}|hdz }|jD]`}||||krdS||||krdSa|d|dkrdS|d|dkrdS|d|dkrdS|d|dkrdS|d|dkrdS|d|dkrdS|t|jz D]`}||||krdS||||krdSadS)aDetermines priority when sorting a list of SoftResources. 1. Custom resources defined in RAY_SERVE_HIGH_PRIORITY_CUSTOM_RESOURCES (sorted by priority) 2. GPU 3. CPU 4. memory 5. Other custom resources This means a resource with a larger number of high-priority resources is always sorted higher than one with fewer, regardless of other types. >CPUGPUmemoryTFrUrTrV)r?r@r(r-)r2r<r@ custom_keysr)s r%__lt__zResources.__lt__Vs499;;#ejjll"3"335555 '  Cxx}}uyy~~--tt#3//uu0 88E??UYYu-- - -4 XXe__uyy// / /5 88E??UYYu-- - -4 XXe__uyy// / /5 88H   ( 3 3 3 34 XXh  %))H"5"5 5 55T%9!:!::  Cxx}}uyy~~--tt#3//uu0ur$)rr r!rr(r str__annotations__r8r-rCrHrNrRrX __classcell__)r4s@r%r'r'*s"JOT#YIIIG s      MMM @@@ ! ! !!!! )))))))r$r'c"eZdZdZdZdZdZdZdS)ReplicaSchedulingRequestStatusz+The status of a replica scheduling request. IN_PROGRESS SUCCEEDEDACTOR_CREATION_FAILEDPLACEMENT_GROUP_CREATION_FAILEDN)rr r!r"r^r_r`rar#r$r%r]r]s,55KI3&G###r$r]ceZdZUdZeed<ejjed<e ed<e ed<e ed<e ed<e j Ze ed<d Zeee eefed <d Zeeed <d Zeeed <ed efdZd S)ReplicaSchedulingRequestzRequest to schedule a single replica. The scheduler is responsible for scheduling based on the deployment scheduling policy. replica_id actor_defactor_resources actor_optionsactor_init_args on_scheduledstatusNplacement_group_bundlesplacement_group_strategymax_replicas_per_nodereturnc,|j6|jdkr+td|jDtSt|j}|j;|jj}tj j |j d|j }d|jz ||<|S)aThe resources required to schedule this replica on a node. If this replica uses a strict pack placement group, the required resources is the sum of the placement group bundles. Otherwise, required resources is simply the actor resources. N STRICT_PACKc,g|]}t|Sr#r'r:bundles r%rGz?ReplicaSchedulingRequest.required_resources..s NNNv6""NNNr$:rJ) rkrlsumr'rfrmrd deployment_idr/r0r1app_namename)r2requiredrwimplicit_resources r%required_resourcesz+ReplicaSchedulingRequest.required_resourcess  ( 4->>NN1MNNN   !!566H )5 $ = {;E$-EE0=0BEE"/2D4N.N*+Or$)rr r!r"rrZr/actor ActorClassr rr r]r^rjrkr r rYfloatrlrmintpropertyr'r|r#r$r%rcrcs y####-K-WF *WWWAEXd4U +;&<=DDD.2hsm222+/8C=/// IXr$rcc(eZdZUdZeed<eed<dS)DeploymentDownscaleRequestz{Request to stop a certain number of replicas. The scheduler is responsible for choosing the replicas to stop. rw num_to_stopN)rr r!r"rrZrr#r$r%rrs6  r$rceZdZUeed<eed<dZeeed<dZ ee eed<dZ ee ed<dZ eeed<edefd Zdefd ZdS) DeploymentSchedulingInforwscheduling_policyNrfrkrlrmrnc|j,|jdkr!t|jtS|j}|jr9t jj|j j d|j j }d|jz ||<|S)a The resources required to schedule a replica of this deployment on a node. If this replicas uses a strict pack placement group, the required resources is the sum of the placement group bundles. Otherwise, required resources is simply the actor resources. NrprurJ) rkrlrvr'rfrmr/r0r1rwrxry)r2rzr{s r%r|z+DeploymentSchedulingInfo.required_resourcess  ( 4->>t3Y[[AA A+H ) O{;O)2OO595G5LOO"/2D4N.N*+Or$c*|jduo |jdkS)Nrp)rkrl)r2s r%is_non_strict_pack_pgz.DeploymentSchedulingInfo.is_non_strict_pack_pgs"  ( 4 ?-> r$)rr r!rrZr rfr r'rkr rlrYrmrrr|boolrr#r$r%rrs+/OXi(///9=Xd9o6===.2hsm222+/8C=/// IX8 t      r$rcXeZdZUdZdZeeed<dZee ee fed<dS)LaunchingReplicaInfoaDescribes a replica for which a schedule request has been sent to core but has not been scheduled (placed on a node) yet. Args: target_node_id: The exact node that's been requested for this replica. This is best effort and may not be fulfilled. target_labels: The node labels that have been requested for this replica. This is best effort and may not be fulfilled. Ntarget_node_id target_labels) rr r!r"rr rYrZrr r r#r$r%rrsO%)NHSM(((.2M8DcN+22222r$rdeployment_to_replicasrnc>d|DS)zKFlattens a dict of {deployment_id: {replica_id: val}} to {replica_id: val}.cHi|]}|D]\}}|| Sr#)items)r:replicasrdr3s r%rQz_flatten..sO    '~~//   J C    r$)values)rs r%_flattenr s/   .5577   r$c :eZdZdZdededefdZdede dd fd Z ded e dd fd Z dedd fd Z dedd fdZdededd fdZdedd fdZ d#dedeedeeeeffdZ d$deedeeeeffdZdeeeffdZdedeeefdeefdZedeeeefdeeefdeeeeffdZ d#dededeedee fd Z!ed!e"dee#ee$ffd"Z%d S)%DeploymentSchedulerz{A centralized scheduler for all Serve deployments. It makes a batch of scheduling decisions in each update cycle. cluster_node_info_cache head_node_idcreate_placement_group_fnci|_tt|_tt|_tt |_tt|_||_||_ ||_ dSr7) _deploymentsrrK_pending_replicas_launching_replicasr?_recovering_replicas_running_replicas_cluster_node_info_cache _head_node_id_create_placement_group_fn)r2rrrs r%__init__zDeploymentScheduler.__init__szKM         %0$4$4!"-T!2!2(?%)*C'''r$rwrrnNc||jvsJ||jvsJ||jvsJ||jvsJt |||j|<dS)z,Called whenever a new deployment is created.)rwrN)rrrrrr)r2rwrs r%on_deployment_createdz)DeploymentScheduler.on_deployment_created=s| D$:::::D$<<<<<D$=====D$:::::+C';L, , , -(((r$replica_configc||jvsJ|j|}t|j|_|j|_|jrd|jD|_|jr|j|_dSdS)Nc,g|]}t|Sr#rrrss r%rGz>DeploymentScheduler.on_deployment_deployed..Vs-,,,&, &!!,,,r$)rr' resource_dictrfrmrkrl)r2rwrinfos r%on_deployment_deployedz*DeploymentScheduler.on_deployment_deployedKs  11111 /()EFF%3%I"  1 ,,0>0V,,,D (  2 T,:,SD ) ) ) T Tr$cf|j|rJ|j|d|j|rJ|j|d|j|rJ|j|d|j|rJ|j|d|j|=dS)z(Called whenever a deployment is deleted.N)rpoprrrr)r2rws r%on_deployment_deletedz)DeploymentScheduler.on_deployment_deleted\s)-8888 ""=$777+M::::  $$]D999,];;;; !%%mT:::)-8888 ""=$777  m , , ,r$rdc|j}|j||d|j||d|j|||j||ddS)z6Called whenever a deployment replica is being stopped.N)rwrrrrdiscardrr2rdrws r%on_replica_stoppingz'DeploymentScheduler.on_replica_stoppingls"0  }-11*dCCC  /33JEEE !-088DDD }-11*dCCCCCr$node_idc|j}||j|vsJ|j||d|j||||j||<dS)zECalled whenever a deployment replica is running with a known node id.N)rwrrrrrr)r2rdrrws r%on_replica_runningz&DeploymentScheduler.on_replica_runningtsu"0 !7 !FFFFF  /33JEEE !-088DDDR)? ? ?  / ;;;r$c\tt}|rA|j|D] \}}|||!nT|jD]:\}}|D] \}}|||!;|Sr7)rr?rrr)r2rwresrdr_rs r%_get_node_to_running_replicasz1DeploymentScheduler._get_node_to_running_replicass#  1'+'=m'L'R'R'T'T - -# GG   ,,,, - $5;;== 1 1 8+3>>+;+;11'JL$$Z00001 r$c |j}|j}d|D d|D |jD]L\}}|j|}|D]%}|j}|r| vr |xx|jzcc<&M|j D]C\}}|j|}|D]}| vr |xx|jzcc<Ddtdtfd fd|j DS)aGets current available resources per node. This returns a conservative view of the available resources currently in the cluster. It returns the minimum of: 1. The available resources per node fetched and cached from the GCS every control loop. 2. The remaining resources left over on each node after subtracting the resources taken up by running (already scheduled by core) and launching (to-be-scheduled and soft targeting that node) replicas. Note that (1) may not be accurate because it uses cached info and there is a delay from fetching data from GCS, and (2) may not be accurate because there can be other actors (not replicas) running in the cluster, and launching replicas may not end up on the node we're targeting. So the information returned from this method is only best effort. c4i|]\}}|t|Sr#rr)r:rrs r%rQzIDeploymentScheduler._get_available_resources_per_node..s$XXXjgqGYq\\XXXr$c4i|]\}}|t|Sr#rr)r:r resourcess r%rQzIDeploymentScheduler._get_available_resources_per_node..s6   " Yy))   r$abc$t|t|z}t}|D];}t||||||<<|Sr7)r?r@r'rLr-)rrr@rr)s r% custom_minzIDeploymentScheduler._get_available_resources_per_node..custom_minsnqvvxx==3qvvxx==0D++C 7 7quuSzz155::66CJr$c i|]O}||t|tPSr#)r-r')r:rrgcs_infototal_minus_replicass r%rQzIDeploymentScheduler._get_available_resources_per_node..se    ZZ Wikk22$(()++>>   r$) r get_available_resources_per_nodeget_total_resources_per_noderrrrrr|rr'get_active_node_ids) r2available_resourcestotal_resourcesrwr deploymentrrrrrrs @@@r%!_get_available_resources_per_nodez5DeploymentScheduler._get_available_resources_per_nodes,  ) J J L L 7TTVVXX";; %  44/ 2 J!9'5/=fE      KjKKK3R#) #C "48### !MM  '"@&T### !MM  &"=m### "N &8&FGG  3 ?-//-/ k*)?? + &;7A )AA,9,>AA  ?-7?$7(8:LL      KjKKK L L L.D  % FF   "= 1* = "" ~] #    )+K L L B1AO$B$L!'' o'VVVVVs#6A""9BB?!E!!9FFallow_new_compactionct)z;Returns a node ID to be compacted and a compaction deadlne.rr2rs r%get_node_to_compactz'DeploymentScheduler.get_node_to_compacts "!r$)NNr7)&rr r!r"rrYr rrrrrrrrrrrr r r rrrr'rrrr rcrrrrrrrrr#r$r%rrsX D!5DD$, DDDD@  #  <        T#T&T  TTTT"-<-D---- DiDDDDDDDYDDDDDDA AdAAAA)-26     !    S#X/     7;  %l3  c3y>! "    A 4Y3GA A A A F"+BFsI~BV #4"|T*B%CCD"'AAB" lC N* + """^"()-:> pWpW4pW&)pW! pW  67 pWpWpWpWd"$(" %U # $"""^"""r$rc eZdZdeeeefdeeefdeeee ffdZ dede dee fdZ de d eee fdeefd Zd edeeeeffd Zd S)DefaultDeploymentSchedulerrrrncl|D]%}|D] }|j}|j}||j||<!&t d|jD}t rtjdtdtr|stt|jdd}|D]G}| |j|} ||d| HnZ|jD]@} | st#| D]}||d Ai} |D]*} || j| j| | j<+| S) a8Called for each update cycle to do batch scheduling. Args: upscales: a dict of deployment name to a list of replicas to schedule. downscales: a dict of deployment name to a downscale request. Returns: The IDs of replicas to stop for each deployment. c3>K|]}|VdSr7)r)r:ds r%r=z6DefaultDeploymentScheduler.schedule..s?( ( *+A # # % %( ( ( ( ( ( r$zThe environment variable 'RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY' is deprecated and will be removed in a v2.55.0 release. Please use 'RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY' instead.) stacklevelc|jSr7)r|)rs r%z5DefaultDeploymentScheduler.schedule..s a2r$T)r)reverseDEFAULT)rrSPREAD)rr)rrdrwranyrrwarningswarnDeprecationWarningrsortedr_find_best_available_noder|rrlist_get_replicas_to_stopr) r2rrupscalerrdrwnon_strict_pack_pgs_existall_scheduling_requests target_nodepending_replicasdeployment_to_replicas_to_stop downscales r%rz#DefaultDeploymentScheduler.schedulesb (( W WG&- W W"/: * 8 DV&}5jAA W %(( ( /3/@/G/G/I/I( ( ( % % ! 5  MO#      2 :S '-/00779922''' #'>  ""<<&9::<< &&&09#.' %)$:$A$A$C$C   '*./?/F/F/H/H*I*I&**+=4<+ *,&#**,,  I**')> +'   .-r$rwmax_num_to_stopcDt}tj|j|j|}|D].}||t||kr|cS/}tj | }| tt}|D] \} } || | !fd} t| | D]C\} } | |vr || D]0} || t||kr|ccS1D|S)arPrioritize replicas running on a node with fewest replicas of all deployments. This algorithm helps to scale down more intelligently because it can relinquish nodes faster. Note that this algorithm doesn't consider other non-serve actors on the same node. See more at https://github.com/ray-project/ray/issues/20599. ch|djkrt|dn tjS)Nrr+)rlensysmaxsize)0node_and_num_running_replicas_of_all_deploymentsr2s r%r)z=DefaultDeploymentScheduler._get_replicas_to_stop..keys@DAF%&&DQGHHH[  r$)r))r?unionrr@rrrrrrrrrrappendr )r2rwrreplicas_to_stop%pending_launching_recovering_replicas$pending_launching_recovering_replica+node_to_running_replicas_of_all_deploymentsordered_running_replicas-ordered_running_replicas_of_target_deploymentrdreplica_node_idr)rrs` r%rz0DefaultDeploymentScheduler._get_replicas_to_stops 5514  "= 1 6 6 8 8  $] 3 8 8 : :  %m 41 1 -3 ( ( 0  !E F F F#$$77''''8  . . 0 0 4 $((>}(M(S(S(U(U#V#V  ((***    6,D   'J 9/ J Q Q          ! 7 = = ? ?S   , ,JGQKKKLGT , ,  $$Z000'((O;;++++++< ,  r$r|available_resources_per_nodec|fd|D}fd|D}|||}|r|S|||}|r|SdS)zChooses best available node to schedule the required resources. If there are available nodes, returns the node ID of the best available node, minimizing fragmentation. Prefers non-idle nodes over idle nodes. c i|];\}}t|tdk8||.&sQ   +//??@@1DD SDDDr$c i|];\}}t|tdk8||.+sQ   +//??@@AEE SEEEr$N)rrr)r2r|r'non_idle_nodes idle_nodesrr-s @r%r z4DefaultDeploymentScheduler._find_best_available_nodes$(#E#E#G#G     < B B D D        < B B D D   ))*&:! # !!!!F$( %U # $r$r)8rloggingrr abcrr collectionsr dataclassesrenumr functoolsrtypingr r r r r rrr/*ray.serve._private.cluster_node_info_cacherray.serve._private.commonrrrray.serve._private.configrray.serve._private.constantsrrrrray.util.scheduling_strategiesrrrr getLoggerrrrKr'rYr]rcrrrrrrr#r$r%r?s  ##############!!!!!!$$$$$$BBBBBBBBBBBBBBBBBB KKKKKK 433333   , - -        TTTTTTTTnHHHHHS$HHH 3333333 3l   ) ) ) ) ) ) )  ) X  3 3 3 3 3 3 3  3  tIsN/C!CD  )S.    o"o"o"o"o"#o"o"o"d uuuuu!4uuuuur$