&`i}@ddlZddlZddlmZddlmZmZmZmZm Z m Z m Z ddl m Z mZmZmZmZmZmZmZddlmZmZmZddlmZddlmZmZddlmZdd l m!Z!dd l"m#Z#m$Z$dd l%m&Z&ej'eZ(Gd d Z)GddZ*GddZ+dS)N) defaultdict)AnyCallableDictListOptionalSetTuple)RUNNING_REQUESTS_KEYApplicationName DeploymentIDHandleMetricReport ReplicaIDReplicaMetricReportTargetCapacityDirection TimeSeries))RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER&RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_SSERVE_LOGGER_NAME)DeploymentInfo)aggregate_timeseriesmerge_instantaneous_total) ServeUsageTag)"get_capacity_adjusted_num_replicas)AutoscalingContextAutoscalingPolicy)metricscFeZdZdZdefdZdededefdZde fd Z defd Z defd Z d e e fd ZdZdZdefdZdedefdZdeddfdZdeddfdZdeeddfdZdedededefdZ d1ded!edefd"Z d2d#eeee fde!fd$Z"de e#fd%Z$de e#fd&Z%de e#fd'Z&d(e e#defd)Z'defd*Z(defd+Z)defd,Z*dee e e#ffd-Z+defd.Z,deeee efffd/Z-deeee e#fffd0Z.dS)3DeploymentAutoscalingStatez,Manages autoscaling for a single deployment. deployment_idc||_t|_t|_d|_d|_d|_d|_g|_d|_ d|_ d|_ d|_ tjddd|_tjddd|_tjddd |_dS) N"serve_autoscaling_desired_replicaszmThe raw autoscaling decision (number of replicas) from the autoscaling policy before applying min/max bounds. deployment application) descriptiontag_keys serve_autoscaling_total_requestszbTotal number of requests as seen by the autoscaler. This is the input to the autoscaling decision.*serve_autoscaling_policy_execution_time_mszpTime taken to execute the autoscaling policy in milliseconds. High values may indicate a slow or complex policy.)r$r% policy_scope)_deployment_iddict_handle_requests_replica_metrics_deployment_info_config_policy _policy_state_running_replicas_target_capacity_target_capacity_direction_last_scale_up_time_last_scale_down_timerGaugeautoscaling_decision_gauge autoscaling_total_requests_gauge'autoscaling_policy_execution_time_gaugeselfr s x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/serve/_private/autoscaling_state.py__init__z#DeploymentAutoscalingState.__init__%s+ @DvvGKff $   8<2415MQ'48 6:"*1- 093 + + + '18 ./3 1 1 1 -8?} 8EC 8 8 8 444infocurr_target_num_replicasreturnc^|jj}|td|j|j|j|r|j|j}n|}||_||_|jj |_ |j |_ |j |_i|_|jjsTt"d|jjjd|jdt(jd||S)zyRegisters an autoscaling deployment's info. Returns the number of replicas the target should be set to. N-Autoscaling config is not set for deployment !Using custom autoscaling policy 'z' for deployment ''.1)deployment_configautoscaling_config ValueErrorr+r/config_changedinitial_replicasr0policy get_policyr1target_capacityr4target_capacity_directionr5r2is_default_policy_functionloggerrApolicy_functionrCUSTOM_AUTOSCALING_POLICY_USEDrecord apply_bounds)r=rArBconfigtarget_num_replicass r>registerz#DeploymentAutoscalingState.register[sD ': >U@SUU   ! )T-B-Q-QRV-W-W )%1"("9  ":  $ |*5577 $ 4*.*H'|"==?? E KK;DL4G4W;;#'#6;;;     8 ? ? D D D  !4555r@ replica_idc,||jvr |j|=dSdSN)r.)r=r[s r>on_replica_stoppedz-DeploymentAutoscalingState.on_replica_stoppeds) . . .%j111 / .r@c|jj4|jtjkrt |jj|jSt |jj|jSr])r0rMr5rUPrr4 min_replicasr=s r>get_num_replicas_lower_boundz7DeploymentAutoscalingState.get_num_replicas_lower_boundsa < ( 4  +/F/I I I5 -%  6 )% r@c@t|jj|jSr])rr0 max_replicasr4rbs r>get_num_replicas_upper_boundz7DeploymentAutoscalingState.get_num_replicas_upper_bounds"1 L %  !   r@running_replicasc||_dS)z=Update cached set of running replica IDs for this deployment.N)r3)r=rgs r>update_running_replica_idsz5DeploymentAutoscalingState.update_running_replica_idss!1r@c6tj|_dS)z2Record a scale up event by updating the timestamp.N)timer6rbs r>record_scale_upz*DeploymentAutoscalingState.record_scale_ups#'9;;   r@c6tj|_dS)z4Record a scale down event by updating the timestamp.N)rkr7rbs r>record_scale_downz,DeploymentAutoscalingState.record_scale_downs%)Y[["""r@&num_replicas_running_at_target_versionc4|||kS)zWhether or not this deployment is within the autoscaling bounds. Returns: True if the number of running replicas for the current deployment version is within the autoscaling bounds. False otherwise. )rW)r=ros r>is_within_boundsz+DeploymentAutoscalingState.is_within_boundss#   D E E5 6 r@ num_replicasct|t||S)zqClips a replica count with current autoscaling bounds. This takes into account target capacity. )maxrcminrf)r=rrs r>rWz'DeploymentAutoscalingState.apply_boundss>   - - / / 1133\ B B   r@replica_metric_reportNcx|j}|j}||jvs||j|jkr ||j|<dSdS)z8Records average number of ongoing requests at a replica.N)r[ timestampr.)r=rvr[send_timestamps r>"record_request_metrics_for_replicaz=DeploymentAutoscalingState.record_request_metrics_for_replicasV+5 .8 d3 3 3 5j A KKK0ED !* - - -LKr@handle_metric_reportcx|j}|j}||jvs||j|jkr ||j|<dSdS)zgRecords average number of queued and running requests at a handle for this deployment. N) handle_idrxr-)r=r{r}rys r>!record_request_metrics_for_handlez|j}|rd|d nd }td |d |d |d d|j d dS)Drops handle metrics that are no longer valid. This includes handles that live on Serve Proxy or replica actors that have died AND handles from which the controller hasn't received an update for too long. NrzDropping metrics for handle 'z%' because the Serve actor it was on (z) is no longer alive. It had z ongoing requestsz on actor 'z' z#Dropping stale metrics for handle 'z#because no update was received for z.1fzs. Ongoing requests was: .)rtr0metrics_interval_srlistr-itemsis_serve_component_sourceactor_idtotal_requestsrSdebugrkrxrA)r=r timeout_sr} handle_metricr actor_infos r>drop_stale_handle_metricsz4DeploymentAutoscalingState.drop_stale_handle_metricss  / / 2  )-T-B-H-H-J-J(K(K   $I}7 !*6!*2GGG))4 /!33LLY YY,9,BYY)6)EYYY}66)CC))4 /!33,5H>F!N!:h!:!:!:!:BJKKQiQQ:QQ>GPQQ1>1MQQQ/  r@decision_num_replicastotal_num_requestspolicy_execution_time_msr*c|jj|jjd}|j|||j|||j|i|d|idS)Nr#)tagsr*)r+nameapp_namer9setr:r;)r=rrrr*rs r>record_autoscaling_metricsz5DeploymentAutoscalingState.record_autoscaling_metricss-2.7   '++,A+MMM -112D41PPP 488 $+Qd+QNL+Q+Q 9     r@F_skip_bound_checkch|jtd|jd||}t j}||\}|_t j|z dz}|||j|d|r|S||S)aDecide the target number of replicas to autoscale to. The decision is based off of the number of requests received for this deployment. After the decision number of replicas is returned by the policy, it is then bounded by the bounds min and max adjusted by the target capacity and returned. If `_skip_bound_check` is True, then the bounds are not applied. Nz!Policy is not set for deployment rr$) r1rKr+get_autoscaling_contextrkr2rrrW)r=rBrautoscaling_context start_timerrs r>get_decision_num_replicasz4DeploymentAutoscalingState.get_decision_num_replicass < WATWWWXX X"::;STTY[[ 48LLAT4U4U1t1$(IKK*$<#D  '' !  2 $       )( (  !6777r@override_policy_statec||}n#|j|j}ni}tdid|jd|jjd|jjdt |jd|d|jd|jd| d | d |d tj d |j d |j d|jd|jd|jd|jS)Nr deployment_namercurrent_num_replicasrYrgrcapacity_adjusted_min_replicascapacity_adjusted_max_replicas policy_state current_timerXtotal_queued_requestsaggregated_metrics raw_metricslast_scale_up_timelast_scale_down_time)copyr2rr+rrlenr3get_total_num_requestsrcrfrkr0_get_queued_requests_get_aggregated_custom_metrics_get_raw_custom_metricsr6r7)r=rBrcurrent_policy_states r>rz2DeploymentAutoscalingState.get_autoscaling_context1s ! ,#8#=#=#?#?   +#'#5#:#:#<#< #% !   --  /44 (11 "%T%;!!_collect_replica_running_requestsz_collect_handle_queued_requestsz:DeploymentAutoscalingState._collect_handle_queued_requestsesK $($9$@$@$B$B I I  " "#7#G H H H Hr@cg}|jD]Z}|jD]P}t|jvs||jtvr%||jt|Q[|S)aCollect running requests timeseries from handles when not collected on replicas. Returns: List of timeseries data. Example: If there are 2 handles, each managing 2 replicas, and the running requests metrics are: - Handle 1: Replica 1: 5, Replica 2: 7 - Handle 2: Replica 1: 3, Replica 2: 1 and the timestamp is 0.1 and 0.2 respectively Then the returned list will be: [ [TimeStampedValue(timestamp=0.1, value=5.0)], [TimeStampedValue(timestamp=0.2, value=7.0)], [TimeStampedValue(timestamp=0.1, value=3.0)], [TimeStampedValue(timestamp=0.2, value=1.0)] ] )r-rr3r rr)r=rrr[s r> _collect_handle_running_requestsz;DeploymentAutoscalingState._collect_handle_running_requestsps&!299;;  M"4   ( 0EEE!)>?S)TTT&&!)*>? K  r@rc|sdSt|}|rM|dj}tj|z }|dkrd}t||jj|}||ndSdS)aAggregate and average a metric from timeseries data using instantaneous merge. Args: timeseries_list: A list of TimeSeries (TimeSeries), where each TimeSeries represents measurements from a single source (replica, handle, etc.). Each list is sorted by timestamp ascending. Returns: The time-weighted average of the metric Example: If the timeseries_list is: [ [ TimeStampedValue(timestamp=0.1, value=5.0), TimeStampedValue(timestamp=0.2, value=7.0), ], [ TimeStampedValue(timestamp=0.2, value=3.0), TimeStampedValue(timestamp=0.3, value=1.0), ] ] Then the returned value will be: (5.0*0.1 + 7.0*0.2 + 3.0*0.2 + 1.0*0.3) / (0.1 + 0.2 + 0.2 + 0.3) = 4.5 / 0.8 = 5.625 rgMbP?)aggregation_function last_window_s)rrxrkrr0r)r=rmerged_timeserieslast_metric_timervalues r>_merge_and_aggregate_timeseriesz:DeploymentAutoscalingState._merge_and_aggregate_timeseriess< 36oFF  704> !IKK*::M!! $ (!%)\%F+E "-553 6sr@c^|}t|dk}|}|s|}ng}g}|||s||||||}|S)a Calculate total requests using aggregate metrics mode with timeseries data. This method works with raw timeseries metrics data and performs aggregation at the controller level, providing more accurate and stable metrics compared to simple mode. Processing Steps: 1. Collect raw timeseries data (eg: running request) from replicas (if available) 2. Collect queued requests from handles (always tracked at handle level) 3. Collect raw timeseries data (eg: running request) from handles (if not available from replicas) 4. Merge timeseries using instantaneous approach for mathematically correct totals 5. Calculate time-weighted average running requests from the merged timeseries Key Differences from Simple Mode: - Uses raw timeseries data instead of pre-aggregated metrics - Performs instantaneous merging for exact gauge semantics - Aggregates at the controller level rather than using pre-computed averages - Uses time-weighted averaging over the look_back_period_s interval for accurate calculations Metrics Collection: Running requests are collected with either replica-level or handle-level metrics. Queued requests are always collected from handles regardless of where running requests are collected. Timeseries Aggregation: Raw timeseries data from multiple sources is merged using an instantaneous approach that treats gauges as right-continuous step functions. This provides mathematically correct totals without arbitrary windowing bias. Example with Numbers: Assume metrics_interval_s = 0.5s, current time = 2.0s Step 1: Collect raw timeseries from 2 replicas (r1, r2) replica_metrics = [ {"running_requests": [(t=0.2, val=5), (t=0.8, val=7), (t=1.5, val=6)]}, # r1 {"running_requests": [(t=0.1, val=3), (t=0.9, val=4), (t=1.4, val=8)]} # r2 ] Step 2: Collect queued requests from handles handle_queued = 2 + 3 = 5 # total from all handles Step 3: No handle metrics needed (replica metrics available) handle_metrics = [] Step 4: Merge timeseries using instantaneous approach # Create delta events: r1 starts at 5 (t=0.2), changes to 7 (t=0.8), then 6 (t=1.5) # r2 starts at 3 (t=0.1), changes to 4 (t=0.9), then 8 (t=1.4) # Merged instantaneous total: [(t=0.1, val=3), (t=0.2, val=8), (t=0.8, val=10), (t=0.9, val=11), (t=1.4, val=15), (t=1.5, val=14)] merged_timeseries = {"running_requests": [(0.1, 3), (0.2, 8), (0.8, 10), (0.9, 11), (1.4, 15), (1.5, 14)]} Step 5: Calculate time-weighted average over full timeseries (t=0.1 to t=1.5+0.5=2.0) # Time-weighted calculation: (3*0.1 + 8*0.6 + 10*0.1 + 11*0.5 + 15*0.1 + 14*0.5) / 2.0 = 10.05 avg_running = 10.05 Final result: total_requests = avg_running + queued = 10.05 + 5 = 15.05 Returns: Total number of requests (average running + queued) calculated from timeseries data aggregation. r)rrrrextendr)r=replica_timeseriesmetrics_collected_on_replicasqueued_timeserieshandle_timeseriesongoing_requests_timeseriesongoing_requestss r>(_calculate_total_requests_aggregate_modezCDeploymentAutoscalingState._calculate_total_requests_aggregate_modes~"CCEE(+,>(?(?!(C%!@@BB, # $ E E G G   " ')# $**+=>>>- B ' . ./@ A A A $**+<=== ?? '   r@cd}|jD]9}||jvr.||j|jtdz }:|dk}|jD]o}||jz }|sa|jD]Y}||jtivr5||jt|z }Zp|S)agCalculate total requests using simple aggregated metrics mode. This method works with pre-aggregated metrics that are computed by averaging (or other functions) over the past look_back_period_s seconds. Metrics Collection: Metrics can be collected at two levels: 1. Replica level: Each replica reports one aggregated metric value 2. Handle level: Each handle reports metrics for multiple replicas Replica-Level Metrics Example: For 3 replicas (r1, r2, r3), metrics might look like: { "r1": 10, "r2": 20, "r3": 30 } Total requests = 10 + 20 + 30 = 60 Handle-Level Metrics Example: For 3 handles (h1, h2, h3), each managing 2 replicas: - h1 manages r1, r2 - h2 manages r2, r3 - h3 manages r3, r1 Metrics structure: { "h1": {"r1": 10, "r2": 20}, "h2": {"r2": 20, "r3": 30}, "h3": {"r3": 30, "r1": 10} } Total requests = 10 + 20 + 20 + 30 + 30 + 10 = 120 Note: We can safely sum all handle metrics because each unique request is counted only once across all handles (no double-counting). Queued Requests: Queued request metrics are always tracked at the handle level, regardless of whether running request metrics are collected at replicas or handles. Returns: Total number of requests (running + queued) across all replicas/handles. r)r3r.rrr r-raggregated_queued_requests)r=ridrrr[s r>%_calculate_total_requests_simple_modez@DeploymentAutoscalingState._calculate_total_requests_simple_mode)s Z(  BT***$"7";"N"R"R(!##)7(:%"299;; * *M mF FN0 *"&"8**J!]%E%I%I,b&&'-*J*N*N0++#j//*r@c`tr|S|S)aGet average total number of requests aggregated over the past `look_back_period_s` number of seconds. If there are 0 running replicas, then returns the total number of requests queued at handles This code assumes that the metrics are either emmited on handles or on replicas, but not both. Its the responsibility of the writer to ensure enclusivity of the metrics. )rrrrbs r>rz1DeploymentAutoscalingState.get_total_num_requestsns1 5 @@@BB B==?? ?r@ctt}|jD]b}||jvrW|j|jrE|j|jD] \}}|||!c|S)z!Get the raw replica metrics dict.)rrr3r.rrr)r= metric_valuesrkvs r>get_replica_metricsz.DeploymentAutoscalingState.get_replica_metrics~s#D)) ( / /BT***t/DR/H/P* 1"5=CCEE//DAq!!$++A....r@ctr-|}|sdS||Std|jDS)zCalculate the total number of queued requests across all handles. Returns: Sum of queued requests at all handles. Uses aggregated values in simple mode, or aggregates timeseries data in aggregate mode. rc3$K|] }|jV dSr])r).0rs r> zBDeploymentAutoscalingState._get_queued_requests..s9!8r@)rrrsumr-r)r=rs r>rz/DeploymentAutoscalingState._get_queued_requestss 5  $ D D F F $ s778IJJ J%)%:%A%A%C%C r@ctt}|jD]_}|j|}||jD]&\}}||g}||||<'`t|S)a4Aggregate custom metrics from replica metric reports. This method aggregates raw timeseries data from replicas on the controller, similar to how ongoing requests are aggregated. Returns: Dict mapping metric name to dict of replica ID to aggregated metric value. )rr,r3r.rrrr)r=rr[rv metric_name timeseriesaggregated_values r>rz9DeploymentAutoscalingState._get_aggregated_custom_metricss)..0 O OJ$($9$=$=j$I$I !$,+@+H+N+N+P+P O O' Z#'#G#G #U#U >N";/ ;; O &'''r@ctt}|jD]I}|j|}||jD]\}}||||<Jt|S)zExtract raw custom metric values from replica metric reports. Returns: Dict mapping metric name to dict of replica ID to raw metric timeseries. )rr,r3r.rrr)r=rr[rvrrs r>rz2DeploymentAutoscalingState._get_raw_custom_metricss"$'' 0 B BJ$($9$=$=j$I$I !$,+@+H+N+N+P+P B B' Z7A K(44 BK   r@Fr])/__name__ __module__ __qualname____doc__r r?rintrZrr^rcrfrrirlrnrqrWrrzrr~r strrfloatrboolrrrrrrrrrrrrrrrrrrrr@r>rr"s664 l4 4 4 4 l"6^"6s"6s"6"6"6"6H2Y2222 c     c    24 ?2222///111  s               F%8 F  F F F F D0 D  D D D D's3x'D''''R " " #(       $HM88(+8@D8 8888F;?   (S#X7       @4 3C( j1A     $z2B    D6j)6 6666p] %] ] ] ] ~CuCCCCJ@@@@@ T)T*5E*E%Fe*(S$y%?O:P5P0Q((((.! c4 :-.. /!!!!!!r@rc eZdZdZdefdZedZdefdZ de fdZ d e d e d edefd Zd e fd Zd e fdZdeee eeefffdZ d&dee efde dee effdZd e deefdZd e fdZd e fdZdefdZd e defdZd e fdZ d e dede fdZ!de"fd Z#d!e$fd"Z%d#e&efd$Z'd%S)'ApplicationAutoscalingStatez-Manages autoscaling for a single application.rc>||_i|_d|_d|_dSr]) _app_name_deployment_autoscaling_statesr1r2r=rs r>r?z$ApplicationAutoscalingState.__init__s6"  +  BFr@c4|jSr])rkeysrbs r> deploymentsz'ApplicationAutoscalingState.deploymentss277999r@autoscaling_policyc||_i|_|sLtd|jd|jdtj ddSdS)aRegister or update application-level autoscaling config and deployments. This will overwrite the deployment-level policies with the application-level policy. Args: autoscaling_policy: The autoscaling policy to register. rFz' for application 'rGrHN) rOr1r2rRrSrArTrrrUrV)r=rs r>rZz$ApplicationAutoscalingState.registers*4466 "<<>> E KK74F4V77$(N777     8 ? ? D D D D D  E Er@rCc|jduSr])r1rbs r> has_policyz&ApplicationAutoscalingState.has_policys|4''r@r rArBcj||jvrt||j|<|jjt d||jjjs2|rt d|d|j| ||S)z4Register a single deployment under this application.NrEzUUser provided both a deployment-level and an application-level policy for deployment z4. The application-level policy will take precedence.) rrrIrJrKrNrRrrSwarningrZ)r=r rArBs r>register_deploymentz/ApplicationAutoscalingState.register_deployments  C C C+=99  /   ! 4 <O OO &9@[[]] !!  NNEhuEEE    2=AJJ  $   r@c||jvr td|ddS|j|dS)Nz3Cannot deregister autoscaling state for deployment z because it is not registered)rrSrpopr<s r>deregister_deploymentz1ApplicationAutoscalingState.deregister_deploymentsX  C C C NNrmrrr    F +// >>>>>r@c||jvSr])rr<s r>should_autoscale_deploymentz7ApplicationAutoscalingState.should_autoscale_deployment&s CCCr@rc "|dSt|ts Jd|D]V}||jvs Jd|t||ts#Jd|dt ||WdS)z`Validate that the returned policy_state from an application-level policy is correctly formatted.NzcApplication-level autoscaling policy must return policy_state as Dict[DeploymentID, Dict[str, Any]]z-Policy state contains invalid deployment ID: zPolicy state for deployment z must be a dictionary, got ) isinstancer,rrtype)r=rr s r>_validate_policy_statez2ApplicationAutoscalingState._validate_policy_state)s   F $   q q p q q *..00 | |M!DDDDN}NNEDD]+T | |{m{{X\]ijw]xXyXy{{ | | |  | |r@F!deployment_to_target_num_replicasrc4r[fdjD}tj}|\}}tj|z dz}||_t|tus Jd| D]+}|jvsJd|d|vsJd|d,i} |D]\\}} j|} | | ||j |ds j| | n| | |<]| SfdjDS) z Decide scaling for all deployments in this application by calling each deployment's autoscaling policy. c i|]D\}}|||jrj|iniESr)rr2r)rr staterr=s r> zIApplicationAutoscalingState.get_decision_num_replicas..Hso$$$)M5 u<<5mD)D&**="===   $$$r@rzWAutoscaling policy must return a dictionary of deployment_name -> decision_num_replicasz Deployment z is not registeredz is invalidr%cRi|]#\}}|||$S))rBr)r)rr deployment_autoscaling_staterrs r>r zIApplicationAutoscalingState.get_decision_num_replicas..~sZ@M#? ;UU-N%.'8 V  r@) rrrrkr1rr2rr,rrrrW) r=rrautoscaling_contextsr decisionsreturned_policy_staterr resultsrrrs ``` r>rz5ApplicationAutoscalingState.get_decision_num_replicas=s7 ??  @ $$$$$-1,O,U,U,W,W$$$ J04||

> >!6D Y4'''h(''"+!1!1 < < !T%HHHHBBBBIHH"%FFFF;;;;GFFFG/8/@/@  + |/3/R!0,-GG (7J,! -&D7 FSS$&  &&NDHCfClClCnCn r@rgcF|j||dSr])rri)r=r rgs r>riz6ApplicationAutoscalingState.update_running_replica_idss1 +M:UU      r@cZ||jvr!|j|dSdS)z)Record a scale up event for a deployment.N)rrlr<s r>rlz+ApplicationAutoscalingState.record_scale_ups9 D? ? ?  / > N N P P P P P @ ?r@cZ||jvr!|j|dSdS)z+Record a scale down event for a deployment.N)rrnr<s r>rnz-ApplicationAutoscalingState.record_scale_downs9 D? ? ?  / > P P R R R R R @ ?r@r[cj|j}||jvr"|j||dSdSr])r rr^)r=r[dep_ids r>r^z.ApplicationAutoscalingState.on_replica_stoppedsB) T8 8 8  / 7 J J: V V V V V 9 8r@c@|j|Sr])rrr<s r>%get_total_num_requests_for_deploymentzAApplicationAutoscalingState.get_total_num_requests_for_deployments#2  " " #r@c@|j|Sr])rrr<s r>$get_replica_metrics_by_deployment_idz@ApplicationAutoscalingState.get_replica_metrics_by_deployment_ids2=AUUWWWr@rocB|j||Sr])rrq)r=r ros r>rqz,ApplicationAutoscalingState.is_within_boundss'2=ARR 2   r@rvct|jj}||jvr"|j||dSdSr])r[r rrz)r=rvrs r>rzz>ApplicationAutoscalingState.record_request_metrics_for_replicasM'1? T8 8 8  / 001FGGGGG 9 8r@r{cj|j}||jvr"|j||dSdSr])r rr~)r=r{rs r>r~z=ApplicationAutoscalingState.record_request_metrics_for_handlesH&3 T8 8 8  / //0DEEEEE 9 8r@rch|jD]}||dS)rN)rrr)r=r dep_states r>rz5ApplicationAutoscalingState.drop_stale_handle_metricssH<CCEE G GI  / /0E F F F F G Gr@Nr)(rrrrr r?propertyrrrZrrr rrrrrrrrrrrrrrirlrnr^rrrrqrrzrr~r rrr@r>rrs77F!FFFF$::X:E-EEEE.(D(((( #  #&      B?<????DDDDD|$T,S#X*F%GH||||.#(II+/ c0A+BI I lC IIIIV ) =A)_    Q\QQQQ S|SSSS WYWWWW #)# ####X,XXXX ) SV      H%8 H H H HF$6FFFFGs3xGGGGGGr@rceZdZdZdZdedededefdZdefdZ d e d e fd Z d e fd Z d e defd Zd e deeefdeeeffdZd e fdZdefdZdedeefdZdefdZdefdZdefdZdedeeeeffdZdedefdZdededefdZdeddfdZ de!ddfd Z"d!e#e$ddfd"Z%dS)#AutoscalingStateManagerzManages all things autoscaling related. Keeps track of request metrics for each application and its deployments, and decides on the target number of replicas to autoscale to. ci|_dSr]_app_autoscaling_statesrbs r>r?z AutoscalingStateManager.__init__s  $$$r@r rArBrCc|jjsJ|j}|j|t |}t d|||||S)z%Register autoscaling deployment info.z-Registering autoscaling state for deployment ) rIrJrr& setdefaultrrSrAr)r=r rArBr app_states r>rz+AutoscalingStateManager.register_deployments%8888 )0;; 1(;;    SMSSTTT,, 4!9   r@c|j|j}|r4td|||dSdS)z Remove deployment from tracking.z/Deregistering autoscaling state for deployment N)r&rrrSrArr=r r)s r>rz-AutoscalingStateManager.deregister_deploymentsj044]5KLL  ; KKQ-QQ     + +M : : : : :  ; ;r@rrc|j|t|}td|||dS)Nz.Registering autoscaling state for application )r&r(rrSrArZ)r=rrr)s r>register_applicationz,AutoscalingStateManager.register_applicationsb 0;; 1(;;    OXOOPPP-.....r@c||jvr:td||j|ddSdS)z!Remove application from tracking.z0Deregistering autoscaling state for application N)r&rSrArrs r>deregister_applicationz.AutoscalingStateManager.deregister_applicationsR t3 3 3 KKU8UU V V V  ( , ,Xt < < < < < 4 3r@cR||jvo|j|Sr])r&rrs r>_application_has_policyz/AutoscalingStateManager._application_has_policys/ 4 4 D,X6AACC r@rcB|j||S)aQ Decide scaling for all deployments in the application. Args: app_name: The name of the application. deployment_to_target_num_replicas: A dictionary of deployment_id to target number of replicas. Returns: A dictionary of deployment_id to decision number of replicas. )r&r)r=rrs r>rz1AutoscalingStateManager.get_decision_num_replicas s'+H5OO -   r@c||jvSr]r%rs r>should_autoscale_applicationz4AutoscalingStateManager.should_autoscale_applications4777r@ch|j|jvo$|j|j|Sr])rr&rr<s r>rz3AutoscalingStateManager.should_autoscale_deployment!s9  "d&B B 9,&))-88  r@rgcx|j|j}|r|||dSdSr])r&rrri)r=r rgr)s r>riz2AutoscalingStateManager.update_running_replica_ids)sO044]5KLL  R  0 0@P Q Q Q Q Q R Rr@cv|j|j}|r||dSdS)zRecord a scale up event for a deployment. Args: deployment_id: The ID of the deployment being scaled up. N)r&rrrlr+s r>rlz'AutoscalingStateManager.record_scale_up0sI 044]5KLL  5  % %m 4 4 4 4 4 5 5r@cv|j|j}|r||dSdS)zRecord a scale down event for a deployment. Args: deployment_id: The ID of the deployment being scaled down. N)r&rrrnr+s r>rnz)AutoscalingStateManager.record_scale_down:sI 044]5KLL  7  ' ' 6 6 6 6 6 7 7r@r[c|j|jj}|r||dSdSr])r&rr rr^)r=r[r)s r>r^z*AutoscalingStateManager.on_replica_stoppedDsJ044Z5M5VWW  5  ( ( 4 4 4 4 4 5 5r@cl|j|jvr%|j|j|SiSr])rr&rr<s r>get_metrics_for_deploymentz2AutoscalingStateManager.get_metrics_for_deploymentIsA  !T%A A A/&22=AA BIr@cl|j|jvr%|j|j|SdS)Nr)rr&rr<s r>rz=AutoscalingStateManager.get_total_num_requests_for_deploymentSsA  !T%A A A/&33MBB C1r@rocR|j|j}|||Sr])r&rrq)r=r ror)s r>rqz(AutoscalingStateManager.is_within_bounds]s201GH )) A   r@rvNc|j|jjj}|r||dSdSr])r&rr[r rrz)r=rvr)s r>rzz:AutoscalingStateManager.record_request_metrics_for_replicaesZ044 ! , : C    P  8 89N O O O O O P Pr@r{c|j|jj}|r||dSdS)z,Update request metric for a specific handle.N)r&rr rr~)r=r{r)s r>r~z9AutoscalingStateManager.record_request_metrics_for_handlensW 044 . 7    N  7 78L M M M M M N Nr@rch|jD]}||dSr])r&rr)r=rr)s r>rz1AutoscalingStateManager.drop_stale_handle_metricsysF5<<>> G GI  / /0E F F F F G Gr@)&rrrrr?r rrrrr rr-r/rr1rrr4rrrrirlrnr^rr;rrrqrrzrr~r rrrr@r>r#r#s   #  #&      ";<;;;; /! /. / / / /=====   D     ! ,0 c0A+B  lC     &8_8888     R)R=A)_RRRR5\55557|77775Y5555 ) ij)) *)  ) SV     P%8P PPPP N0 N  N N N NGs3xGDGGGGGGr@r#),loggingrk collectionsrtypingrrrrrr r ray.serve._private.commonr r r rrrrrray.serve._private.constantsrrr"ray.serve._private.deployment_infor ray.serve._private.metrics_utilsrrray.serve._private.usagerray.serve._private.utilsrray.serve.configrrray.utilr getLoggerrSrrr#rr@r>rMsD ######BBBBBBBBBBBBBBBBBB                     >=====322222GGGGGGBBBBBBBB  , - -e !e !e !e !e !e !e !e !PAGAGAGAGAGAGAGAGHmGmGmGmGmGmGmGmGmGmGr@