&`i tNddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlmZddlmZmZmZmZmZmZmZmZddlmZmZddlmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,dd l-m.Z.m/Z/ej0e*Z1e/d GddZ2e/d GddeZ3dZ4e/d Gdde5eZ6e/d GddeZ7e/d GddeZ8e.Gdde5eZ9e/d Gdde5eZ:e/d GddeZ;e/d Gd d!eZ+.   ! !       & " ) *<&"-/M+.L+     ! !) returnc`t|jr|S|jSr<)callabler?rAs rBr2z%AutoscalingContext.aggregated_metrics0 D2 3 3 41133 3--rDc`t|jr|S|jSr<)rGr@rHs rBr3zAutoscalingContext.raw_metricss0 D+ , , -**,, ,&&rDc`t|jr|S|jSr<)rGr=rHs rBr0z%AutoscalingContext.total_num_requestsrIrDc`t|jr|S|jSr<)rGr>rHs rBr1z(AutoscalingContext.total_queued_requestss0 D5 6 6 74466 600rDc |j|jz Sr<)r0r1rHs rBtotal_running_requestsz)AutoscalingContext.total_running_requestss&)CCCrDN)__name__ __module__ __qualname____doc__rstrr intrrr floatrrrrrCrr2r3r0r1propertyrNrDrBr)r))s  M#MM3- M " M ! My/M"%"e))<"<=M (eXb%i5H.H(IJM% S$y%/001T#tIu,<'="=>>?A  M  S$y*4556T#tIz,A'B"BCCDF  !M,),-M.),/M038n1M2%UO3M4'uo5M6uo7M8 9MMMM^.HT#tIu5:E6 M555"M6;U7 W666#]Y&t444  54 1c3h111111$L-2L LLLL $(IN [38E?333388880(rDrYg$@ceZdZdZdZdZdS)AggregationFunctionmeanmaxminN)rOrPrQMEANMAXMINrWrDrBrrNs D C CCCrDrceZdZUedZeed<eedZ e e e fed<fdZ ded d fd Zed eded dfd Zd eefdZddZd efdZd e fdZxZS)AutoscalingPolicyrDrZ_serialized_policy_defzPolicy function can be a string import path or a function callable. If it's a string import path, it must be of the form `path.to.module:function_name`. r]policy_functionc |dd}tjdi||r ||_dS|dSNrrW)rurvrCrserialize_policy)rArsserialized_policy_defrys rBrCzAutoscalingPolicy.__init__`sc & +CT J J""6"""  $*?D ' ' '  ! ! # # # # #rDrrENc||_dSr<r)rArs rBset_serialized_policy_defz+AutoscalingPolicy.set_serialized_policy_defhs&;###rD policy_configcD|}||d<|di|Srr)rorrr:s rBfrom_serialized_policy_defz,AutoscalingPolicy.from_serialized_policy_defks2##%%+@'(s}}V}}rDc|jSr<rrHs rBget_serialized_policy_defz+AutoscalingPolicy.get_serialized_policy_defss **rDc>|j}t|tr|jd|j}|jsat |\}}tj|| tj |tj |||_dS)zSerialize policy with cloudpickle. Import the policy if it's passed in as a string import path. Then cloudpickle the policy and set `serialized_policy_def` if it's empty. rhN) rrirrPrOrrr rrrlr)rA policy_path policy_modulers rBrz"AutoscalingPolicy.serialize_policyvs * k8 , , M(3LLk6JLLK* B-CK-P-P *M?  0 ? ? ?  * *;+<_+M+M N N N  2= A A A*rDc"|jtkSr<)rrrHs rBis_default_policy_functionz,AutoscalingPolicy.is_default_policy_functions#'FFFrDc tj|jS#ttf$r}t d|d|d}~wwxYw)z+Deserialize policy from cloudpickled bytes.z1Failed to deserialize custom autoscaling policy: a This typically happens when the policy depends on external modules that aren't available in the current environment. To fix this: - Ensure all dependencies are installed in your Docker image or environment - Package your policy as a Python package and install it - Place the policy module in PYTHONPATH For more details, see: https://docs.ray.io/en/latest/serve/advanced-guides/advanced-autoscaling.html#gotchas-and-limitationsN)r rrrrrs rB get_policyzAutoscalingPolicy.get_policysp $T%@AA A#[1   DADDD    rr)rOrPrQrrrjrr rrr rSrrCrrrrr rrboolrrrrs@rBrrUs[%0K$<$<$<E<<<,1E/`---OU3=) $$$$$ [+8E?++++++++&GDGGGGHrDrstablecDeZdZUdZdZeed<dZeeed<dZ e ed<e Z ee ed<eed Ze ed <ed d Ze ed <edd Ze ed<edd Zee ed<edd Zee ed<edd Zee ed<edd Zee ed<edd Zeed<edd Zeeed<ed d Zeed<eejd Zeeefed!<ee d"#Z!e ed$<e"dd%&d'Z#e"d d(e d)e fd*Z$e"d!d%&d(eeeffd+Z%e&d,Z'd)e fd-Z(d)e fd.Z)d)e fd/Z*dS)0AutoscalingConfigz Config for the Serve Autoscaler. min_replicasNinitial_replicas max_replicastarget_ongoing_requestsz[DEPRECATED] How often to scrape for metrics. Will be replaced by the environment variables `RAY_SERVE_REPLICA_AUTOSCALING_METRIC_PUSH_INTERVAL_S` and `RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S` in a future release.r]metrics_interval_sg>@z(Time window to average over for metrics.look_back_period_sg?z8[DEPRECATED] Smoothing factor for autoscaling decisions.smoothing_factorz3[DEPRECATED] Please use `upscaling_factor` instead.upscale_smoothing_factorz5[DEPRECATED] Please use `downscaling_factor` instead.downscale_smoothing_factorz:Multiplicative "gain" factor to limit upscaling decisions.upscaling_factorz>>-9>>>   ',.. J\JJ5EJJJ"L00 J\JJ5EJJJ rDrprEcP|tkrtjdt|S)NzThe `metrics_interval_s` field in AutoscalingConfig is deprecated and will be replaced by the environment variables `RAY_SERVE_REPLICA_AUTOSCALING_METRIC_PUSH_INTERVAL_S` and `RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S` in a future release.)DEFAULT_METRICS_INTERVAL_SwarningswarnDeprecationWarningrorps rB&metrics_interval_s_deprecation_warningz8AutoscalingConfig.metrics_interval_s_deprecation_warnings4 * * * M]#    rDct|tr|Stt|Sr<)rirrSlowerrs rBaggregation_function_validz,AutoscalingConfig.aggregation_function_valid s6 a, - - H"3q66<<>>222rDc(|tddS)Nrd)rrr)r )ros rBr[zAutoscalingConfig.defaults$s$C    rDc:|jr|jS|jp|jSr<)rrrrHs rBget_upscaling_factorz&AutoscalingConfig.get_upscaling_factors'   )( (,E0EErDc:|jr|jS|jp|jSr<)rrrrHs rBget_downscaling_factorz(AutoscalingConfig.get_downscaling_factors'  " +* *.G$2GGrDc|jSr<)rrHs rBget_target_ongoing_requestsz-AutoscalingConfig.get_target_ongoing_requests%s ++rD)+rOrPrQrRrrrrr rrr rrr rrrrrrrrrrrrrrrr rSrrrrrrrr[rrrrWrDrBrrs** $%L.$$$15h~.555 !L+!!!7VXm4VVV(-*U))) )."L))) ',eN'''m 9>"W999h}5;@%K;;; 7 16P111h}-38%R333/+0%^+++' =BEL===x(89)."P)))O%=BE#(N===%%8 89 !&)]!!!F  Y~d+++,+.Y#$$ }    %$ Y%d333356I1I+J333433   [ FmFFFF H HHHH ,],,,,,,rDrceZdZdZdZdZdS)DeploymentModeNoServerHeadOnly EveryNodeN)rOrPrQrrrrWrDrBrr*sHHIIIrDrceZdZdZdZdZdZedede fde fdZ ede ee e fde dfd Z d S) ProxyLocationaConfig for where to run proxies to receive ingress traffic to the cluster. Options: - Disabled: don't run proxies at all. This should be used if you are only making calls to your applications via deployment handles. - HeadOnly: only run a single proxy on the head node. - EveryNode: run a proxy on every node in the cluster that has at least one replica actor. This is the default. Disabledrrproxy_locationrEct|trt|}n5t|ts tdt |d|tjkr t jSt |jS)Nz'Must be a `ProxyLocation` or str, got: rh) rirSrrmtyperrrvalue)rors rB_to_deployment_modez!ProxyLocation._to_deployment_modeBs nc * * *>::NNNM:: Q$~:N:NQQQ  ]3 3 3!* *!."677 7rDdeployment_modec|dSt|trt|}n5t|ts tdt |d|tjkr t jSt |jS)zConverts DeploymentMode enum into ProxyLocation enum. DeploymentMode is a deprecated version of ProxyLocation that's still used internally throughout Serve. Nz(Must be a `DeploymentMode` or str, got: rh) rirSrrmrrrrr)rors rB_from_deployment_modez#ProxyLocation._from_deployment_modeRs  "4  - - ,_==OOO^<< S4;P;PSSS  n5 5 5 ) ) !677 7rDN)rOrPrQrRrrrrr rSrrr rrWrDrBrr1s  HHI 8"?C#78 8  8 8 8[ 88&u^S-@'AB8 / "888[888rDrceZdZUdZeZeeed<e Z e ed<gZ e eed<ejZeeed<dZe ed<dZeed <dZeed <d Zeeed <eZe ed <d Zeeed<d Zeeed<d Zeeed<d Zeeed<edddZeddZedddZ edddZ!GddZ"d S) HTTPOptionsaHTTP options for the proxies. Supported fields: - host: Host that the proxies listens for HTTP on. Defaults to "127.0.0.1". To expose Serve publicly, you probably want to set this to "0.0.0.0". - port: Port that the proxies listen for HTTP on. Defaults to 8000. - root_path: An optional root path to mount the serve application (for example, "/prefix"). All deployment routes are prefixed with this path. - request_timeout_s: End-to-end timeout for HTTP requests. - keep_alive_timeout_s: Duration to keep idle connections alive when no requests are ongoing. - ssl_keyfile: Path to the SSL key file for HTTPS. If provided with ssl_certfile, the HTTP server will use HTTPS. - ssl_certfile: Path to the SSL certificate file for HTTPS. If provided with ssl_keyfile, the HTTP server will use HTTPS. - ssl_keyfile_password: Optional password for the SSL key file. - ssl_ca_certs: Optional path to CA certificate file for client certificate verification. - location: [DEPRECATED: use `proxy_location` field instead] The deployment location of HTTP servers: - "HeadOnly": start one HTTP server on the head node. Serve assumes the head node is the node you executed serve.start on. This is the default. - "EveryNode": start one HTTP server per node. - "NoServer": disable HTTP server. - num_cpus: [DEPRECATED] The number of CPU cores to reserve for each internal Serve HTTP proxy actor. hostport middlewareslocationrnum_cpusroot_url root_pathNrequest_timeout_skeep_alive_timeout_s ssl_keyfile ssl_certfilessl_keyfile_password ssl_ca_certsTrec2|d| tjS|S)Nr)rrrorprs rBlocation_backfill_no_serverz'HTTPOptions.location_backfill_no_servers &> !QY!* *rDcP|d}t|||S)Nr)rr#)rorprrs rBvalidate_ssl_certfilez!HTTPOptions.validate_ssl_certfiles(jj// A{+++rDc2|rtjd|S)NzPassing `middlewares` to HTTPOptions is deprecated and will be removed in a future version. Consider using the FastAPI integration to configure middlewares on your deployments: https://docs.ray.io/en/latest/serve/http-guide.html#fastapi-http-deploymentsrrrs rBwarn_for_middlewaresz HTTPOptions.warn_for_middlewaress+   M_    rDc2|rtjd|S)NzXPassing `num_cpus` to HTTPOptions is deprecated and will be removed in a future version.rrs rBwarn_for_num_cpuszHTTPOptions.warn_for_num_cpuss*   M/   rDceZdZdZdZdS)HTTPOptions.ConfigTN)rOrPrQvalidate_assignmentarbitrary_types_allowedrWrDrBConfigr s""&rDr)#rOrPrQrRrrr rSrrrrTrrrrrrrrrrrUr!rrrrrrrrrr rrWrDrBrrksB,D(3-+++!D#!!!Kc)7)@Hh~&@@@HcHcIs)-x--- D#DDD!%K#%%%"&L(3-&&&*.(3-..."&L(3-&&&Yz$'''(' Y~ Y}T***+*Yz$'''('''''''''''rDrc~eZdZUdZeZeed<gZe e ed<dZ e e ed<ede efdZdS) gRPCOptionsagRPC options for the proxies. Supported fields: Args: port (int): Port for gRPC server if started. Default to 9000. Cannot be updated once Serve has started running. Serve must be shut down and restarted with the new port instead. grpc_servicer_functions (List[str]): List of import paths for gRPC `add_servicer_to_server` functions to add to Serve's gRPC proxy. Default to empty list, which means no gRPC methods will be added and no gRPC server will be started. The servicer functions need to be importable from the context of where Serve is running. request_timeout_s: End-to-end timeout for gRPC requests. rgrpc_servicer_functionsNrrEcg}|jD]r} t|}t|r||n|d}t |L#t $r}|d}t ||d}~wwxYw|S)zReturn a list of callable functions from the grpc_servicer_functions. If the function is not callable or not found, it will be ignored and a warning will be logged. zQ is not a callable function! Please make sure the function is imported correctly.z can't be imported! Please make sure there are no typo in those functions. Or you might want to rebuild service definitions if .proto file is changed.N)rrrGappendrnr)rA callablesfunc imported_funcmessagerqs rBgrpc_servicer_func_callablez'gRPCOptions.grpc_servicer_func_callables 0 : :D : +D 1 1 M**.$$]3333 >>>%W--- 4' : : :=== *'229  :sAA A: A55A:)rOrPrQrRrrrTrrrrSrr rUrVrrrWrDrBrrs  "D#!!!)+T#Y+++)-x--- T(^XrDr)=rkloggingrenumr functoolsrtypingrrrrr r rayr ray._common.pydantic_compatr r rrrrrrray._common.utilsrrray.serve._private.commonrrrray.serve._private.constantsrrrrrrrr r!r"ray.serve._private.utilsr#ray.util.annotationsr$r% getLoggerloggerr)rYrrSrrrrrrrrWrDrBr&s %%%%%%================                    BAAAAAAAJIIIIIIIII                        98888866666666  , - - WzDzDzDzDzDzDzDzDz W`````)```F" W#t  WDDDDD DDDN XH,H,H,H,H, H,H,H,X S$   X6868686868C686868r XS'S'S'S'S')S'S'S'l W/////)/////rD