&`i.h ddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z m Z ddlmZddlmZmZddlmZddlZddlmZddlmZdd lmZdd lmZmZmZmZdd l m!Z!m"Z"m#Z#dd l$m%Z%m&Z&dd l'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9m:Z:ddl;mZ>m?Z?ddl@mAZAmBZBddlCmDZDddlEmFZFddlGmHZHddlImJZJmKZKmLZLddlMmNZNmOZOddlPmQZRejSe#ZTeOd d^de deUe8fde deVe7fde deVe:fde deVeJffd ZWeOdd!ZXeOd"d#ZYeNd$ed?Z`eN d_dAe e`dBeadCeadDead$e eFf dEZbeOddr?global_logging_configN)r _private_api serve_start)r=r>r?r@kwargss a/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/serve/api.pystartrHDsP@3><PPL!!,  c t}n+#t$rtdYdSwxYw|t ddS)zqCompletely shut down Serve on the cluster. Deletes all applications and shuts down Serve system actors. ONothing to shut down. There's no Serve application running on this Ray cluster.N)r-r2loggerinfoshutdownr/clients rGrNrNmsy#%%   +      OOts $99alphacK t}n+#t$rtdYdSwxYw|d{Vt ddS)zCompletely shut down Serve on the cluster asynchronously. Deletes all applications and shuts down Serve system actors. rKN)r-r2rLrMshutdown_asyncr/rOs rGrSrSs#%%   +         ! !!!!!!!!ts $;;returncDt}|td|S)a!Returns the deployment and replica tag from within a replica at runtime. A replica tag uniquely identifies a single replica for a Ray Serve deployment. Raises: RayServeException: if not called from within a Ray Serve deployment. Example: .. code-block:: python from ray import serve @serve.deployment class MyDeployment: def __init__(self): # Prints "MyDeployment" print(serve.get_replica_context().deployment) NzT`serve.get_replica_context()` may only be called from within a Ray Serve deployment.)r.r2)internal_replica_contexts rGget_replica_contextrWs5, =>>' $   $#rIappc^ddtttdtffd }|S)aWrap a deployment class with an ASGI application for HTTP request parsing. There are a few different ways to use this functionality. Example: FastAPI app routes are defined inside the deployment class. .. code-block:: python from ray import serve from fastapi import FastAPI app = FastAPI() @serve.deployment @serve.ingress(app) class MyFastAPIDeployment: @app.get("/hi") def say_hi(self) -> str: return "Hello world!" app = MyFastAPIDeployment.bind() You can also use a standalone FastAPI app without registering routes inside the deployment. .. code-block:: python from ray import serve from fastapi import FastAPI app = FastAPI() @app.get("/hi") def say_hi(): return "Hello world!" deployment = serve.deployment(serve.ingress(app)()) app = deployment.bind() You can also pass in a builder function that returns an ASGI app. The builder function is evaluated when the deployment is initialized on replicas. This example shows how to use a sub-deployment inside the routes defined outside the deployment class. .. code-block:: python from ray import serve @serve.deployment class SubDeployment: def __call__(self): return "Hello world!" def build_asgi_app(): from fastapi import FastAPI app = FastAPI() def get_sub_deployment_handle(): return serve.get_deployment_handle(SubDeployment.name, app_name="my_app") @app.get("/hi") async def say_hi(handle: Depends(get_sub_deployment_handle)): return await handle.remote() return app deployment = serve.deployment(serve.ingress(build_asgi_app)()) app = deployment.bind(SubDeployment.bind(), name="my_app", route_prefix="/") Args: app: the FastAPI app to wrap this class with. Can be any ASGI-compatible callable. You can also pass in a builder function that returns an ASGI app. NclsrTc Gdd}|tjstdttjjrtdtttfrtdtj rn1ttjtdGfddt }t#||S) NceZdZdZdS)9ingress..decorator..ASGIIngressDeploymentc"||_||_dSN)argsrF)selfr`rFs rG__init__zBingress..decorator..ASGIIngressDeployment.__init__s $DI"(DKKKrIN)__name__ __module__ __qualname__rbrCrIrGASGIIngressDeploymentr]s#)))))rIrfz)@serve.ingress must be used with a class.z>Classes passed to @serve.ingress may not have __call__ method.z!Failed to serialize the ASGI app.) error_msgc&eZdZfdZfdZdS)6ingress..decorator..ASGIIngressWrappercj|g|Ri|tjdt j|dS)N1)rbr FASTAPI_USEDrecordr)rar`rFrZfrozen_app_or_funcs rGrbz?ingress..decorator..ASGIIngressWrapper.__init__(sV T3D333F333*11#666%.t5GHHHHHrIcKtj|d{VtdrMtjjr|d{VdS|dSdS)N__del__)rrphasattrinspectiscoroutinefunction)rarZs rGrpz>ingress..decorator..ASGIIngressWrapper.__del__/s+3D9999999993 ***23;??*!kk$/////////// D))))) **rIN)rcrdrerbrp)rZrnsrGASGIIngressWrapperri'sR I I I I I I * * * * * * *rIrt)rrisclass ValueError issubclass collectionsabcr isinstancerrr isfunctionr#rloadsrrr")rZrfrtrnrXs` @rG decoratorzingress..decoratorsY ; ) ) ) ) ) ) ) ) (Cs## JHII I c;?3 4 4 P  cGY/ 0 0 4 )#s 3 3 37;  c " " !$   ) * * *!,!2S,OPPP""  * * * * * * * *&; * * *$ .444!!rIr_)r r rr)rXr}s` rGingressr~sG^7"7"xS *7"h7"7"7"7"7"7"r rI_func_or_classnameversion num_replicas route_prefixray_actor_optionsplacement_group_bundlesplacement_group_strategymax_replicas_per_node user_configmax_ongoing_requestsmax_queued_requestsautoscaling_configgraceful_shutdown_wait_loop_sgraceful_shutdown_timeout_shealth_check_period_shealth_check_timeout_srequest_router_configmax_constructor_retry_countc|tjurtd| td|dkr4d}t| | \} } tjddtD}|dkrtd|tjddfvr| tjdfvrtd tjurt d t|tr| }tj||nd | | | | | |||||| t!|_fd }t%|r ||n|S)a4Decorator that converts a Python class to a `Deployment`. Example: .. code-block:: python from ray import serve @serve.deployment(num_replicas=2) class MyDeployment: pass app = MyDeployment.bind() Args: _func_or_class: The class or function to be decorated. name: Name uniquely identifying this deployment within the application. If not provided, the name of the class or function is used. version: Version of the deployment. Deprecated. num_replicas: Number of replicas to run that handle requests to this deployment. Defaults to 1. route_prefix: Route prefix for HTTP requests. Defaults to '/'. Deprecated. ray_actor_options: Options to pass to the Ray Actor decorator, such as resource requirements. Valid options are: `accelerator_type`, `memory`, `num_cpus`, `num_gpus`, `resources`, and `runtime_env`. placement_group_bundles: Defines a set of placement group bundles to be scheduled *for each replica* of this deployment. The replica actor will be scheduled in the first bundle provided, so the resources specified in `ray_actor_options` must be a subset of the first bundle's resources. All actors and tasks created by the replica actor will be scheduled in the placement group by default (`placement_group_capture_child_tasks` is set to True). This cannot be set together with max_replicas_per_node. placement_group_strategy: Strategy to use for the replica placement group specified via `placement_group_bundles`. Defaults to `PACK`. max_replicas_per_node: The max number of replicas of this deployment that can run on a single node. Valid values are None (default, no limit) or an integer in the range of [1, 100]. This cannot be set together with placement_group_bundles. user_config: Config to pass to the reconfigure method of the deployment. This can be updated dynamically without restarting the replicas of the deployment. The user_config must be fully JSON-serializable. max_ongoing_requests: Maximum number of requests that are sent to a replica of this deployment without receiving a response. Defaults to 5. max_queued_requests: [EXPERIMENTAL] Maximum number of requests to this deployment that will be queued at each *caller* (proxy or DeploymentHandle). Once this limit is reached, subsequent requests will raise a BackPressureError (for handles) or return an HTTP 503 status code (for HTTP requests). Defaults to -1 (no limit). autoscaling_config: Parameters to configure autoscaling behavior. If this is set, `num_replicas` should be "auto" or not set. graceful_shutdown_wait_loop_s: Duration that replicas wait until there is no more work to be done before shutting down. Defaults to 2s. graceful_shutdown_timeout_s: Duration to wait for a replica to gracefully shut down before being forcefully killed. Defaults to 20s. health_check_period_s: Duration between health check calls for the replica. Defaults to 10s. The health check is by default a no-op Actor call to the replica, but you can define your own health check using the "check_health" method in your deployment that raises an exception when unhealthy. health_check_timeout_s: Duration in seconds, that replicas wait for a health check method to return before considering it as failed. Defaults to 30s. logging_config: Logging config options for the deployment. If provided, the config will be used to set up the Serve logger on the deployment. request_router_config: Config for the request router used for this deployment. max_constructor_retry_count: Maximum number of times to retry the deployment constructor. Defaults to 20. Returns: `Deployment` z`route_prefix` can no longer be specified at the deployment level. Pass it to `serve.run` or in the application config instead.Nz2`max_ongoing_requests` must be non-null, got None.autorkc@g|]\}}|dk |tju|S)r)r VALUE).0optionvalues rG zdeployment..s>$$$ FE % % %%w}*D*D *D*D*DrIrz)num_replicas is expected to larger than 0zQManually setting num_replicas is not allowed when autoscaling_config is provided.zDeprecationWarning: `version` in `@serve.deployment` has been deprecated. Explicitly specifying version will raise an error in the future!) rrrrrrrrrr@rrc ,tj|ddtjurndtjurndtjurndtjurnd}t tjurn|j|tjurnddS)N) init_args init_kwargsrrrrT)r _internal)rcreater rr1rc) rreplica_configdeployment_configrrrrrrs rGr}zdeployment..decorators&- %6gm%K%K!!QU+'-??(',7=@@)() ==&%)   0 --DD>3J   'w} < > .   gm## O   .-00/',,..(5%1%=\\11/-&C$?35%3$?   699U5V5V2           F)1(@(@ O99^ $ $ $iOrIT)frozenc~eZdZUdZeed<eZeed<dZ e eed<dZ e e e efed<dZeed <dS) RunTargetz;Represents a Serve application to run for `serve.run_many`.targetr/rNr@Fexternal_scaler_enabled)rcrdre__doc__r0__annotations__rrstrrr r@r rr5rboolrCrIrGrrs{FE &D#&&&"%L(3-%%%;?NHU4#678???$)T)))))rIrFtargets$wait_for_ingress_deployment_creationwait_for_applications_running_local_testing_modec |stdtr|stdd}g}|D]}t |jdkrt dt|jtstdt|j | t|j|j|j |j|rt nd|st#jjnd|j|rW|jp t+}t|t*st+di|pi}t-d d |d d |DSt/jd did}t2jd||||}|||S)zRun many applications and return the handles to their ingress deployments. This is only used internally with the _blocking not totally blocking the following code indefinitely until Ctrl-C'd. zNo applications provided.z=Overriding local_testing_mode=True from environment variable.Trz)Application name must a non-empty string.zE`serve.run` expects an `Application` returned by `Deployment.bind()`.N)rrr@make_deployment_handledefault_runtime_envr local_test-)component_name component_idr@stream_handler_onlyc2g|]}|j|jSrC)deployment_handlesingress_deployment_name)rbs rGrz_run_many..Os#TTTA$Q%>?TTTrIlocation EveryNode)r>rBv2)rr)rrC)rvrrLrMlenrr2rzrr0 TypeErrorr%rappendrr@rrayget_runtime_context runtime_envrr5rrDrEr API_VERSIONrmdeploy_applicationswait_for_proxies_serving) rrrr built_appstr@rPhandless rG _run_manyrs4 64555)#" Y KKW X X X"J    qv;;!  #$OPP P!(K00 W  an--- V^ /&('C'C*%C$;$=$=$I$I()(A       )<]__.-88 E*DDn.BDDN"') $     UTTTTT)$k2"&    !((...,, 1U*G-   ''*G (   rIr) _blockingrrr@rrrrrc Xtt|||||g||dS)zRun an application and return a handle to its ingress deployment. This is only used internally with the _blocking not totally blocking the following code indefinitely until Ctrl-C'd. )rrrr@r)rrr)rr)rrrrr@rrs rG_runresS  )-(?     '0/     rIblockingcLt||||}|rt|S)arRun many applications and return the handles to their ingress deployments. Args: targets: A sequence of `RunTarget`, each containing information about an application to deploy. blocking: Whether this call should be blocking. If True, it will loop and log status until Ctrl-C'd. wait_for_ingress_deployment_creation: Whether to wait for the ingress deployments to be created. wait_for_applications_running: Whether to wait for the applications to be running. Note that this effectively implies `wait_for_ingress_deployment_creation=True`, because the ingress deployments must be created before the applications can be running. Returns: List[DeploymentHandle]: A list of handles that can be used to call the applications. )rrr)rr&)rrrrrrs rGrun_manyrs?8-Q&C/ G NrIcPt||||||}|rt|S)aRun an application and return a handle to its ingress deployment. The application is returned by `Deployment.bind()`. Example: .. code-block:: python handle = serve.run(MyDeployment.bind()) ray.get(handle.remote()) Args: target: A Serve application returned by `Deployment.bind()`. blocking: Whether this call should be blocking. If True, it will loop and log status until Ctrl-C'd. name: Application name. If not provided, this will be the only application running on the cluster (it will delete all others). route_prefix: Route prefix for HTTP requests. Defaults to '/'. If `None` is passed, the application will not be exposed over HTTP (this may be useful if you only want the application to be exposed via gRPC or a `DeploymentHandle`). logging_config: Application logging config. If provided, the config will be applied to all deployments which doesn't have logging config. external_scaler_enabled: Whether external autoscaling is enabled for this application. Returns: DeploymentHandle: A handle that can be used to call the application. )rrrr@rr)rr&)rrrrr@rrhandles rGrunrsFL !%/ 7 F MrIcRt}||g|dS)z`Delete an application by its name. Deletes the app with all corresponding deployments. )rN)r- delete_apps)rrrPs rGdeleters0  ! !F v 22222rIbetafunc.max_num_models_per_replicac|t|stdtj|stdtj|}t |jdkst |jdkrtdttstddkrdkrtd d tffd }t|r ||n|S) a Wrap a callable or method used to load multiplexed models in a replica. The function can be standalone function or a method of a class. The function must have exactly one argument, the model id of type `str` for the model to be loaded. It is required to define the function with `async def` and the function must be an async function. It is recommended to define coroutines for long running IO tasks in the function to avoid blocking the event loop. The multiplexed function is called to load a model with the given model ID when necessary. When the number of models in one replica is larger than max_num_models_per_replica, the models will be unloaded using an LRU policy. If you want to release resources after the model is loaded, you can define a `__del__` method in your model class. The `__del__` method will be called when the model is unloaded. Example: .. code-block:: python from ray import serve @serve.deployment class MultiplexedDeployment: def __init__(self): # Define s3 base path to load models. self.s3_base_path = "s3://my_bucket/my_models" @serve.multiplexed(max_num_models_per_replica=5) async def load_model(self, model_id: str) -> Any: # Load model with the given tag # You can use any model loading library here # and return the loaded model. load_from_s3 is # a placeholder function. return load_from_s3(model_id) async def __call__(self, request): # Get the model_id from the request context. model_id = serve.get_multiplexed_model_id() # Load the model for the requested model_id. # If the model is already cached locally, # this will just be a dictionary lookup. model = await self.load_model(model_id) return model(request) Args: max_num_models_per_replica: the maximum number of models to be loaded on each replica. By default, it is 3, which means that each replica can cache up to 3 models. You can set it to a larger number if you have enough memory on the node resource, in opposite, you can set it to a smaller number if you want to save memory on the node resource. NzCThe `multiplexed` decorator must be used with a function or method.zK@serve.multiplexed can only be used to decorate async functions or methods.rzp@serve.multiplexed can only be used to decorate functions or methods with at least one 'model_id: str' argument.z.max_num_models_per_replica must be an integer.z,max_num_models_per_replica must be positive.rc@tfd}|S)Nc6Kd}|s"t|dt|}|@t|dkr"t|d}|d}n?t|dkr"t|d|}|d}d}t ||s#t |}t |||nt||}||d{VS)NzmFunctions decorated with `@serve.multiplexed` must take exactly onethe multiplexed model ID (str), but got {}zno arguments are provided.rzmore than one arguments.rr__serve_multiplex_wrapper) rformatr$rrqr4setattrgetattr load_model) r`args_check_error_msgramultiplex_objectmodel_idmultiplex_attrmodel_multiplex_wrapperrrs rG_multiplex_wrapperzEmultiplexed.._multiplex_decorator.._multiplex_wrapperGsf= ! (//0LMM/tT::D |t99>>#,334NOO$( 7t99>>#,334NOO$( 78N+^<< T*@$ :++'(.:QRRRR*12BN*S*S'0;;HEEEEEEEE ErIr)rrrs` rG_multiplex_decoratorz)multiplexed.._multiplex_decoratorFsA t' F' F' F' F' F ' FR"!rI) rrrrrs signaturer parametersrzintrvr)rrrrs ` rG multiplexedrsD@ ~~ U  *400 ( %d++ y# $ $ ) )S1E-F-F-J-J>  0# 6 6JHIII!R'',F!,K,KGHHH+"8+"+"+"+"+"+"Z*2$ Q   % % %=QQrIcVtjj}|jS)aGet the multiplexed model ID for the current request. This is used with a function decorated with `@serve.multiplexed` to retrieve the model ID for the current request. .. code-block:: python import ray from ray import serve import requests # Set the multiplexed model id with the key # "ray_serve_multiplexed_model_id" in the request # headers when sending requests to the http proxy. requests.get("http://localhost:8000", headers={"ray_serve_multiplexed_model_id": "model_1"}) # This can also be set when using `DeploymentHandle`. handle.options(multiplexed_model_id="model_1").remote("blablabla") # In your deployment code, you can retrieve the model id from # `get_multiplexed_model_id()`. @serve.deployment def my_deployment_function(request): assert serve.get_multiplexed_model_id() == "model_1" )rservecontext_get_serve_request_contextmultiplexed_model_id)_request_contexts rGget_multiplexed_model_idrvs$8y(CCEE  00rIctd}|tStjdt di|}|S)aGet the status of Serve on the cluster. Includes status of all HTTP Proxies, all active applications, and their deployments. .. code-block:: python @serve.deployment(num_replicas=2) class MyDeployment: pass serve.run(MyDeployment.bind()) status = serve.status() assert status.applications["default"].status == "RUNNING" F)raise_if_no_controller_runningNrkrC)r-r7rSERVE_STATUS_API_USEDrmr6get_serve_details _get_status)rPdetailss rGstatusrsk$ u E E EF ~}}'..s333"@@V%=%=%?%?@@G     rIct}tj|jj|}|t d|dtj d| ||dS)aNGet a handle to the application's ingress deployment by name. Args: name: Name of application to get a handle to. Raises: RayServeException: If no Serve controller is running, or if the application does not exist. .. code-block:: python import ray from ray import serve @serve.deployment def f(val: int) -> int: return val * 2 serve.run(f.bind(), name="my_app") handle = serve.get_app_handle("my_app") assert handle.remote(3).result() == 6 Nz Application 'z' does not exist.rkF check_exists) r-rget _controllerget_ingress_deployment_nameremoter2rSERVE_GET_APP_HANDLE_API_USEDrm get_handle)rrPr~s rGget_app_handlers2 ! !Fgf(DKKDQQRRG G G G GHHH/66s;;;   Wd  ? ??rIdeployment_nameapp_name _check_exists_record_telemetryc*t}t}||td|j}|rtjd||||}|!|j||j |S)a Get a handle to a deployment by name. This is a developer API and is for advanced Ray users and library developers. Args: deployment_name: Name of deployment to get a handle to. app_name: Application in which deployment resides. If calling from inside a Serve application and `app_name` is not specified, this will default to the application from which this API is called. Raises: RayServeException: If no Serve controller is running, or if calling from outside a Serve application and no application name is specified. The following example gets the handle to the ingress deployment of an application, which is equivalent to using `serve.get_app_handle`. .. testcode:: import ray from ray import serve @serve.deployment def f(val: int) -> int: return val * 2 serve.run(f.bind(), name="my_app") handle = serve.get_deployment_handle("f", app_name="my_app") assert handle.remote(3).result() == 6 serve.shutdown() The following example demonstrates how you can use this API to get the handle to a non-ingress deployment in an application. .. testcode:: import ray from ray import serve from ray.serve.handle import DeploymentHandle @serve.deployment class Multiplier: def __init__(self, multiple: int): self._multiple = multiple def __call__(self, val: int) -> int: return val * self._multiple @serve.deployment class Adder: def __init__(self, handle: DeploymentHandle, increment: int): self._handle = handle self._increment = increment async def __call__(self, val: int) -> int: return await self._handle.remote(val) + self._increment # The app calculates 2 * x + 3 serve.run(Adder.bind(Multiplier.bind(2), 3), name="math_app") handle = serve.get_app_handle("math_app") assert handle.remote(5).result() == 13 # Get handle to Multiplier only handle = serve.get_deployment_handle("Multiplier", app_name="math_app") assert handle.remote(5).result() == 10 serve.shutdown() NzcPlease specify an application name when getting a deployment handle outside of a Serve application.rkr ) r-r.r2rr$SERVE_GET_DEPLOYMENT_HANDLE_API_USEDrmr_handle_registration_callback deployment_id)rrrrrPrVrs rGget_deployment_handlers` ! !F<>> # +#2  08HG:AA#FFF%00  1  F !, $ B N >>v?STTT MrI)NNNN)TTF)FTTF)T)Nr)NTT)lrxrrlogging functoolsrtypingrrrrr r r r attrr fastapirrstarlette.typesrrrray._common.serializationrray.serve._private.build_apprray.serve._private.configrrrrray.serve._private.constantsrrrray.serve._private.http_utilrr%ray.serve._private.local_testing_moder ray.serve._private.logging_utilsrray.serve._private.usagerray.serve._private.utilsr r!r"r#r$r%r&ray.serve.configr'r(r)r*r+ray.serve.contextr,r-r.r/ray.serve.deploymentr0r1ray.serve.exceptionsr2ray.serve.handler3ray.serve.multiplexr4ray.serve.schemar5r6r7ray.util.annotationsr8r9ray.serve._privater:rD getLoggerrLrrrHrNrSrWr~rrfloatrrrrrrrrrrrrrrCrIrGr6sB MMMMMMMMMMMMMMMMMMMM&&&&&&&&###### 222222222222  ONNNNNGGGGGG222222 98888888222222------666666MMMMMMMMMM88888888222222  , - - X6:37377; %%$]23%dK/0%dK/0%$m34 %%%%P X& W&$^$$$$> XGw()GhGGGGT X)- #M7>}.5m'.}?F}-4]*1-*1-)0(/ HO 4;M29-,3M-4]@G   07 -BPBPX&BP #,BPS\BP(5c?34 BP %T *+ BP t} BP%T$sEz*:%;<BP&clBP#3<BP#'BP"#,BP!BP d,=t&C DEBP$+5>BP")BP #5>!BP"$EN#BP$E$ t";<=%BP&# d'-.'BP,")-BP.xj*$%/BPBPBPBPJ $********26*. % OO i O*.O$(O O   OOOOd X&"%;? %$)        3-  U4#678   "     <15*. % %% i %%+/%$( %  %   %%%%P X&"%;? %$)11 11 13- 1 U4#678 1  1"11111h X3333333 VQRFRFR 8CH% &FRKNFRFRFRFRR V1#1111> W! !!!!6 W @ @!1 @ @ @ @F#" iiismii i  iiiiiirI