%`ihdZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z m Z ddlmZddlmZmZmZmZddlZddlZddlZddlmcmcmZddlmcmZddl m!Z!ddl"m#Z#m$Z$dd l%m&Z&m'Z'ej(e)Z*e$j+Z+e d Gd d Z,e d GddZ-e d GddZ.e d GddZ/Gdde Z0e1a2ej3Z4e5a6ej3Z7de8de8fdZ9de8dee8fdZ:de8fdZ;de8fdZ< dPde+de8d ee!fd!Z=dPde8de8d ee!fd"Z>de8fd#Z?de8fd$Z@d%ZAd&ZBd'ZCd(ZDejjEjFGeCd)ZHd*ZId+ZJde0fd,ZKdeLfd-ZMdeLfd.ZNd/ZOd0eLfd1ZPd2eLddfd3ZQdQd4ZRdQd5ZSd e!d0eLde5fd6ZTdeeUfd7ZVdPdeeUfd8ZWdee8fd9ZXdee8fd:ZYd e!dee8e8ffd;ZZd e!de-fd<Z[d e!de-fd=Z\de8fd>Z]d?e8de,fd@Z^d e!de5fdAZ_d ej`j!deLfdBZade.fdCZbdDe,dEeUdFeUdGeUdHe8dIe8de.fdJZcdKe.dLe8de/fdMZdGdNdOZedS)RaThis is the module that is in charge of Ray usage report (telemetry) APIs. NOTE: Ray's usage report is currently "on by default". One could opt-out, see details at https://docs.ray.io/en/master/cluster/usage-stats.html. # noqa Ray usage report follows the specification from https://docs.ray.io/en/master/cluster/usage-stats.html#usage-stats-collection # noqa # Module The module consists of 2 parts. ## Public API It contains public APIs to obtain usage report information. APIs will be added before the usage report becomes opt-in by default. ## Internal APIs for usage processing/report The telemetry report consists of 5 components. This module is in charge of the top 2 layers. Report -> usage_lib --------------------- Usage data processing -> usage_lib --------------------- Data storage -> Ray API server --------------------- Aggregation -> Ray API server (currently a dashboard server) --------------------- Usage data collection -> Various components (Ray agent, GCS, etc.) + usage_lib (cluster metadata). Usage report is currently "off by default". You can enable the report by setting an environment variable RAY_USAGE_STATS_ENABLED=1. For example, `RAY_USAGE_STATS_ENABLED=1 ray start --head`. Or `RAY_USAGE_STATS_ENABLED=1 python [drivers with ray.init()]`. "Ray API server (currently a dashboard server)" reports the usage data to https://usage-stats.ray.io/. Data is reported every hour by default. Note that it is also possible to configure the interval using the environment variable, `RAY_USAGE_STATS_REPORT_INTERVAL_S`. To see collected/reported data, see `usage_stats.json` inside a temp folder (e.g., /tmp/ray/session_[id]/*). N)asdict dataclass)Enumauto)Path)DictListOptionalSet) GcsClient)gcs_pb2 usage_pb2)_internal_kv_initialized_internal_kv_putT)initceZdZUdZeeed<dZeeed<dZ eeed<dZ eeed<dZ ee eed<dS)ClusterConfigToReportNcloud_provider min_workers max_workershead_node_instance_typeworker_node_instance_types) __name__ __module__ __qualname__rr str__annotations__rintrrrr o/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_common/usage/usage_lib.pyrrMs}$(NHSM(((!%K#%%%!%K#%%%-1Xc]1116:c 3:::::r rcxeZdZUdZeeed<dZeeed<dZee ed<dZ ee ed<dS)ClusterStatusToReportNtotal_num_cpustotal_num_gpustotal_memory_gbtotal_object_store_memory_gb) rrrr$r rrr%r&floatr'rr r!r#r#Vsc$(NHSM((($(NHSM((('+OXe_+++48 (5/88888r r#ceZdZUdZeed<eed<eed<dZeeed<dZ eeed<dZ eeed<dZ eeed <dZ eeed <dZ eeed <dZeeed <dZeeed <dZeeed<dZeeed<dZeeed<dZeeed<dZeeed<dZeeeed<dZeeed<dZeeed<dZeeed<dZeeed<dZeeeed<dZeeeefed<dZeeed<dZ eeed<dZ!eeed<dZ"eeeed<dS)UsageStatsToReportzUsage stats to reportschema_versionsourcecollect_timestamp_msN total_success total_failed seq_number ray_versionpython_version session_id git_commitossession_start_timestamp_msrrrrrr$r%r&r'library_usagesextra_usage_tagstotal_num_nodestotal_num_running_jobs libc_versionhardware_usages)#rrr__doc__rrrr.r r/r0r1r2r3r4r5r6rrrrrr r$r%r&r(r'r7r8rr9r:r;r<rr r!r*r*^s= KKK#'M8C='''"&L(3-&&& $J $$$!%K#%%%$(NHSM((( $J $$$ $J $$$B 04 444$(NHSM(((!%K#%%%!%K#%%%-1Xc]1116:c 3:::$(NHSM((($(NHSM((('+OXe_+++48 (5/888*.NHT#Y'...26htCH~.555%)OXc])))-1HSM000"&L(3-&&&+/OXd3i(/////r r*c2eZdZUdZeed<eed<eed<dS)UsageStatsToWritezUsage stats to write to `USAGE_STATS_FILE` We are writing extra metadata such as the status of report to this file. usage_statssuccesserrorN)rrrr=r*rboolrrr r!r?r?s: $### MMM JJJJJr r?cJeZdZeZeZeZdS)UsageStatsEnablednessN)rrrrENABLED_EXPLICITLYDISABLED_EXPLICITLYENABLED_BY_DEFAULTrr r!rErEs4$&&r rEset_namevaluec &tsJ t||dtjdS#t $r.}t d|d|d|Yd}~dSd}~wwxYw)Nr  namespacezFailed to add z to usage set , )rrencodeusage_constantUSAGE_STATS_NAMESPACE Exceptionloggerdebug)rIrJes r!_add_to_usage_setrVs # % %%% %L  ' ' ) ) $:AACC      LLL JeJJ8JJqJJKKKKKKKKKLsAA B"#B  Breturnc t}||tj}|D]A}|d}||t|dB|S#t$r8}t d|d|tcYd}~Sd}~wwxYw)NrLutf-8zFailed to get usage set rN) setinternal_kv_keysrOrPrQdecodeaddlenrRrSrT) gcs_clientrIresultusagesusagerUs r!_get_usage_setrcs ,, OO  $:AACC-   / /ELL))E JJuS]]__- . . . .  ???A??@@@uu sBB C%-CCC library_usagec:ttj|dSN)rVrPLIBRARY_USAGE_SET_NAMErds r!_put_library_usagerisn;]KKKKKr hardware_usagec:ttj|dSrf)rVrPHARDWARE_USAGE_SET_NAMErjs r!_put_hardware_usagernsnA;c tj|}|}tj}|||||dSt sJt |||dS#t$r(}t d|Yd}~dSd}~wwxYw)NrLzFailed to put extra usage tag, ) rPEXTRA_USAGE_TAG_PREFIXrOrQinternal_kv_putrrrRrSrT)rorJr_valrMrUs r!rwrws <6===DDFFllnn"8??AA  !  & &sC9 & E E E E E,-- - -- S# ; ; ; ; ; ; <<< :q::;;;;;;;;;ttdSrf)rrrr r!put_pre_init_usage_statsr?s  """"$$$$$r ct5tadddn #1swxYwYt5t addddS#1swxYwYdSrf)rrZrrtdictrurr r!reset_global_staterDs '))#&55 ))))))))))))))) (,,%)VV",,,,,,,,,,,,,,,,,,s#''AAAc,tjddS)NRAY_USAGE_STATS_REPORT_URLzhttps://usage-stats.ray.io/)r5getenvrr r!_usage_stats_report_urlrPs 913P Q QQr cFttjddS)N!RAY_USAGE_STATS_REPORT_INTERVAL_Sirr5rrr r!_usage_stats_report_interval_srVs ry>>>>>>>?&..$88 '4 / /$77 ' 3 K0H0J0J K K*H K K   ! 33sB'C*B8, C8B<<C?B<C D DC;;Dc>tjdko dtjvS)Nz{{RAY_COMMIT_SHA}}dev)r __commit__ __version__rr r!is_nightly_wheelrs >1 1 Nes6NNr c6ttjuSrf)rrErGrr r!usage_stats_enabledrs # % %-B-V VVr cNttjdddkS)NRAY_USAGE_STATS_PROMPT_ENABLEDrrrr r!usage_stats_prompt_enabledrs" ry93?? @ @A EEr ray_init_clusterctjj\}}|||d}t r|tjtjttj dzdtjdkrLtj \}}|s|ddin|d|d|i|S)zReturn a dictionary of cluster metadata. Params: ray_init_cluster: Whether the cluster is started by ray.init() Returns: A dictionary of cluster metadata. )r1r2r)r4r5r6linuxr;NA:) rrutilscompute_version_inforupdatersysplatformrtimelibc_ver)rr1r2metadatalibvers r!_generate_cluster_metadatars#&,"4"I"I"K"KK#(,H B!nl.1$)++2D.E.E      <7 " "!*,,JS# B 67777C# @AAA Or clictsdSddlm}|r|jnt}t }|t jur|tjdS|t j ur|s|tj dS|j r| dtj dd}t| t|n4#t $r'}t"d|Yd}~nd}~wwxYw|r|tjdS|tjdS|tjdS|t jusJ||r tjn tjdS)Nr) cli_loggerFT )_default _timeout_sz:Failed to persist usage stats choice for future clusters: )r"ray.autoscaler._private.cli_loggerrprintrrErGrPUSAGE_STATS_DISABLED_MESSAGErH3USAGE_STATS_ENABLED_BY_DEFAULT_FOR_RAY_INIT_MESSAGE interactiveconfirm USAGE_STATS_CONFIRMATION_MESSAGE#set_usage_stats_enabled_via_env_var"set_usage_stats_enabled_via_configrRrSrT#USAGE_STATS_ENABLED_FOR_CLI_MESSAGE.USAGE_STATS_ENABLED_BY_DEFAULT_FOR_CLI_MESSAGErF(USAGE_STATS_ENABLED_FOR_RAY_INIT_MESSAGE)rr prompt_printusage_stats_enablednessenabledrUs r!show_usage_stats_promptrs  % ' '======'*5:##L688"7"KKK ^@AAAAA $9$L L L  LR      #  ((? )G 0 8 8 8 27;;;;    TQRTT  J ^OPPPPP ^HIIIII LM     '*?*RRRRR  IN > >H     s;C C<C77C<c 8i} tt5}tj|}dddn #1swxYwYt |t s,t dt|i}n?#t$rYn3t$r'}t d|Yd}~nd}~wwxYw||d< tj tj tdttd5}tj||ddddS#1swxYwYdS#t$r2}td|rdnd d |rd nd d t|d}~wwxYw)Nz7Invalid ray config file, should be a json dict but got zFailed to load ray config file r@T)exist_okwz Failed to enabledisablez( usage stats by writing {"usage_stats": truefalsez} to )rrrr isinstancerrSrTtyperrRr5makedirsrdirnamedump)rrrrUs r!rrsd F < *,, - - "Yq\\F " " " " " " " " " " " " " " "&$''  LLX$v,,XX   F      <<< :q::;;;;;;;;<$F=  BGOO$<$>$>??$OOOO *,,c 2 2 !a Ifa  ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !  ,"1xx  , ,!-vvg , , ())  , ,   s}B A B AB AAB C  C "CC AE-E EEEEE F'-FFc@|rdndtjtj<dS)Nrr)r5environrPr)rs r!rrs!DK=TSSQTBJ~9:::r ct|}|tjt j|dtj|S)aFGenerate the cluster metadata and store it to GCS. It is a blocking API. Params: gcs_client: The GCS client to perform KV operation PUT. ray_init_cluster: Whether the cluster is started by ray.init() Raises: gRPC exceptions: If PUT fails. Returns: The cluster metadata. )rT) overwriterM) rr{rPCLUSTER_METADATA_KEYrdumpsrO ray_constantsKV_NAMESPACE_CLUSTER)r_rrs r!put_cluster_metadatar sa*;KLLLH+ 8##%%4  Or c( |dd}d}|D]-}|js$|jjds|dz }.|S#t $r(}td|Yd}~dSd}~wwxYw)zNReturn the total number of running jobs in the cluster excluding internal onesT)skip_submission_job_info_fieldskip_is_running_tasks_fieldr _ray_internalrz7Failed to query number of running jobs in the cluster: N) get_all_job_infovaluesis_deadr ray_namespace startswithrRrSinfo)r_r`r:job_inforUs r!$get_total_num_running_jobs_to_reportr"s ,,+/T-  "#  , ,H# ,HO,I,T,T-- ,'!+&%%  QaQQRRRtttttsAA B)B  Bc ||tjjj}t |S#t$r(}t d|Yd}~dSd}~wwxYw)z5Return the total number of alive nodes in the cluster)timeout state_filterz0Failed to query number of nodes in the cluster: N) get_all_node_infor GcsNodeInfo GcsNodeStateALIVEr^rrRrSr)r_rr`rUs r!#get_total_num_alive_nodes_to_reportr4s--'*=*J*P.  6<<>>"""  JqJJKKKtttttsA A BA;;BcPtt|tjSrf)listrcrPrgr_s r!get_library_usages_to_reportr@s z>+PQQ R RRr cPtt|tjSrf)rrcrPrlrs r!get_hardware_usages_to_reportrDs z>+QRR S SSr ct}tjdd}|r |dd}|D]}|d\}}|||< n4#t $r'}t d|Yd}~nd}~wwxYwdt D} | tj tj } || tj }|D]Y\} } | d} | t%tj d} | |vsJ| d|| <Zn4#t $r'}t d |Yd}~nd}~wwxYw|S) aGet the extra usage tags from env var and gcs kv store. The env var should be given this way; key=value;key=value. If parsing is failed, it will return the empty data. Params: gcs_client: The GCS client. Returns: Extra usage tags as kv pairs. RAY_USAGE_STATS_EXTRA_TAGSN;=z*Failed to parse extra usage tags env var: c6g|]}|Sr)rs).0tag_keys r! z2get_extra_usage_tags_to_report..`s CCC'gmmooCCCr rLrYz.Failed to get extra usage tags from kv store: )rr5rstripsplitrRrSrrqkeysr[rPrzrOrQinternal_kv_multi_getrr\r^) r_r8extra_usage_tags_env_varkvskvrrrUvalid_tag_keysrrorJs r!get_extra_usage_tags_to_reportr%Hs5vv!y)EtLLJ J*0055;;C@@C ( (xx}}1&' ## ( J J J KKHQHH I I I I I I I I JDCV[[]]CCCNJ**  1 8 8 : :$:AACC+   - - N@GGII.  ((** : :JC**W%%Cc.?@@BBCC.(((($)LL$9$9 S ! !  : JJJ HQHHIIIIIIIIJ s1A A22 B#<BB# C0F;; G,G''G,cddlm}t} ||j}|}t |dd|_t |dd|_d}|dd|z|_ |dd|z|_ n4#t$r'}t d|Yd }~nd }~wwxYw|S#|ccYSxYw) a+ Get the current status of this cluster. A temporary proxy for the autoscaler v2 API. It is a blocking API. Params: gcs_client: The GCS client. Returns: The current cluster status or empty ClusterStatusToReport if it fails to get that information. r)get_cluster_statusCPUGPU>memoryobject_store_memory'Failed to get cluster status to report N)ray.autoscaler.v2.sdkr'r#addresstotal_resourcesrrvr$r%r&r'rRrSr)r_r'r`cluster_statusr0to_GiBrUs r! _get_cluster_status_to_report_v2r3ss9988888 " $ $F++J,>??(88:: #O$7$7q$A$A B B #O$7$7q$A$A B B!0!4!4Xq!A!AF!J    5q 9 9F B ++ CCC AaAABBBBBBBBC v s0B2C C? C:C50C?5C::C??Dc ddlm}|rt|S|tjjjd}|stSt}d}tj | d}d|vs d|dvrtS|dd}d |vr t|d d |_d |vr t|d d |_d |vr|d d |z|_d |vr|d d |z|_|S#t$$r5}t&d|tcYd}~Sd}~wwxYw)zGet the current status of this cluster. It is a blocking API. Params: gcs_client: The GCS client to perform KV operation GET. Returns: The current cluster status or empty if it fails to get that information. r)is_autoscaler_v2NrLr*rYload_metrics_reportrbr(rr)r+r,r-)ray.autoscaler.v2.utilsr5r3internal_kv_getrrrDEBUG_AUTOSCALING_STATUSrOr#rloadsr\rr$r%r&r'rRrSr)r_r5r1r`r2rbrUs r!get_cluster_status_to_reportr;s&'<<<<<<     @3J?? ?#33 L & ? F F H H4   +(** *&((N$9$9'$B$BCC ! 7 7n-BCCC(** *45g> E>>$'e Q$8$8F ! E>>$'e Q$8$8F ! u  %*8_Q%7&%@F " E ) )+,Q/&8  / ''' AaAABBB$&&&&&&&&'s1E A E .AE B E F *FF F cdtdttttfdtfd}d}ddi}d}d d i}d }d}|||rd S|||rd S|||rdSdS)NurlheadersrWc tj||d}|jdkrdSnI#tjj$rYn3t $r'}t d|Yd}~nd}~wwxYwdS)Nr)r>rTzBUnexpected exception when making cloud provider metadata request: F)requestsrv status_code exceptionsConnectionErrorrRrSr)r=r>resrUs r!cloud_metadata_requestz@get_cloud_from_metadata_requests..cloud_metadata_requests ,sGQ???C#%%t&"2    D    KKXUVXX         us"'A- A-A((A-z?http://169.254.169.254/metadata/instance?api-version=2021-12-13Metadatarz2http://metadata.google.internal/computeMetadata/v1zMetadata-FlavorGooglez(http://169.254.169.254/latest/meta-data/azuregcpawsunknown)rr rrC)rFAZURE_METADATA_URLAZURE_METADATA_HEADERSGCP_METADATA_URLGCP_METADATA_HEADERSAWS_METADATA_URLAWS_METADATA_HEADERSs r! get_cloud_from_metadata_requestsrSsC(4S>2Jt" J)&1K-x8A02HIIw   02F G Gu   02F G Guyr cluster_config_file_pathcd} t|5}tj|}t}d|vr |d|_d|vr |d|_d|vrd|dvr|dd|_d|vr|cdddSd|vr|cdddS|d}|d}|D]}||kr.|||d }|r||_6|||d } | r4|j p t|_ |j | |j rt|j |_ |cdddS#1swxYwYdS#t$rt}tjt jvr;tjt jvrtj|_ntj|_|jt+|_n|xjd t+z c_|cYSt,$r5} t.d | tcYd} ~ Sd} ~ wwxYw) a Get the static cluster (autoscaler) config used to launch this cluster. Params: cluster_config_file_path: The file path to the cluster config file. Returns: The cluster (autoscaler) config or empty if it fails to get that information. cv|sdSd|vr|dSd|vr|dSd|vrd|dvr|ddSdS)N InstanceType machineTypeazure_arm_parametersvmSizer) node_configs r!get_instance_typez7get_cluster_config_to_report..get_instance_typesm 4 [ ( (~. . K ' '}- - "k 1 1K(>???56x@ @tr rrproviderrhead_node_typeNavailable_node_typesr[z_$z'Failed to get cluster config to report )ryaml safe_loadrrrrrvrrrZr]rrrPKUBERNETES_SERVICE_HOST_ENVr5r KUBERAY_ENVPROVIDER_KUBERAYPROVIDER_KUBERNETES_GENERICrSrRrSr) rTr\rrr`r^r_available_node_typerworker_node_instance_typerUs r!get_cluster_config_to_reportrhsi =' * + +% q^A&&F*,,F&&%+M%:"&&%+M%:"V##&2D(D(D(.z(:6(B%v--% % % % % % % % &V33% % % % % % % % $$45N#)*@#A '; Y Y#&.88.?.?,-@AEEmTT//+/Q9P60A0A,-@AEEmTT11-1Y"=F99==>WXXX0 485551K% % % % % % % % % % % % % % % % % % L &((  5 C C)RZ77(6(G%%)7(R%  ($D$F$FF ! !  ! !%N*J*L*L%N%N N ! ! ''' AaAABBB$&&&&&&&&'saE<A+E/? E< E/ E<CE/" E</E33E<6E37E<>! "-88# $-88% &!9 P P' ($<#V#V) *0>>+ ,0>>- .1@@/ 0&>%Z%Z1 24J???3 48 CCC5 6>!@@ ^<BBBB YYCY  6I OODJvd||44 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 <7 " "   $  / / / K     s5BB B r=c~tjd|ddit|d}||S)zReport the usage data to the usage server. Params: url: The URL to update resource usage. data: Data to report. Raises: requests.HTTPError: If requests fails. POSTz Content-Typezapplication/jsonr)r>rr)rArequestrraise_for_status)rr=rrrs r!report_usage_dataz#UsageReportClient.report_usage_datasO    #%78     r ) rrrr=r?rrr*rrr r!r~r~sr !%6!#!$!!!!&S0Btr r~rf)rWN)fr=rloggingr5rr threadingr dataclassesrrenumrrpathlibrtypingrr r r rAr`r!ray._common.usage.usage_constants_commonrbusage_constantsrPray._private.ray_constantsrr ray._rayletr ray.core.generatedr rray.experimental.internal_kvrr getLoggerrrSrqrr#r*r?rErZrLockrrrurtrrVrcrirnrxrwr~rrrrrr_post_init_hooksappendrrrrrCrrrrrrrrrrrrrr%r3r;rSrhrjrxrmrsrzr|r~rr r!rs"**V    )))))))),,,,,,,,,,,, ::::::::::::222222222!!!!!!11111111  8 $ $   ;;;;;;;; 99999999 :0:0:0:0:0:0:0:0z              D   355 .  0 0!TVV"0)."2"2 L LC L L L LS"LcLLLLNNNNN @D11 11)1))<1111D <