&`ixddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z m Z m Z mZmZmZddlmZddlmZddlmZmZddlmZmZmZmZddlmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z(m)Z*m+Z,dd l-m.Z.m/Z/m0Z0m1Z2ddl3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9m:Z:ddl;mm?Z?ej@eAZBdZCdZDejEdZFGdde#ZGe dgdZHde1fdZIGddZJGdd ZKGd!d"ZLGd#d$ZMGd%d&ejNZOdS)'N) defaultdict namedtuple)AnyDictListSetTupleUnion)MetricDescriptorType) ValueDouble) aggregationmeasure)CountAggregationDataDistributionAggregationDataLastValueAggregationDataSumAggregationData) StatsExporter) StatsRecorder)View) ViewManager)tag_keytag_map tag_value)CounterMetricFamilyGaugeMetricFamilyHistogramMetricFamilyMetric build_address)env_bool)WORKER_ID_TAG_KEYMetricCardinality) GcsClient)r)_is_invalid_metric_nameRAY_WORKER_TIMEOUT_SCOREz [^a-zA-Z0-9]ceZdZdZdeefdZedZedZ edZ edZ dS) GaugezGauge representation of opencensus view. This class is used to collect process metrics from the reporter agent. Cpp metrics should be collected in a different way. tagsct|rtd|dtj||||_||_d|D}t ||||jtj |_ dS)NzInvalid metric name: z. Metric will be discarded and data will not be collected or published. Metric names can only contain letters, numbers, _, and :. Metric names cannot start with numbers.c6g|]}tj|S)tag_key_moduleTagKey).0tags n/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/ray/_private/metrics_agent.py z"Gauge.__init__..Js#;;;s%c**;;;) r$ ValueErrormeasure_module MeasureInt_measure _descriptionrrr LastValueAggregation_view)selfname descriptionunitr)s r1__init__zGauge.__init__@s "4 ( ( ::::  '1$ TJJ ';;d;;; +tT\;3S3U3U   r3c|jSN)r7r;s r1rz Gauge.measureOs }r3c|jSrA)r:rBs r1viewz Gauge.viewS zr3c|jjSrA)rr<rBs r1r<z Gauge.nameWs |  r3c|jSrA)r8rBs r1r=zGauge.description[s   r3N) __name__ __module__ __qualname____doc__rstrr?propertyrrDr<r=r,r3r1r(r(9s  d3i     XX!!X!!!X!!!r3r(Record)gaugevaluer)metriccp|jjdsdStd|jj|j_|jD]]}|jD]S}|dr<|j}|j j j }t|dkr|ddkrd|d<T^dS)a Fix the inbound `opencensus.proto.metrics.v1.Metric` protos to make it acceptable by opencensus.stats.DistributionAggregationData. - metric name: gRPC OpenCensus metrics have names with slashes and dots, e.g. `grpc.io/client/server_latency`[1]. However Prometheus metric names only take alphanums,underscores and colons[2]. We santinize the name by replacing non-alphanum chars to underscore, like the official opencensus prometheus exporter[3]. - distribution bucket bounds: The Metric proto asks distribution bucket bounds to be > 0 [4]. However, gRPC OpenCensus metrics have their first bucket bound == 0 [1]. This makes the `DistributionAggregationData` constructor to raise Exceptions. This applies to all bytes and milliseconds (latencies). The fix: we update the initial 0 bounds to be 0.000_000_1. This will not affect the precision of the metrics, since we don't expect any less-than-1 bytes, or less-than-1-nanosecond times. [1] https://github.com/census-instrumentation/opencensus-specs/blob/master/stats/gRPC.md#units # noqa: E501 [2] https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels [3] https://github.com/census-instrumentation/opencensus-cpp/blob/50eb5de762e5f87e206c011a4f930adb1a1775b1/opencensus/exporters/stats/prometheus/internal/prometheus_utils.cc#L39 # noqa: E501 [4] https://github.com/census-instrumentation/opencensus-proto/blob/master/src/opencensus/proto/metrics/v1/metrics.proto#L218 # noqa: E501 zgrpc.io/N_distribution_valuergHz>) metric_descriptorr< startswithRE_NON_ALPHANUMSsub timeseriespointsHasFieldrTbucket_optionsexplicitboundslen)rQseriespoint dist_value bucket_boundss r1fix_grpc_metricrdcs,  # ( 3 3J ? ?$4$8$8 V % *%%F!#33] 3 3E~~233 3"5 * 9 B I }%%))mA.>!.C.C'2M!$  333r3c eZdZdedededeefdZedZedZedZ ed Z ed Z d Z d e d efdZdefdZdS)OpencensusProxyMetricr<descr> label_keyscL||_||_||_||_i|_dS)z>Represents the OpenCensus metrics that will be proxy exported.N)_name_desc_unit _label_keys_data)r;r<rgr>rhs r1r?zOpencensusProxyMetric.__init__s,   % r3c|jSrA)rjrBs r1r<zOpencensusProxyMetric.namerEr3c|jSrA)rkrBs r1rgzOpencensusProxyMetric.descrEr3c|jSrA)rlrBs r1r>zOpencensusProxyMetric.unitrEr3c|jSrA)rmrBs r1rhz OpencensusProxyMetric.label_keyss r3c|jSrArnrBs r1datazOpencensusProxyMetric.datarEr3ct|jdkoEttt |jt S)z8Check if the metric is a distribution aggreation metric.r)r_rn isinstancenextitervaluesrrBs r1 is_distribution_aggregation_dataz6OpencensusProxyMetric.is_distribution_aggregation_datasL4:" z dj''))** + +-H( (  r3 label_valuesruc||j|<dS)zAdd the data to the metric. Args: label_values: The label values of the metric. data: The data to be added. Nrt)r;r|rus r1add_datazOpencensusProxyMetric.add_datas$( <   r3rQc |j}t|dkrdS|D]M}td|jD}|jD]#}|jjtjkrt|j }n|jjtj krtt|j}n|jjtjkrt!t|j}n}|jjtjkrT|j}d|jD}|jjj} t/|j|jz |j|j|| }nt7d||j|<%OdS)zzParse the Opencensus Protobuf and store the data. The data can be accessed via `data` API once recorded. rNc3$K|] }|jV dSrArP)r/vals r1 z/OpencensusProxyMetric.record..s$DD39DDDDDDr3cg|] }|j Sr,)count)r/buckets r1r2z0OpencensusProxyMetric.record..s(W(W(W&(W(W(Wr3zSummary is not supported)rYr_tupler|rZrUtyper CUMULATIVE_INT64r int64_valueCUMULATIVE_DOUBLErr double_value GAUGE_DOUBLErCUMULATIVE_DISTRIBUTIONrTbucketsr\r]r^rsumrsum_of_squared_deviationr4rn) r;rQrYr`labelsrarurbcounts_per_bucketrcs r1recordzOpencensusProxyMetric.records & z??a   F!! *! *FDD0CDDDDDF  * *,1+<==00ABBDD,1+=>>.k5;MNNDD-26J6WWW3KASTTDD,1+CDD"'!9J(W(WJDV(W(W(W%$.$=$F$MM6")99"(";)% DD%%?@@@%) 6""; * ! *! *r3N)rHrIrJrLrr?rMr<rgr>rhrur{r rr~rrr,r3r1rfrfs S  3 DI    XXX  X X   (U(#((((,*V,*,*,*,*,*,*r3rfcteZdZdefdZedeeeffdZedZ de e fdZ dS) ComponentidcR||_tj|_i|_dS)zyRepresent a component that requests to proxy export metrics Args: id: Id of this component. N)rtime monotonic_last_reported_time_metrics)r;rs r1r?zComponent.__init__s* $(>#3#3  r3returnc|jS)zAReturn the metrics requested to proxy export from this component.)rrBs r1metricszComponent.metricss }r3c|jSrA)rrBs r1last_reported_timezComponent.last_reported_times ''r3rc6tj|_|D]}}t||j}|j}d|jD}||jvr$t||j |j ||j|<|j| |~dS)zParse the Opencensus protobuf and store metrics. Metrics can be accessed via `metrics` API for proxy export. Args: metrics: A list of Opencensus protobuf for proxy export. cg|] }|j Sr,)key)r/ label_keys r1r2z$Component.record.. sOOOI)-OOOr3N) rrrrdrUr<rhrrfr=r>r)r;rrQ descriptorr<rhs r1rzComponent.records$(>#3#3  / /F F # # #1J?DOO9NOOOJ4=((&;*0*/:'' d# M$  & &v . . . . / /r3N) rHrIrJrLr?rMrrfrrrrrr,r3r1rrs 3    c#889X((X(/d6l//////r3rceZdZddedefdZddeedefdZd Z d ed ed eed ede e j de deeeefddfdZdeeeeefdeeeeffdZdeedeefdZdZdS)OpenCensusProxyCollector< namespacecomponent_timeout_sctj|_||_||_i|_t dd|_dS)a)Prometheus collector implementation for opencensus proxy export. Prometheus collector requires to implement `collect` which is invoked whenever Prometheus queries the endpoint. The class is thread-safe. Args: namespace: Prometheus namespace. RAY_EXPORT_COUNTER_AS_GAUGETN) threadingLock_components_lock_component_timeout_s _namespace _componentsr _export_counter_as_gauge)r;rrs r1r?z!OpenCensusProxyCollector.__init__sK!* 0 0%8!#)11NPT(U(U%%%r3Nr worker_id_hexc|stn|}|j5||jvrt||j|<|j||ddddS#1swxYwYdS)a.Record the metrics reported from the component that reports it. Args: metrics: A list of opencensus protobuf to proxy export metrics. worker_id_hex: A worker id that reports these metrics. If None, it means they are reported from Raylet or GCS. N)GLOBAL_COMPONENT_KEYrrrr)r;rrrs r1rzOpenCensusProxyCollector.record5s+8J""]  " 2 2$***(1# %  S ! ( ( 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2sAA!!A%(A%c|j5g}g}|jD]n\}}tj|jz }||jkrC||t d ||o|D]/}||j |0|cdddS#1swxYwYdS)zClean up stale components. Stale means the component is dead or unresponsive. Stale components won't be reported to Prometheus anymore. zSMetrics from a worker ({}) is cleaned up due to timeout. Time since last report {}sN) rritemsrrrrappendloggerinfoformatpop)r;stale_componentsstale_component_idsr componentelapseds r1clean_stale_componentsz/OpenCensusProxyCollector.clean_stale_componentsCs? " $ $! "$ !%!1!7!7!9!9   I.**Y-IIT666'..r222KK>>DfR>Q>Q* B B ''(8(<(.zs!@@@Rb(b@@@r3)r< documentationr>rr)rrPN)r<rr_totalz(DEPRECATED, use z_total metric instead) z+Inf)rr sum_valuezunsupported aggregation type )rlockedrr_rwrgetr add_metric count_datarsum_datarendswithrrrr^sorted enumeraterrLrrrrPr4r)r;rrrhrr|rrrrQr cum_countiiboundrs r1to_prometheus_metricsz.OpenCensusProxyCollector.to_prometheus_metricsZs 4$++-----88;88 <  C OO333lJ5O333A@<@@@ h 4 5 5 !ook22G 3,$"4%%  "(+2 K( AJ ! !X=P ! Q Q Q F h 2 3 3\ O"ook22G 3,$"4% "(+2 K( AJ ! !X=N ! O O O0 T%%h// T w<<1$$.(4 44144 * FNN6***7||q(((( %%\AR%SSS F "= > >. O?fX_&=&=====GI&x77 ' ' EX7;; e**i0v&&&& NNFH$78 9 9 9!ook22G 3.$"4% "(+2 K( AJ ! !#", "    F ": ; ; O!ook22G 3*$"4% "(+2 K( AJ ! !X^ ! L L L FMT(^^MMNN Nr3datasc ,t|dksJ|d}t|tr,ttt d|DSt|t r&t t d|DSt|t r,t tt d|DStdt|dt dtdt d|d ) Nrcg|] }|j Sr,rr/rus r1r2zCOpenCensusProxyCollector._aggregate_metric_data..s!?!?!?$*!?!?!?r3cg|] }|j Sr,)rrs r1r2zCOpenCensusProxyCollector._aggregate_metric_data..s,O,O,OT_,O,O,Or3cg|] }|j Sr,)rrs r1r2zCOpenCensusProxyCollector._aggregate_metric_data..s!B!B!BD$-!B!B!Br3zUnsupported aggregation type z. Supported types are z, z.Got .) r_rwrr rrrr4r)r;rsamples r1_aggregate_metric_dataz/OpenCensusProxyCollector._aggregate_metric_datasC 5zzA~~~~q f6 7 7 +S!?!?!?!?!?@@  f2 3 3 R',O,O,O,O,O(P(PQQ Q f0 1 1 %S!B!BE!B!B!BCC  DLL  #  '?  CU        r3per_worker_metricsc tt|d}|rt|jvr|S|jt}t t }|D]R}|jD]6\}}||d|||dzdz |7St|j |j |j |jd||j|dzdz}|D].\}}||||/|gS)aCollect per-worker metrics, aggregate them into per-node metrics and convert them to Prometheus format. Args: per_worker_metrics: A list of per-worker metrics for the same metric name. Returns: A list of per-node metrics for the same metric name, with the high cardinality labels removed and the values aggregated. Nr)r<rgr>rh)rxryr!rhindexrlistrurrrfr<rgr>r~r) r;rrQworker_id_label_indexlabel_value_to_datar|ruaggregated_metricrs r1'_aggregate_with_recommended_cardinalityz@OpenCensusProxyCollector._aggregate_with_recommended_cardinalitysd-..55 &*&2CCC% % & 1 7 78I J J    )  F&,k&7&7&9&9  " d# !7"7!78"#81#<#>#>?@&,,,,  2()?*?)?@ 5 9 ; ;<=    $7#<#<#>#>   L%  & &++E22     """r3c #(K|j5g}tt}tj}|jD]x}|jD]\}|tjkr5| s!||j  |G| |]y|D]*}| | |+i}|D]P}|jD]4\}} ||j |j|j|j|| |5Q dddn #1swxYwY|D] } | D]}|V dS)a,Collect fetches the statistics from OpenCensus and delivers them as Prometheus Metrics. Collect is invoked every time a prometheus.Gatherer is run for example when the HTTP endpoint is invoked by Prometheus. This method is required as a Prometheus Collector. N)rrrr"get_cardinality_levelrrzr RECOMMENDEDr{r<rextendrrurrrgrhr>) r;open_cencus_metricsto_lower_cardinalitycardinality_levelrrQrprometheus_metrics_mapr|rurs r1collectz OpenCensusProxyCollector.collect9sM ", ,  @B LWLL !2 G I I !-4466 ; ; '/6688 ; ;F)->-JJJ & G G I IK-V[9@@HHHH+226:::: ;';&A&A&C&C  "#**@@* &( "-  *0+*;*;*=*=  &L$..  ) $.  E, , , , , , , , , , , , , , , \.4466  G!      sEE''E+.E+)rrA)rHrIrJrLintr?rrrrr tag_value_moduleTagValuerrPrometheusMetricrr rrrrrfrrr,r3r1rrsVV#VCVVVV> 2 2d6l 23 2 2 2 2$$$.LOLO LOI LO  LO ,56 LOLO#t$4556LO LOLOLOLO\  *,@BTT U  ')=?QQ R     45# !675# # $5#5#5#5#n88888r3rceZdZ ddededefdZddeefdZ de d e d e fd Z dd eed efdZdd eed efdZdZdS) MetricsAgentN view_managerstats_recorderstats_exporterc ztj|_||_||_||_d|_|jd|_nd|j|t|jj j ttj td|_t|_dS)a+A class to record and export metrics. The class exports metrics in 2 different ways. - Directly record and export metrics using OpenCensus. - Proxy metrics from other core components (e.g., raylet, GCS, core workers). This class is thread-safe. Nx)r)rr_lockrrrproxy_exporter_collectorregister_exporterroptionsrrosgetenvr%set_registered_views)r;rrrs r1r?zMetricsAgent.__init__us"^%% )-,)-%   &!%D     / / ? ? ?,D#+5$' 2F(L(L$M$M---D ) ,/55r3recordscp|pi}|j5|js ddddS|D]x}|j}|j}|j} |||i||4#t $r8}td|j d|d|d|d| Yd}~qd}~wwxYw ddddS#1swxYwYdS)z7Directly record and export stats from the same process.NzFailed to record metric z with value z with tags z and global tags due to: ) rrrOrPr) _record_gauge Exceptionrerrorr<)r;r  global_tagsrrOrPr)es r1record_and_exportzMetricsAgent.record_and_exports!'R Z  $         "    {&&ue5L5L 5LMMMM LLT5:TT5TT]aTTwBTTOPTT                   s? B+B+AB+ B".BB+BB++B/2B/rOrPr)c |j|jvr>|j|j|j|j|j}tj }| D]\}} tj |}n<#t$r/} td|d|jd| | d} ~ wwxYw t!j|} n?#t$r2} td|d|d|jd| | d} ~ wwxYw||| ||j|||dS)NzFailed to create tag key z for metric r zFailed to create tag value z for key )r<r r register_viewrDaddrnew_measurement_maptag_map_moduleTagMaprr-r.r4rrrrinsertmeasure_float_putrr) r;rOrPr)measurement_maprrtag_valrrrs r1rzMetricsAgent._record_gauges :T3 3 3   + +EJ 7 7 7  " & &uz 2 2 2-AACC ')) JJLL / /LC (/44    [[[[[VW[[    ,5g>>     o'ooCooUZU_oojkoo   NN7I . . . .))%-???w'''''s0B(( C!2*CC!%C:: D6-D11D6rrc|j5|js ddddS dddn #1swxYwY|||dS)a{Proxy export metrics specified by a Opencensus Protobuf. This API is used to export metrics emitted from core components. Args: metrics: A list of protobuf Metric defined from OpenCensus. worker_id_hex: The worker ID it proxies metrics export. None if the metric is not from a worker (i.e., raylet, GCS). N)rr_proxy_export_metricsr;rrs r1proxy_export_metricsz!MetricsAgent.proxy_export_metricssZ  $                          ""7M::::: +//c<|j||dSrA)rrr s r1rz"MetricsAgent._proxy_export_metricss! %,,WmDDDDDr3c|j5|js ddddS dddn #1swxYwY|jdS)zClean dead worker's metrics. Worker metrics are cleaned up and won't be exported once it is considered as dead. This method has to be periodically called by a caller. N)rrrrrBs r1clean_all_dead_worker_metricsz*MetricsAgent.clean_all_dead_worker_metricssZ  $                          %<<>>>>>r"rA)rHrIrJrrrr?rrNrr(floatdictrrrLr!rr%r,r3r1rrts )- /1/1!/1&/1& /1/1/1/1bf$(5((d((((4;;DL;;;;;"EET&\E#EEEE ? ? ? ? ?r3rcFeZdZdZfdZdZdZdZdZdZ dZ xZ S) PrometheusServiceDiscoveryWriteraFA class to support Prometheus service discovery. It supports file-based service discovery. Checkout https://prometheus.io/docs/guides/file-sd/ for more details. Args: gcs_address: Gcs address for this cluster. temp_dir: Temporary directory used by Ray to store logs and metadata. cRtjj|ddd}||_tjjj|||_d|_ g|_ tj |_ tdS)NTF)allow_cluster_id_nilfetch_cluster_id_if_nil)ray_rayletGcsClientOptionscreate gcs_address_privatestate_initialize_global_statetemp_dir&default_service_discovery_flush_period latest_service_discovery_contentrRLock _content_locksuperr?)r;r2r6gcs_client_options __class__s r1r?z)PrometheusServiceDiscoveryWriter.__init__s [9@@ DRWA  '  99:LMMM  67313-&_.. r3cR|j5|jcdddS#1swxYwYdS)z3Return the latest stored service discovery content.N)r:r8rBs r1$get_latest_service_discovery_contentzEPrometheusServiceDiscoveryWriter.get_latest_service_discovery_contentsu   9 98 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9s   ctj}d|D}t|j}|dd}|r(||d|dd}|r(||dddi|d g}|j5||_dddn #1swxYwYtj |S) z4Return the content for Prometheus service discovery.cZg|](}|ddu t|d|d)S)aliveTNodeManagerAddressMetricsExportPortr)r/nodes r1r2zOPrometheusServiceDiscoveryWriter.get_file_discovery_content..sG$ $ $ G}$$ $34d;N6O P P$$$r3)addresssAutoscalerMetricsAddressNzutf-8sDashboardMetricsAddressjobr.)rtargets) r.nodesr#r2internal_kv_getrdecoder:r8jsondumps)r;rImetrics_export_addresses gcs_clientautoscaler_addrdashboard_addrcontents r1get_file_discovery_contentz;PrometheusServiceDiscoveryWriter.get_file_discovery_contentsP $ $ $ $ $  t'7888 $445PRVWW  M $ + +O,B,B7,K,K L L L#334NPTUU  L $ + +N,A,A',J,J K K K$en9QRRS   < <4;D 1 < < < < < < < < < < < < < < <z'"""sCCCc|}t|d5}||dddn #1swxYwYt j||dS)Nw)get_temp_file_nameopenwriterSrreplaceget_target_file_name)r;temp_file_name json_files r1rXz&PrometheusServiceDiscoveryWriter.write*s 0022 .# & & ?) OOD;;== > > > ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? >4#<#<#>#>?????s(AA Acttj|jtjjjSrA)rpathjoinr6r.r3 ray_constants!PROMETHEUS_SERVICE_DISCOVERY_FILErBs r1rZz5PrometheusServiceDiscoveryWriter.get_target_file_name7s)w|| M3<5W   r3ctj|jddt jjjS)Nz{}_{}tmp) rr^r_r6rr.r3r`rarBs r1rVz3PrometheusServiceDiscoveryWriter.get_temp_file_name<s=w|| M NNs|1S     r3c |n#t$r}td|tt jtd|Yd}~nd}~wwxYwtj |j )NTz,Writing a service discovery file, {},failed.zError message: ) rXrrwarningrrZ traceback format_excrsleepr7)r;rs r1runz$PrometheusServiceDiscoveryWriter.runDs D 6  6 6 6$fT%>%>%@%@AAy35566644455555555  6 JtB C C C Ds B3BB..B3) rHrIrJrKr?r?rSrXrZrVri __classcell__)r=s@r1r)r)s   999 ###( @ @ @       D D D D D D Dr3r))PrLloggingrrerrrf collectionsrrtypingrrrrr r +opencensus.metrics.export.metric_descriptorr opencensus.metrics.export.valuer opencensus.statsr rr5!opencensus.stats.aggregation_datarrrropencensus.stats.base_exporterropencensus.stats.stats_recorderropencensus.stats.viewropencensus.stats.view_managerropencensus.tagsrr-rrrrprometheus_client.corerrrrrr.ray._common.network_utilsrray._private.ray_constantsr )ray._private.telemetry.metric_cardinalityr!r" ray._rayletr#ray.core.generated.metrics_pb2ray.util.metricsr$ getLoggerrHrr%rcompilerWr(rNrdrfrrrThreadr)r,r3r1rs   ////////5555555555555555LLLLLL777777CCCCCCCC 988888999999&&&&&&555555  333333//////"!!!!!111111444444  8 $ $ .2:o..$!$!$!$!$!D$!$!$!N H888 9 9#3F#3#3#3#3L[*[*[*[*[*[*[*[*|+/+/+/+/+/+/+/+/\\\\\\\\\~ ~?~?~?~?~?~?~?~?B[D[D[D[D[Dy'7[D[D[D[D[Dr3