Duibonduil commited on
Commit
0d3a3a6
·
verified ·
1 Parent(s): c532cde

Upload 5 files

Browse files
aworld/metrics/README.md ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Metrics Module
2
+
3
+ ## Overview
4
+ The `aworld.core.metrics` module provides a unified interface for collecting and exporting metrics. It supports various types of metrics (e.g., counters, histograms) and allows exporting data to different monitoring systems (e.g., Prometheus).
5
+
6
+ ## Key Features
7
+ - **Metric Types**:
8
+ - `Counter`: A cumulative metric that represents a single numerical value that only ever increases.
9
+ - `UpDownCounter`: A cumulative metric that can increase or decrease.
10
+ - `Gauge`: A metric that represents a single numerical value that can arbitrarily go up and down.
11
+ - `Histogram`: A metric that represents the distribution of a set of values.
12
+
13
+ - **Adapters**:
14
+ - `PrometheusAdapter`: Exports metrics to Prometheus.
15
+ - `ConsoleAdapter`: Prints metrics to the console (for debugging purposes).
16
+
17
+ ## Usage Example
18
+
19
+ ```python
20
+ import random
21
+ import time
22
+ from aworld.core.metrics.metric import set_metric_provider, MetricType
23
+ from aworld.core.metrics.prometheus.prometheus_adapter import PrometheusConsoleMetricExporter,
24
+
25
+ PrometheusMetricProvider
26
+ from aworld.core.metrics.context_manager import MetricContext, ApiMetricTracker
27
+ from aworld.core.metrics.template import MetricTemplate
28
+
29
+ # Set OpenTelemetry as the metric provider
30
+ # set_metric_provider(OpentelemetryMetricProvider())
31
+
32
+ # Set Prometheus as the metric provider
33
+ set_metric_provider(PrometheusMetricProvider(PrometheusConsoleMetricExporter(out_interval_secs=2)))
34
+
35
+ # Define metric templates
36
+ my_counter = MetricTemplate(
37
+ type=MetricType.COUNTER,
38
+ name="my_counter",
39
+ description="My custom counter",
40
+ unit="1"
41
+ )
42
+
43
+ my_gauge = MetricTemplate(
44
+ type=MetricType.GAUGE,
45
+ name="my_gauge"
46
+ )
47
+
48
+ my_histogram = MetricTemplate(
49
+ type=MetricType.HISTOGRAM,
50
+ name="my_histogram",
51
+ buckets=[2, 4, 6, 8, 10]
52
+ )
53
+
54
+
55
+ # Track API metrics using decorator
56
+ @ApiMetricTracker()
57
+ def test_api():
58
+ time.sleep(random.uniform(0, 1))
59
+
60
+
61
+ # Track custom code block using context manager
62
+ def test_custom_code():
63
+ with ApiMetricTracker("test_custom_code"):
64
+ time.sleep(random.uniform(0, 1))
65
+
66
+
67
+ # Main loop to generate and record metrics
68
+ while 1:
69
+ MetricContext.count(my_counter, random.randint(1, 10))
70
+ MetricContext.gauge_set(my_gauge, random.randint(1, 10))
71
+ MetricContext.histogram_record(my_histogram, random.randint(1, 10))
72
+ test_api()
73
+ test_custom_code()
74
+ time.sleep(random.random())
75
+ ```
76
+
77
+ ## Notes
78
+ - Before using metrics, you must set a metric provider ( set_metric_provider ).
79
+ - Different metric types serve different purposes; choose the appropriate type based on your needs.
80
+ - For production environments, it is recommended to use Prometheus as the exporter.
aworld/metrics/__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ # coding: utf-8
2
+ # Copyright (c) 2025 inclusionAI.
3
+ import os
4
+ from aworld.metrics.context_manager import MetricContext
5
+
6
+ # MetricContext.configure(provider="otlp",
7
+ # backend="logfire",
8
+ # write_token=os.getenv("LOGFIRE_WRITE_TOKEN")
9
+ # )
aworld/metrics/context_manager.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ import asyncio
3
+ from typing import Callable
4
+ from functools import wraps
5
+ from aworld.metrics.metric import get_metric_provider, MetricType, BaseMetric
6
+ from aworld.metrics.template import MetricTemplate, MetricTemplates
7
+
8
+ _GLOBAL_METIRCS = {}
9
+
10
+
11
+ class MetricContext:
12
+
13
+ _initialized = False
14
+
15
+ @classmethod
16
+ def configure(cls,
17
+ provider: str,
18
+ backend: str,
19
+ base_url: str = None,
20
+ write_token: str = None,
21
+ **kwargs):
22
+ """
23
+ Configure the metric provider.
24
+ Args:
25
+ provider: The provider of the metric provider.
26
+ backend: The backend of the metric provider.
27
+ base_url: The base url of the metric provider.
28
+ write_token: The write token of the metric provider.
29
+ export_console: Whether to export the metrics to console.
30
+ **kwargs: The other parameters of the metric provider.
31
+ """
32
+ if cls._initialized:
33
+ cls.shutdown()
34
+ if provider == "prometheus":
35
+ from aworld.metrics.prometheus.prometheus_adapter import configure_prometheus_provider
36
+ configure_prometheus_provider(
37
+ backend, base_url, write_token, **kwargs)
38
+ elif provider == "otlp":
39
+ from aworld.metrics.opentelemetry.opentelemetry_adapter import configure_otlp_provider
40
+ configure_otlp_provider(backend, base_url, write_token, **kwargs)
41
+ cls._initialized = True
42
+
43
+ @classmethod
44
+ def metric_initialized(cls):
45
+ return cls._initialized
46
+
47
+ @staticmethod
48
+ def get_or_create_metric(template: MetricTemplate):
49
+ if template.name in _GLOBAL_METIRCS:
50
+ return _GLOBAL_METIRCS[template.name]
51
+
52
+ metric = None
53
+ if template.type == MetricType.COUNTER:
54
+ metric = get_metric_provider().create_counter(template.name, template.description, template.unit,
55
+ template.labels)
56
+ elif template.type == MetricType.UPDOWNCOUNTER:
57
+ metric = get_metric_provider().create_un_down_counter(template.name, template.description, template.unit,
58
+ template.labels)
59
+ elif template.type == MetricType.GAUGE:
60
+ metric = get_metric_provider().create_gauge(template.name, template.description, template.unit,
61
+ template.labels)
62
+ elif template.type == MetricType.HISTOGRAM:
63
+ metric = get_metric_provider().create_histogram(template.name, template.description, template.unit,
64
+ template.buckets, template.labels)
65
+
66
+ _GLOBAL_METIRCS[template.name] = metric
67
+ return metric
68
+
69
+ @classmethod
70
+ def _validate_type(cls, metric: BaseMetric, type: str):
71
+ if type != metric._type:
72
+ raise ValueError(f"metric type {metric._type} is not {type}")
73
+
74
+ @classmethod
75
+ def count(cls, template: MetricTemplate, value: int, labels: dict = None):
76
+ """
77
+ Increment a counter metric.
78
+ """
79
+ metric = cls.get_or_create_metric(template)
80
+ cls._validate_type(metric, MetricType.COUNTER)
81
+ metric.add(value, labels)
82
+
83
+ @classmethod
84
+ def inc(cls, template: MetricTemplate, value: int, labels: dict = None):
85
+ """
86
+ Increment a updowncounter metric.
87
+ """
88
+ metric = cls.get_or_create_metric(template)
89
+ cls._validate_type(metric, MetricType.UPDOWNCOUNTER)
90
+ metric.inc(value, labels)
91
+
92
+ @classmethod
93
+ def dec(cls, template: MetricTemplate, value: int, labels: dict = None):
94
+ """
95
+ Decrement a updowncounter metric.
96
+ """
97
+ metric = cls.get_or_create_metric(template)
98
+ cls._validate_type(metric, MetricType.UPDOWNCOUNTER)
99
+ metric.dec(value, labels)
100
+
101
+ @classmethod
102
+ def gauge_set(cls, template: MetricTemplate, value: int, labels: dict = None):
103
+ """
104
+ Set a value to a gauge metric.
105
+ """
106
+ metric = cls.get_or_create_metric(template)
107
+ cls._validate_type(metric, MetricType.GAUGE)
108
+ metric.set(value, labels)
109
+
110
+ @classmethod
111
+ def histogram_record(cls, template: MetricTemplate, value: int, labels: dict = None):
112
+ """
113
+ Set a value to a histogram metric.
114
+ """
115
+ metric = cls.get_or_create_metric(template)
116
+ cls._validate_type(metric, MetricType.HISTOGRAM)
117
+ metric.record(value, labels)
118
+
119
+ @classmethod
120
+ def shutdown(cls):
121
+ """
122
+ Shutdown the metric provider.
123
+ """
124
+ provider = get_metric_provider()
125
+ if provider:
126
+ provider.shutdown()
127
+ cls._initialized = False
128
+
129
+
130
+ class ApiMetricTracker:
131
+ """
132
+ Decorator to track API metrics.
133
+ """
134
+
135
+ def __init__(self, api_name: str = None, func: Callable = None):
136
+ self.start_time = None
137
+ self.status = "success"
138
+ self.func = func
139
+ self.api_name = api_name
140
+ if self.api_name is None and self.func is not None:
141
+ self.api_name = self.func.__name__
142
+
143
+ def _new_tracker(self, func: Callable):
144
+ return self.__class__(func=func)
145
+
146
+ def __enter__(self):
147
+ self.start_time = time.time() * 1000
148
+
149
+ def __exit__(self, exc_type, value, traceback):
150
+ if exc_type is None:
151
+ self.status = "success"
152
+ else:
153
+ self.status = "failure"
154
+ self._record_metrics(self.api_name, self.start_time, self.status)
155
+
156
+ def __call__(self, func: Callable = None) -> Callable:
157
+ if func is None:
158
+ return self
159
+ return self.decorator(func)
160
+
161
+ def _record_metrics(self, api_name: str, start_time: float, status: str) -> None:
162
+ """
163
+ Record metrics for the API.
164
+ """
165
+ elapsed_time = time.time() * 1000 - start_time
166
+ MetricContext.count(MetricTemplates.REQUEST_COUNT, 1,
167
+ labels={"method": api_name, "status": status})
168
+ MetricContext.histogram_record(MetricTemplates.REQUEST_LATENCY, elapsed_time,
169
+ labels={"method": api_name, "status": status})
170
+
171
+ def decorator(self, func):
172
+ """
173
+ Decorator to track API metrics.
174
+ """
175
+
176
+ @wraps(func)
177
+ async def async_wrapper(*args, **kwargs):
178
+ with self._new_tracker(func):
179
+ return await func(*args, **kwargs)
180
+
181
+ @wraps(func)
182
+ def wrapper(*args, **kwargs):
183
+ with self._new_tracker(func):
184
+ return func(*args, **kwargs)
185
+
186
+ return async_wrapper if asyncio.iscoroutinefunction(func) else wrapper
aworld/metrics/metric.py ADDED
@@ -0,0 +1,287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Optional, Sequence
3
+
4
+
5
+ class MetricType:
6
+ """
7
+ MetricType is a class for defining the type of a metric.
8
+ """
9
+ COUNTER = "counter"
10
+ UPDOWNCOUNTER = "updowncounter"
11
+ GAUGE = "gauge"
12
+ HISTOGRAM = "histogram"
13
+
14
+
15
+ class MetricProvider(ABC):
16
+ """
17
+ MeterProvider is the entry point of the API.
18
+ """
19
+
20
+ def __init__(self):
21
+ # list of exporters
22
+ self._exporters = []
23
+
24
+ @abstractmethod
25
+ def shutdown(self):
26
+ """
27
+ shutdown the metric provider.
28
+ """
29
+
30
+ def add_exporter(self, exporter):
31
+ """
32
+ Add an exporter to the list of exporters.
33
+ """
34
+ self._exporters.append(exporter)
35
+
36
+ @abstractmethod
37
+ def create_counter(self, name: str, description: str, unit: str,
38
+ label_names: Optional[Sequence[str]] = None) -> "Counter":
39
+ """
40
+ Create a counter.
41
+
42
+ Args:
43
+ name: The name of the instrument to be created
44
+ description: A description for this instrument and what it measures.
45
+ unit: The unit for observations this instrument reports. For
46
+ example, ``By`` for bytes. UCUM units are recommended.
47
+ """
48
+
49
+ @abstractmethod
50
+ def create_un_down_counter(self, name: str, description: str, unit: str,
51
+ label_names: Optional[Sequence[str]] = None) -> "UnDownCounter":
52
+ """
53
+ Create a un-down counter.
54
+ Args:
55
+ name: The name of the instrument to be created
56
+ description: A description for this instrument and what it measures.
57
+ unit: The unit for observations this instrument reports. For
58
+ example, ``By`` for bytes. UCUM units are recommended.
59
+ """
60
+
61
+ @abstractmethod
62
+ def create_gauge(self, name: str, description: str, unit: str,
63
+ label_names: Optional[Sequence[str]] = None) -> "Gauge":
64
+ """
65
+ Create a gauge.
66
+ Args:
67
+ name: The name of the instrument to be created
68
+ description: A description for this instrument and what it measures.
69
+ unit: The unit for observations this instrument reports. For
70
+ example, ``By`` for bytes. UCUM units are recommended.
71
+ """
72
+
73
+ @abstractmethod
74
+ def create_histogram(self,
75
+ name: str,
76
+ description: str,
77
+ unit: str,
78
+ buckets: Optional[Sequence[float]] = None,
79
+ label_names: Optional[Sequence[str]] = None) -> "Histogram":
80
+ """
81
+ Create a histogram.
82
+ Args:
83
+ name: The name of the instrument to be created
84
+ description: A description for this instrument and what it measures.
85
+ unit: The unit for observations this instrument reports. For
86
+ example, ``By`` for bytes. UCUM units are recommended.
87
+ """
88
+
89
+
90
+ class BaseMetric(ABC):
91
+ """
92
+ Metric is the base class for all metrics.
93
+ Args:
94
+ name: The name of the metric.
95
+ description: The description of the metric.
96
+ unit: The unit of the metric.
97
+ provider: The provider of the metric.
98
+ """
99
+
100
+ def __init__(self,
101
+ name: str,
102
+ description: str,
103
+ unit: str,
104
+ provider: MetricProvider,
105
+ label_names: Optional[Sequence[str]] = None):
106
+ self._name = name
107
+ self._description = description
108
+ self._unit = unit
109
+ self._provider = provider
110
+ self._label_names = label_names
111
+ self._type = None
112
+
113
+
114
+ class Counter(BaseMetric):
115
+ """
116
+ Counter is a subclass of BaseMetric, representing a counter metric.
117
+ A counter is a cumulative metric that represents a single numerical value that only ever goes up.
118
+ """
119
+
120
+ def __init__(self,
121
+ name: str,
122
+ description: str,
123
+ unit: str,
124
+ provider: MetricProvider,
125
+ label_names: Optional[Sequence[str]] = None):
126
+ """
127
+ Initialize the Counter.
128
+ Args:
129
+ name: The name of the metric.
130
+ description: The description of the metric.
131
+ unit: The unit of the metric.
132
+ provider: The provider of the metric.
133
+ """
134
+ super().__init__(name, description, unit, provider, label_names)
135
+ self._type = MetricType.COUNTER
136
+
137
+ @abstractmethod
138
+ def add(self, value: int, labels: dict = None) -> None:
139
+ """
140
+ Add a value to the counter.
141
+ Args:
142
+ value: The value to add to the counter.
143
+ labels: The labels to associate with the value.
144
+ """
145
+
146
+
147
+ class UpDownCounter(BaseMetric):
148
+ """
149
+ UpDownCounter is a subclass of BaseMetric, representing an un-down counter metric.
150
+ An un-down counter is a cumulative metric that represents a single numerical value that only ever goes up.
151
+ """
152
+
153
+ def __init__(self,
154
+ name: str,
155
+ description: str,
156
+ unit: str,
157
+ provider: MetricProvider,
158
+ label_names: Optional[Sequence[str]] = None):
159
+ """
160
+ Initialize the UnDownCounter.
161
+ Args:
162
+ name: The name of the metric.
163
+ description: The description of the metric.
164
+ unit: The unit of the metric.
165
+ provider: The provider of the metric.
166
+ """
167
+ super().__init__(name, description, unit, provider, label_names)
168
+ self._type = MetricType.UPDOWNCOUNTER
169
+
170
+ @abstractmethod
171
+ def inc(self, value: int, labels: dict = None) -> None:
172
+ """
173
+ Add a value to the gauge.
174
+ Args:
175
+ value: The value to add to the gauge.
176
+ labels: The labels to associate with the value.
177
+ """
178
+
179
+ @abstractmethod
180
+ def dec(self, value: int, labels: dict = None) -> None:
181
+ """
182
+ Subtract a value from the gauge.
183
+ Args:
184
+ value: The value to subtract from the gauge.
185
+ labels: The labels to associate with the value.
186
+ """
187
+
188
+
189
+ class Gauge(BaseMetric):
190
+ """
191
+ Gauge is a subclass of BaseMetric, representing a gauge metric.
192
+ A gauge is a metric that represents a single numerical value that can arbitrarily go up and down.
193
+ """
194
+
195
+ def __init__(self,
196
+ name: str,
197
+ description: str,
198
+ unit: str,
199
+ provider: MetricProvider,
200
+ label_names: Optional[Sequence[str]] = None):
201
+ """
202
+ Initialize the Gauge.
203
+ Args:
204
+ name: The name of the metric.
205
+ description: The description of the metric.
206
+ unit: The unit of the metric.
207
+ provider: The provider of the metric.
208
+ """
209
+ super().__init__(name, description, unit, provider, label_names)
210
+ self._type = MetricType.GAUGE
211
+
212
+ @abstractmethod
213
+ def set(self, value: int, labels: dict = None) -> None:
214
+ """
215
+ Set the value of the gauge.
216
+ Args:
217
+ value: The value to set the gauge to.
218
+ labels: The labels to associate with the value.
219
+ """
220
+
221
+
222
+ class Histogram(BaseMetric):
223
+ """
224
+ Histogram is a subclass of BaseMetric, representing a histogram metric.
225
+ A histogram is a metric that represents the distribution of a set of values.
226
+ """
227
+
228
+ def __init__(self,
229
+ name: str,
230
+ description: str,
231
+ unit: str,
232
+ provider: MetricProvider,
233
+ buckets: Sequence[float] = None,
234
+ label_names: Optional[Sequence[str]] = None):
235
+ """
236
+ Initialize the Histogram.
237
+ Args:
238
+ name: The name of the metric.
239
+ description: The description of the metric.
240
+ unit: The unit of the metric.
241
+ provider: The provider of the metric.
242
+ buckets: The buckets of the histogram.
243
+ """
244
+ super().__init__(name, description, unit, provider, label_names)
245
+ self._type = MetricType.HISTOGRAM
246
+ self._buckets = buckets
247
+
248
+ @abstractmethod
249
+ def record(self, value: int, labels: dict = None) -> None:
250
+ """
251
+ Record a value in the histogram.
252
+ Args:
253
+ value: The value to record in the histogram.
254
+ labels: The labels to associate with the value.
255
+ """
256
+
257
+
258
+ class MetricExporter(ABC):
259
+ """
260
+ MetricExporter is the base class for all metric exporters.
261
+ """
262
+ @abstractmethod
263
+ def shutdown(self):
264
+ """
265
+ Export the metrics.
266
+ """
267
+
268
+
269
+ _GLOBAL_METRIC_PROVIDER: Optional[MetricProvider] = None
270
+
271
+
272
+ def set_metric_provider(provider):
273
+ """
274
+ Set the global metric provider.
275
+ """
276
+ global _GLOBAL_METRIC_PROVIDER
277
+ _GLOBAL_METRIC_PROVIDER = provider
278
+
279
+
280
+ def get_metric_provider():
281
+ """
282
+ Get the global metric provider.
283
+ """
284
+ global _GLOBAL_METRIC_PROVIDER
285
+ if _GLOBAL_METRIC_PROVIDER is None:
286
+ raise ValueError("No metric provider has been set.")
287
+ return _GLOBAL_METRIC_PROVIDER
aworld/metrics/template.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Sequence
2
+ from pydantic import BaseModel, Field, root_validator
3
+ from typing import Optional
4
+
5
+
6
+ class MetricTemplate(BaseModel):
7
+ """
8
+ MetricTemplate is a class for defining a metric template.
9
+ """
10
+ type: str
11
+ name: str
12
+ description: Optional[str] = None
13
+ unit: Optional[str] = Field(default="1")
14
+ labels: Optional[list[str]] = None
15
+ buckets: Optional[Sequence[float]] = None
16
+
17
+ @root_validator(pre=True)
18
+ def set_default_description(cls, values):
19
+ """
20
+ Set the default description if it is not set.
21
+ """
22
+ if 'description' not in values or values['description'] is None:
23
+ values['description'] = values['name']
24
+ return values
25
+
26
+
27
+ class MetricTemplates:
28
+ REQUEST_COUNT = MetricTemplate(**{
29
+ "type": "counter",
30
+ "name": "request_count",
31
+ "description": "The number of requests received",
32
+ "unit": "1",
33
+ "labels": ["method", "status"]
34
+ })
35
+
36
+ REQUEST_LATENCY = MetricTemplate(**{
37
+ "type": "histogram",
38
+ "name": "request_latency",
39
+ "description": "The latency of requests",
40
+ "unit": "ms",
41
+ "labels": ["method", "status"],
42
+ # "buckets": [0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000]
43
+ })