feat: интеграция prometheus в ml_service, конфигурация prometheus

services/ml_service/Dockerfile

@@ -15,5 +15,5 @@ ENV MODELS_PATH=/models
 
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
 
-# docker build -t ml_service:1 services/ml_service/
-# docker run -v "$(pwd)/services/models:/models" -p 8000:8000 ml_service:1
+# docker build -t ml_service:2 services/ml_service/
+# docker run -v "$(pwd)/services/models:/models" -p 8000:8000 ml_service:2

services/ml_service/app/main.py

@@ -2,6 +2,7 @@ from os import getenv
 from pathlib import Path
 
 from fastapi import FastAPI
+from prometheus_fastapi_instrumentator import Instrumentator
 from pydantic import BaseModel, Field
 
 from ._meta import PACKAGE_PATH
@@ -35,6 +36,13 @@ app = FastAPI(
 )
 
 
+_ = (
+    Instrumentator(excluded_handlers=['/metrics'])
+    .instrument(app)
+    .expose(app, endpoint='/metrics')
+)
+
+
 @app.get('/', summary='Тестовый эндпоинт')
 async def root():
     return {'Hello': 'World'}

services/ml_service/app/predictor.py

@@ -1,7 +1,9 @@
 from dataclasses import dataclass
 from enum import Enum
+from itertools import chain
 from pandas import DataFrame
 from pickle import load
+from prometheus_client import Counter, Histogram
 
 
 def open_model_file(file, *, buffering=-1, opener=None, **kwargs_extra):
@@ -61,6 +63,27 @@ class PricePredictionFeatures:
     transmission_type: TransmissionType
 
 
+metric_prediction_latency = Histogram(
+    'model_prediction_seconds', 'Время вычислений в модели',
+    buckets=(
+        list(chain.from_iterable((v * (10 ** p) for v in (1, 2, 5)) for p in range(-4, (1 + 1))))
+        + [float('+inf')]
+    ),
+)
+
+metric_prediction_errors = Counter(
+    'model_prediction_errors_total', 'Ошибки вычислений в модели по типу', ('error_type',),
+)
+
+metric_prediction_value = Histogram(
+    'model_prediction_value', 'Предсказанное значение цены',
+    buckets=(
+        list(chain.from_iterable((v * (10 ** p) for v in (1, 2, 5)) for p in range(-1, (2 + 1))))
+        + [float('+inf')]
+    ),
+)
+
+
 class PricePredictor:
 
     def __init__(self, model_path):
@@ -76,6 +99,13 @@ class PricePredictor:
             'transmission': features.transmission_type.value,
             'age':  features.age,
         }])
-        predictions = self._model.predict(features_df)
+        try:
+            with metric_prediction_latency.time():
+                predictions = self._model.predict(features_df)
+        except Exception as err:
+            metric_prediction_errors.labels(error_type=type(err).__name__).inc()
+            raise
         assert len(predictions) == 1
-        return float(predictions[0])
+        value = float(predictions[0])
+        metric_prediction_value.observe(value)
+        return value

services/ml_service/requirements.txt

@@ -1,5 +1,7 @@
 fastapi ~=0.120.4
 mlxtend ~=0.23.4
 pandas >=2.3.1,<3
+prometheus_client ~=0.23.1
+prometheus_fastapi_instrumentator >=7.0.2,<8
 scikit-learn >=1.7.2,<2
 uvicorn ~=0.38.0

services/prometheus/prometheus.yaml

@@ -0,0 +1,15 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 5s
+
+scrape_configs:
+
+  - job_name: "prices_predictor"
+    static_configs:
+      - targets:
+        - "prices-predictor:8000"
+    scheme: http
+    metrics_path: "/metrics"
+    #relabel_configs:
+    #  - source_labels: ["__address__"]
+    #    target_labels: "instance"