diff --git a/services/README.md b/services/README.md index a233418..cd90206 100644 --- a/services/README.md +++ b/services/README.md @@ -8,7 +8,7 @@ |-----------------|------------------|------------------|------------------| | — | `prices-predictor` | код: `ml_service/` | Веб-сервис предсказания цен, только stateless API. Об используемой предсказательной модели см. `research/README.md`. | | — | `prometheus` | конфигурация: `prometheus/` | Мониторинг сервиса ([Prometheus](https://prometheus.io/)). | -| — | `grafana` | | Аналитика и визуализация данных мониторига сервиса ([Grafana](https://grafana.com/)). | +| — | `grafana` | сохранённая конфигурация: `grafana/` | Аналитика и визуализация данных мониторига сервиса ([Grafana](https://grafana.com/)). | | `with-testers` | `load-tester` | код: `load-tester/` | Генератор потока случайных запросов к `prices-predictor` для тестирования. | Дополнительно: @@ -56,6 +56,44 @@ Может использоваться для проверки состояния сервиса. +## Мониторинг + +### Prometheus UI + +#### Примеры запросов + +Гистограмма предсказанных цен `model_prediction_value_bucket` (запрос: `rate(model_prediction_value_bucket[5m]`): + +![Гистограмма предсказанных цен как временные ряды](docs/screenshot-prometheus-query-model-1.png) + +Гистограмма продолжительности предсказания цен моделью ML `model_prediction_seconds_bucket` (запрос: `rate(model_prediction_seconds_bucket[5m]`): + +![Гистограмма продолжительности предсказания цен моделью ML как временные ряды](docs/screenshot-prometheus-query-model-2.png) + +Интенсивность потока запросов к сервису предсказания цен с разными результатами (успех — коды HTTP `2xx`, ошибки со стороны клиента — коды HTTP `4xx`) `http_requests_total{handler="/predict"}` (запрос: `rate(http_requests_total{handler="/predict"}[5m]`): + +![Интенсивность потока запросов к сервису предсказания цен с разными результатами](docs/screenshot-prometheus-query-http-1.png) + +Интенсивность потока запросов к **веб-серверу** сервиса предсказания цен **с ошибками** `http_requests_total{handler="/predict"}` (запрос: `sum without(handler, method) (rate(http_requests_total{status=~"4..|5.."}[5m]))`): + +![Интенсивность потока запросов к веб-серверу сервиса предсказания цен, заканчивающихся ошибками](docs/screenshot-prometheus-query-http-1.png) + +### Дашборд в Grafana + +Дашборд экспортирован в файл: `grafana/objects/dashboard-1765200932880.json`. + +![Дашборд в Grafana](docs/screenshot-grafana-dashboard.png) + +Элементы: + +* мониторинг модели: + * гистограмма распределения предсказанных цен за период времени (10 мин); +* прикладной уровень: + * интенсивность потока запросов (всех запросов; запросов, заканчивающихся ошибкой); +* инфраструктурный уровень: + * состояние сервиса (up/down); + * выделенный процессу объём VRAM. + ## Развёртывание ### Файл модели diff --git a/services/docs/screenshot-grafana-dashboard.png b/services/docs/screenshot-grafana-dashboard.png new file mode 100644 index 0000000..08a1ae1 Binary files /dev/null and b/services/docs/screenshot-grafana-dashboard.png differ diff --git a/services/docs/screenshot-prometheus-query-http-1.png b/services/docs/screenshot-prometheus-query-http-1.png new file mode 100644 index 0000000..e2bad6b Binary files /dev/null and b/services/docs/screenshot-prometheus-query-http-1.png differ diff --git a/services/docs/screenshot-prometheus-query-http-2.png b/services/docs/screenshot-prometheus-query-http-2.png new file mode 100644 index 0000000..4c9c40b Binary files /dev/null and b/services/docs/screenshot-prometheus-query-http-2.png differ diff --git a/services/docs/screenshot-prometheus-query-model-1.png b/services/docs/screenshot-prometheus-query-model-1.png new file mode 100644 index 0000000..c26d3d7 Binary files /dev/null and b/services/docs/screenshot-prometheus-query-model-1.png differ diff --git a/services/docs/screenshot-prometheus-query-model-2.png b/services/docs/screenshot-prometheus-query-model-2.png new file mode 100644 index 0000000..d01051b Binary files /dev/null and b/services/docs/screenshot-prometheus-query-model-2.png differ diff --git a/services/grafana/.gitattributes b/services/grafana/.gitattributes new file mode 100644 index 0000000..dd3f7b1 --- /dev/null +++ b/services/grafana/.gitattributes @@ -0,0 +1 @@ +objects/*.json -text diff --git a/services/grafana/objects/dashboard-1765200932880.json b/services/grafana/objects/dashboard-1765200932880.json new file mode 100644 index 0000000..81ac185 --- /dev/null +++ b/services/grafana/objects/dashboard-1765200932880.json @@ -0,0 +1,543 @@ +{ + "apiVersion": "dashboard.grafana.app/v2beta1", + "kind": "Dashboard", + "metadata": { + "name": "adcdfv7", + "generation": 9, + "creationTimestamp": "2025-12-08T12:51:53Z", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "process_virtual_memory_bytes", + "instant": false, + "interval": "10s", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 1, + "links": [], + "title": "Выделенный объём VRAM", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "12.4.0-20012734117" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "up", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 2, + "links": [], + "title": "Состояние", + "vizConfig": { + "group": "state-timeline", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bool_on_off" + }, + "overrides": [] + }, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "12.4.0-20012734117" + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "sum without(instance, method, status) (rate(http_requests_total{handler=\"/predict\"}[$__rate_interval]))", + "interval": "1m", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "expr": "rate(http_requests_total{handler=\"/predict\", status=~\"4..|5..\"}[$__rate_interval])", + "instant": false, + "interval": "1m", + "legendFormat": "__auto", + "range": true + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 3, + "links": [], + "title": "HTTP-запросы", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "12.4.0-20012734117" + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "builder", + "exemplar": false, + "expr": "sum without(instance) (increase(model_prediction_value_bucket[10m])) / on() group_left sum(increase(model_prediction_value_count[10m]))", + "format": "heatmap", + "instant": false, + "interval": "10m", + "legendFormat": "<{{le}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Подпись под каждым столбцом обозначает его максимальное соответствующее значение", + "id": 4, + "links": [], + "title": "Предсказанные цены за 10 минут", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "valueMode": "color" + } + }, + "version": "12.4.0-20012734117" + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-4" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-3" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "autoRefresh": "30s", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-1h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Сервис предсказания цен", + "variables": [] + }, + "status": {} +} \ No newline at end of file