Мониторинг и Alerting

1# Мониторинг конфигурация (flowlink.config.yml)
2monitoring:
3  enabled: true
4  
5  # Health thresholds
6  thresholds:
7    healthy:
8      cpu: 50
9      memory: 70
10      disk: 70
11    degraded:
12      cpu: 75
13      memory: 85
14      disk: 85
15    critical:
16      cpu: 95
17      memory: 95
18      disk: 95
19    offline_seconds: 60
20  
21  # Metrics
22  metrics:
23    enabled: true
24    path: "/metrics"
25    namespace: "flowlink"
26  
27  # Timeseries retention
28  timeseries:
29    retention_days: 30
30    resolution_seconds: 60

1# Метрики конкретного агента
2curl https://api.flowlink.io/api/v1/agents/prod-db-01/health \
3  -H "Authorization: Bearer $TOKEN"
4
5# Timeseries за последние 24 часа
6curl "https://api.flowlink.io/api/v1/agents/prod-db-01/health/timeseries?from=2026-04-21T00:00:00Z&to=2026-04-22T00:00:00Z" \
7  -H "Authorization: Bearer $TOKEN"
8
9# Обзор здоровья всех агентов
10curl https://api.flowlink.io/api/v1/agents/health/overview \
11  -H "Authorization: Bearer $TOKEN"
12
13# Только критические агенты
14curl "https://api.flowlink.io/api/v1/agents/health/overview?status=critical" \
15  -H "Authorization: Bearer $TOKEN"

1{
2  "agent_id": "prod-db-01",
3  "status": "healthy",
4  "cpu": { "usage_percent": 42.5, "load_avg": 1.2 },
5  "memory": { "usage_percent": 65.3, "total_mb": 16384, "used_mb": 10701 },
6  "disk": { "usage_percent": 58.2, "total_gb": 100, "used_gb": 58.2 },
7  "network": {
8    "in_mb_per_s": 12.4,
9    "out_mb_per_s": 8.7,
10    "connections": 15
11  },
12  "last_updated": "2026-04-21T20:00:00Z",
13  "uptime_seconds": 86400
14}

1curl https://api.flowlink.io/api/v1/monitoring/overview \
2  -H "Authorization: Bearer $TOKEN"

1{
2  "total_agents": 25,
3  "healthy": 22,
4  "degraded": 2,
5  "critical": 1,
6  "offline": 0,
7  "avg_cpu": 35.7,
8  "avg_memory": 62.3,
9  "avg_disk": 54.8,
10  "relay": {
11    "requests_per_second": 12.5,
12    "avg_latency_ms": 28,
13    "p95_latency_ms": 85,
14    "error_rate": 0.02,
15    "active_connections": 45
16  },
17  "shield": {
18    "scans_per_second": 12.5,
19    "avg_scan_time_ms": 8,
20    "blocked_rate": 0.03
21  },
22  "system": {
23    "db_connections_active": 12,
24    "queue_size": 42,
25    "memory_usage_mb": 512
26  },
27  "last_updated": "2026-04-21T20:00:00Z"
28}

1{
2  "agent_id": "prod-db-01",
3  "resolution": "1m",
4  "metrics": {
5    "cpu": [
6      { "timestamp": "2026-04-21T19:00:00Z", "value": 38.2 },
7      { "timestamp": "2026-04-21T19:01:00Z", "value": 41.5 },
8      { "timestamp": "2026-04-21T19:02:00Z", "value": 42.5 }
9    ],
10    "memory": [
11      { "timestamp": "2026-04-21T19:00:00Z", "value": 64.8 },
12      { "timestamp": "2026-04-21T19:01:00Z", "value": 65.0 },
13      { "timestamp": "2026-04-21T19:02:00Z", "value": 65.3 }
14    ]
15  }
16}

1# prometheus.yml
2scrape_configs:
3  - job_name: 'flowlink'
4    scrape_interval: 15s
5    static_configs:
6      - targets: ['flowlink-relay:9090']
7    metrics_path: '/metrics'
8    relabel_configs:
9      - source_labels: [__address__]
10        target_label: instance
11        replacement: 'flowlink-relay'

1# Доступные метрики
2curl -s "https://api.flowlink.io/metrics" | head -30
3
4# flowlink_relay_requests_total{method="POST",path="/mcp/call",status="200"} 15420
5# flowlink_relay_request_duration_seconds_bucket{le="0.01"} 8500
6# flowlink_relay_request_duration_seconds_bucket{le="0.05"} 14200
7# flowlink_shield_scan_duration_seconds{layer="L1"} 0.003
8# flowlink_shield_scan_duration_seconds{layer="L2"} 0.005
9# flowlink_shield_scan_total{layer="L1",result="passed"} 15000
10# flowlink_shield_scan_total{layer="L1",result="blocked"} 420
11# flowlink_policy_evaluation_duration_seconds 0.003
12# flowlink_policy_evaluation_total{action="allow"} 14200
13# flowlink_policy_evaluation_total{action="deny"} 450
14# flowlink_agent_cpu_percent{agent_id="agent-prod-01"} 23.5
15# flowlink_agent_ram_percent{agent_id="agent-prod-01"} 61.2

1# Alerting rules (flowlink.config.yml)
2alerting:
3  rules:
4    - name: "agent_cpu_critical"
5      description: "CPU usage above 95%"
6      condition:
7        metric: "flowlink_agent_cpu_percent"
8        operator: ">"
9        threshold: 95
10        for_seconds: 300
11      actions:
12        - type: "notify"
13          channel: ["telegram", "email"]
14        - type: "webhook"
15          url: "https://hooks.slack.com/services/xxx"
16    
17    - name: "shield_blocked_spike"
18      description: "Shield blocked commands spike"
19      condition:
20        metric: "flowlink_shield_scan_total"
21        filter: { result: "blocked" }
22        operator: ">"
23        threshold: 10
24        for_seconds: 60
25      actions:
26        - type: "notify"
27          channel: "telegram"
28    
29    - name: "agent_offline"
30      description: "Agent has not sent heartbeat"
31      condition:
32        metric: "flowlink_agent_status"
33        filter: { status: "offline" }
34        operator: "=="
35        threshold: 1
36        for_seconds: 0
37      actions:
38        - type: "notify"
39          channel: ["telegram", "email", "pagerduty"]
40        - type: "webhook"
41          url: "https://your-webhook.example.com/alert"

1{
2  "alert_id": "alert_abc123",
3  "rule_name": "agent_cpu_critical",
4  "severity": "critical",
5  "description": "CPU usage above 95% for agent-prod-01",
6  "entity": {
7    "type": "agent",
8    "id": "agent-prod-01",
9    "name": "prod-deploy-agent"
10  },
11  "metric": {
12    "name": "flowlink_agent_cpu_percent",
13    "value": 97.3,
14    "threshold": 95,
15    "operator": ">"
16  },
17  "duration_seconds": 300,
18  "timestamp": "2026-04-21T20:00:00Z",
19  "dashboard_url": "https://flowlink.io/dashboard/agents/agent-prod-01"
20}

1# Импорт дашборда в Grafana
2# Menu → Dashboards → Import → Paste JSON
3
4# Или через API:
5curl -X POST "https://grafana.example.com/api/dashboards/import" \
6  -H "Authorization: Bearer $GRAFANA_TOKEN" \
7  -H "Content-Type: application/json" \
8  -d '{"dashboard": {"json": ...}, "overwrite": true}'