Мониторинг AI-агентов с нуля: метрики, heartbeat и auto-recovery за час

1# curl http://localhost:9095/metrics | grep ai_agent
2ai_agent_tokens_total{agent="cursor",model="gpt-4o",direction="input"} 284729
3ai_agent_tokens_total{agent="cursor",model="gpt-4o",direction="output"} 42103
4ai_agent_llm_latency_seconds_bucket{le="1"} 842
5ai_agent_llm_latency_seconds_bucket{le="5"} 2891
6ai_agent_cost_usd{agent="cursor",team="backend"} 12.47
7ai_agent_policy_violations{agent="claude-code",rule="no_prod_write"} 3

1[Unit]
2Description=FlowLink AI Agent Monitor
3After=network-online.target
4Wants=network-online.target
5​
6[Service]
7Type=simple
8ExecStart=/usr/local/bin/flowlink agent --config /etc/flowlink/agent.yaml
9Restart=on-failure
10RestartSec=10
11WatchdogSec=30
12​
13# Health check через HTTP endpoint
14ExecStartPost=/bin/sleep 3
15ExecStartPost=/usr/bin/curl -sf http://localhost:9095/health || exit 1
16​
17# Resource limits
18MemoryMax=2G
19CPUQuota=200%
20​
21# Environment
22Environment=FLOWLINK_LOG_LEVEL=info
23Environment=FLOWLINK_METRICS_PORT=9095
24​
25[Install]
26WantedBy=multi-user.target

1groups:
2  - name: ai_agent_health
3    rules:
4      # Агент зациклился: > 100 tool calls за 5 минут
5      - alert: AgentToolCallStorm
6        expr: rate(ai_agent_tool_calls_total[5m]) > 0.33
7        for: 2m
8        labels:
9          severity: warning
10        annotations:
11          summary: "Agent {{$labels.agent}} makes >100 tool calls/min"
12          runbook: "https://wiki.internal/runbooks/tool-storm"
13​
14      # LLM latency деградировала
15      - alert: LLMLatencyHigh
16        expr: histogram_quantile(0.95, rate(ai_agent_llm_latency_seconds_bucket[5m])) > 15
17        for: 5m
18        labels:
19          severity: warning
20        annotations:
21          summary: "p95 LLM latency > 15s for {{$labels.model}}"
22​
23      # Перерасход бюджета: > $50/час
24      - alert: BudgetOverrun
25        expr: rate(ai_agent_cost_usd[1h]) > 50
26        for: 10m
27        labels:
28          severity: critical
29        annotations:
30          summary: "Agent {{$labels.agent}} spending >$50/hr"
31​
32      # Policy violations spike
33      - alert: PolicyViolationsSpike
34        expr: increase(ai_agent_policy_violations[1h]) > 10
35        labels:
36          severity: warning
37        annotations:
38          summary: "{{$labels.agent}} triggered {{$value}} policy violations in 1h"
39​
40      # Context window почти полон (>80%)
41      - alert: ContextBloat
42        expr: ai_agent_context_size_tokens > 160000
43        for: 5m
44        labels:
45          severity: info
46        annotations:
47          summary: "{{$labels.agent}} context at {{$value}} tokens (cap: 200k)"

1auto_recovery:
2  enabled: true
3​
4  # При budget overrun — переключаем на дешёвую модель
5  triggers:
6    - alert: "BudgetOverrun"
7      actions:
8        - type: switch_model
9          from: "gpt-4o"
10          to: "gpt-4o-mini"
11        - type: notify
12          channel: "slack"
13          message: "Switched {{agent}} to gpt-4o-mini due to budget overrun"
14​
15  # При tool storm — замораживаем агента на 60 секунд
16    - alert: "AgentToolCallStorm"
17      actions:
18        - type: throttle
19          max_calls_per_minute: 5
20          duration: 60s
21        - type: notify
22          channel: "pagerduty"
23          severity: "high"
24​
25  # При latency — fallback на другой provider
26    - alert: "LLMLatencyHigh"
27      actions:
28        - type: switch_provider
29          from: "anthropic"
30          to: "openai"
31          fallback_model: "gpt-4o"
32        - type: log
33          message: "Provider switched due to latency"

1# Top-left: cost ($/час по агентам)
2sum(rate(ai_agent_cost_usd[1h])) by (agent)
3​
4# Top-right: p95 latency по моделям
5histogram_quantile(0.95, rate(ai_agent_llm_latency_seconds_bucket[5m]))
6​
7# Bottom-left: нарушения политик за час
8increase(ai_agent_policy_violations[1h])
9​
10# Bottom-right: размер контекста во времени
11ai_agent_context_size_tokens