services.yml42 lines · main

1	# Alert rules. Routed via alertmanager → alertmanager-discord bridge →
2	# #briven-alerts (severity=critical|warning) or #briven-deploys (info).
3	# Severity labels MUST match the route matchers in
4	# infra/observability/alertmanager/alertmanager.yml.
5
6	groups:
7	- name: briven-services
8	interval: 30s
9	rules:
10	- alert: ServiceDown
11	expr: up == 0
12	for: 2m
13	labels:
14	severity: critical
15	annotations:
16	summary: '{{ $labels.service }} is unreachable'
17	description: '/metrics on {{ $labels.instance }} has been down for >2 minutes.'
18
19	- alert: HighErrorRate
20	# Triggers when a service's 5xx rate exceeds 5% over the last 5
21	# minutes. Requires the service to expose http_requests_total
22	# with a `status` label — apps/api should add this when its
23	# /metrics endpoint lands.
24	expr: |
25	sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
26	/
27	sum by (service) (rate(http_requests_total[5m]))
28	> 0.05
29	for: 5m
30	labels:
31	severity: warning
32	annotations:
33	summary: '{{ $labels.service }} 5xx rate above 5%'
34
35	- alert: PostgresConnectionsHigh
36	expr: pg_stat_activity_count > 80
37	for: 5m
38	labels:
39	severity: warning
40	annotations:
41	summary: 'postgres has {{ $value }} active connections (>80)'
42	description: 'pool exhaustion is likely if this trends up.'