konnosDashboard
page.tsx308 lines · main
1import { DocsShell } from '../../components/shell';
2
3export const metadata = { title: 'operator runbook' };
4
5export default function OperatorPage() {
6 return (
7 <DocsShell>
8 <h1 className="font-mono text-2xl tracking-tight">operator runbook</h1>
9 <p className="mt-2 font-mono text-sm text-[var(--color-text-muted)]">
10 wake-up-at-3am playbook for self-hosted briven. every section assumes you have shell
11 access to the host running <code>docker compose</code> for the briven stack.
12 </p>
13 <p className="mt-2 font-mono text-xs text-[var(--color-text-subtle)]">
14 <strong className="text-[var(--color-text)]">Doltgres-first:</strong> production
15 briven.tech stores control + project SQL on <strong>Doltgres</strong> (not stock
16 Postgres as the product engine). files are MinIO S3. see monorepo{' '}
17 <code>DOLTGRES-FIRST.md</code>. self-host compose may still label a service{' '}
18 <code>postgres</code> in older recipes — treat that as “the SQL service,” and prefer
19 Doltgres-native backup (<code>dolt_backup</code>) for real DR once cut over.
20 </p>
21 <p className="mt-2 font-mono text-xs text-[var(--color-text-subtle)]">
22 the first thing to run before any of the recipes below: <code>briven doctor</code>. it
23 prints which sub-system is unhealthy in seconds and rules out half of the diagnostics
24 below.
25 </p>
26
27 <Section title="api won't boot">
28 <p>
29 look at <code>docker compose logs api</code>. the api refuses to start with a clear
30 error message when a required env var is missing or invalid:
31 </p>
32 <ul className="list-disc pl-5">
33 <li>
34 <code>BRIVEN_ENCRYPTION_KEY must be 64 hex chars</code> — generate with{' '}
35 <code>openssl rand -hex 32</code>, paste into <code>.env</code>, redeploy. if you
36 already have project env vars stored encrypted with a different key, see{' '}
37 <em>rotate encryption key</em> below before changing this value or every secret in
38 the database becomes unreadable.
39 </li>
40 <li>
41 <code>BRIVEN_BETTER_AUTH_SECRET must be set</code> — same fix; this one is safe to
42 rotate without data loss but every active session is invalidated.
43 </li>
44 <li>
45 <code>BRIVEN_DATABASE_URL: ECONNREFUSED</code> — the SQL engine (Doltgres / compose
46 service) isn&apos;t up or isn&apos;t reachable on the docker network. check{' '}
47 <code>docker compose ps</code> for the db service and its logs; usually a stale
48 pid lock, fixed by restarting that service.
49 </li>
50 </ul>
51 </Section>
52
53 <Section title="magic link doesn't arrive">
54 <p>
55 if <code>BRIVEN_MITTERA_API_URL</code> or <code>BRIVEN_MITTERA_API_KEY</code> is
56 unset, briven prints the magic link to api stdout instead of sending email. that&rsquo;s
57 intentional for self-host first boot. find it:
58 </p>
59 <Snippet>{`docker compose logs api 2>&1 | grep magic_link | tail -1`}</Snippet>
60 <p>
61 if mittera is configured but mail still isn&rsquo;t arriving, check the api log for{' '}
62 <code>mittera_send_failed</code> entries — the API key may be wrong, the sender
63 domain isn&rsquo;t verified on the mittera side, or mittera is rejecting the request
64 for another reason (the response body is logged, truncated to 240 chars). The link
65 itself is always valid for 10 minutes; re-requesting just sends a fresh one.
66 </p>
67 </Section>
68
69 <Section title="promote yourself to platform admin">
70 <p>
71 <code>/admin</code> is gated by the <code>users.is_admin</code> column. The first user
72 gets it via SQL; everyone after that gets it via the admin tab itself.
73 </p>
74 <Snippet>{`docker compose exec postgres psql -U postgres -d briven_control \\
75 -c "UPDATE users SET is_admin = true WHERE email = '<your-email>'"`}</Snippet>
76 </Section>
77
78 <Section title="rotate encryption key (per-project env vars)">
79 <p>
80 <code>BRIVEN_ENCRYPTION_KEY</code> is the AES-256-GCM key for project env vars at rest.
81 rotating it requires a re-encrypt pass before the new key takes effect. plan a brief
82 maintenance window — the api refuses writes while the migration runs.
83 </p>
84 <ol className="list-decimal pl-5">
85 <li>generate the new key: <code>openssl rand -hex 32</code></li>
86 <li>
87 stop the api: <code>docker compose stop api</code>
88 </li>
89 <li>
90 run the rotation script with the OLD key in <code>OLD_KEY</code> and NEW key in{' '}
91 <code>NEW_KEY</code>:{' '}
92 <Snippet>{`docker compose run --rm \\
93 -e BRIVEN_ENCRYPTION_KEY_OLD=<old> \\
94 -e BRIVEN_ENCRYPTION_KEY=<new> \\
95 api node packages/cli/dist/scripts/rotate-encryption-key.js`}</Snippet>
96 </li>
97 <li>
98 update <code>.env</code> with the new key and start the api:{' '}
99 <code>docker compose up -d api</code>
100 </li>
101 </ol>
102 </Section>
103
104 <Section title="restore from backup">
105 <p>
106 <strong className="text-[var(--color-text)]">hosted briven.tech:</strong> prefer
107 Doltgres-native recovery (<code>dolt_backup</code> / recovery points + MCP{' '}
108 <code>db_recover</code> for project DBs). platform off-site vault (R2/B2) is operator
109 work — not the same as project MinIO file storage at <code>s3.briven.tech</code>.
110 </p>
111 <p className="mt-2">
112 <strong className="text-[var(--color-text)]">self-host sql dumps:</strong> if your
113 compose still dumps via <code>pg_dump</code>-style tools into MinIO (or S3/B2 via{' '}
114 <code>BRIVEN_BACKUP_DESTINATION</code>), list them:
115 </p>
116 <Snippet>{`docker compose exec minio mc ls local/briven-backups/`}</Snippet>
117 <p>
118 restore a single dump (control plane shown — replace db name for a project DB). service
119 name may be <code>doltgres</code> or <code>postgres</code> depending on compose file:
120 </p>
121 <Snippet>{`# 1. download the dump
122docker compose exec minio mc cp local/briven-backups/2026-05-09.sql.gz /tmp/
123
124# 2. stop the api so nothing writes during restore
125docker compose stop api
126
127# 3. drop + recreate the database (use your compose SQL service name)
128docker compose exec doltgres psql -U postgres \\
129 -c "DROP DATABASE briven_control" \\
130 -c "CREATE DATABASE briven_control"
131
132# 4. pipe the dump in
133gunzip -c /tmp/2026-05-09.sql.gz | docker compose exec -T doltgres psql -U postgres -d briven_control
134
135# 5. start the api back up
136docker compose up -d api`}</Snippet>
137 <p>
138 monthly restore drill is wired up in <code>infra/backups/restore-drill.sh</code> and
139 runs against an ephemeral db so you don't need to take prod down to verify the dumps
140 are valid. see also monorepo <code>infra/backups/RESTORE.md</code>.
141 </p>
142 </Section>
143
144 <Section title="suspend a project (abuse, billing past-due, customer ask)">
145 <Snippet>{`docker compose exec doltgres psql -U postgres -d briven_control \\
146 -c "UPDATE projects SET status = 'suspended', suspended_reason = '<reason>' WHERE id = '<p_...>'"`}</Snippet>
147 <p>
148 suspended projects refuse all api calls (404 from <code>/v1/projects/:id/*</code>) and
149 their realtime subscriptions are forcibly closed within the next pump cycle. resume by
150 flipping <code>status</code> back to <code>active</code>.
151 </p>
152 </Section>
153
154 <Section title="invocations are slow / rate-limited">
155 <p>
156 first stop in Grafana: the <em>runtime invocations</em> dashboard. p95 over 500ms or a
157 429 spike points to one of:
158 </p>
159 <ul className="list-disc pl-5">
160 <li>
161 <strong>cold-start storm</strong> — runtime is killing isolates faster than it can
162 warm them. raise <code>BRIVEN_RUNTIME_ISOLATE_TTL_SEC</code> from the default 600
163 and bounce <code>runtime</code>.
164 </li>
165 <li>
166 <strong>rate-limit at the gateway</strong> — a single project is hitting tier
167 ceilings. confirm in the api log: <code>rate_limited project=p_...</code>. resolve
168 with the customer (upgrade tier or reduce hot-loop traffic).
169 </li>
170 <li>
171 <strong>SQL engine saturated</strong> — the database health dashboard will show
172 connection pool exhaustion or lock waits. <code>pg_stat_activity</code> (Postgres
173 wire) tells you which query; remember production is Doltgres.
174 </li>
175 </ul>
176 </Section>
177
178 <Section title="websocket subs flapping">
179 <p>
180 if many clients are reconnecting in a tight loop, the <em>realtime subs</em> dashboard
181 shows a sawtooth open/close pattern. usual causes:
182 </p>
183 <ul className="list-disc pl-5">
184 <li>
185 traefik is timing out idle connections. set{' '}
186 <code>traefik.http.middlewares.briven-ws.headers.customRequestHeaders.X-WS-Timeout=300</code>{' '}
187 and confirm <code>ws_keepalive_ms</code> in the realtime env is &lt; the proxy
188 timeout.
189 </li>
190 <li>
191 the SQL engine is closing the LISTEN connection because the api went idle. the
192 realtime service auto-reconnects in 1s steps with backoff; if you see hundreds of
193 these in seconds the Doltgres host needs investigation.
194 </li>
195 </ul>
196 </Section>
197
198 <Section title="discord alerts stopped firing">
199 <p>
200 alerts route through alertmanager →{' '}
201 <code>benjojo/alertmanager-discord</code> bridge → discord webhook. when the
202 channel goes quiet either prometheus stopped firing (rule drift, scrape down) or
203 the discord webhook url is invalid.
204 </p>
205 <ol className="list-decimal pl-5">
206 <li>
207 confirm prometheus thinks rules are alerting:{' '}
208 <code>docker compose exec prometheus promtool query instant 'ALERTS'</code>.
209 </li>
210 <li>
211 confirm alertmanager received them:{' '}
212 <code>docker compose logs alertmanager --tail 100</code> — look for{' '}
213 <code>msg="Notify success"</code>.
214 </li>
215 <li>
216 confirm the bridge fired:{' '}
217 <code>docker compose logs alertmanager-discord-alerts --tail 100</code>. a 401
218 here means the discord webhook url has been revoked — regenerate it in the
219 discord channel and update <code>DISCORD_WEBHOOK_ALERTS</code> /{' '}
220 <code>DISCORD_WEBHOOK_DEPLOYS</code> in the dokploy env, then restart the two
221 bridge services.
222 </li>
223 </ol>
224 </Section>
225
226 <Section title="backup off-site upload failed">
227 <p>
228 systemd journal: <code>journalctl -u briven-backup.service -n 200</code>. the most
229 common failure is the b2 application key being revoked server-side. easy fix:
230 re-issue at backblaze, update the env, then{' '}
231 <code>systemctl start briven-backup.service</code> to retry without waiting for
232 the timer.
233 </p>
234 <p className="mt-2">
235 required env on the kvm running pg-dump.sh + restore-drill.sh:
236 </p>
237 <ul className="list-disc pl-5">
238 <li><code>BRIVEN_BACKUP_B2_KEY_ID</code> — Backblaze B2 application key id.</li>
239 <li><code>BRIVEN_BACKUP_B2_APP_KEY</code> — secret half of the application key. write-only scope is enough.</li>
240 <li><code>BRIVEN_BACKUP_B2_BUCKET</code> — bucket name (e.g. <code>briven-prod-backups-eu-central</code>).</li>
241 <li><code>BRIVEN_BACKUP_PREFIX</code> — key prefix inside the bucket. defaults to <code>prod</code>.</li>
242 <li><code>BRIVEN_BACKUP_CONTROL_URL</code> + <code>BRIVEN_BACKUP_DATA_URL</code> — postgres dsns for the meta-db + the data plane.</li>
243 </ul>
244 <p className="mt-2">
245 set a bucket lifecycle rule directly in the b2 UI: keep daily snapshots 30 days,
246 monthly snapshots 12 months. the scripts don't manage retention — they assume
247 the bucket does.
248 </p>
249 </Section>
250
251 <Section title="incident disclosure">
252 <p>
253 incidents that touch customer data — get a short message out within 72h and post the
254 full post-mortem at <code>docs.briven.tech/changelog</code> within 30 days. template:
255 </p>
256 <Snippet>{`Title: <one-line summary, no jargon>
257Detected: <utc>
258Resolved: <utc>
259Customer impact: <which projects, what data, exposure window>
260Root cause: <one paragraph>
261Mitigation in place: <bullets>
262Long-term fixes: <bullets>`}</Snippet>
263 </Section>
264
265 <Section title="when in doubt — collect a snapshot">
266 <p>
267 before opening an issue or paging support, run this on the host. attach the resulting
268 tarball:
269 </p>
270 <Snippet>{`d=$(date -u +%Y%m%d-%H%M%S)
271mkdir briven-snapshot-$d && cd briven-snapshot-$d
272docker compose ps > ps.txt
273docker compose logs --tail 500 api > api.log 2>&1
274docker compose logs --tail 500 runtime > runtime.log 2>&1
275docker compose logs --tail 500 realtime > realtime.log 2>&1
276docker compose logs --tail 500 postgres > postgres.log 2>&1
277docker compose exec postgres psql -U postgres -d briven_control \\
278 -c "SELECT count(*) FROM projects" \\
279 -c "SELECT count(*) FROM users" \\
280 -c "SELECT version()" > db.txt 2>&1
281cd .. && tar czf briven-snapshot-$d.tar.gz briven-snapshot-$d/`}</Snippet>
282 <p>
283 everything in the snapshot is operator-side metadata — no customer secrets, no project
284 env vars, no row data.
285 </p>
286 </Section>
287 </DocsShell>
288 );
289}
290
291function Section({ title, children }: { title: string; children: React.ReactNode }) {
292 return (
293 <section className="mt-10">
294 <h2 className="font-mono text-lg">{title}</h2>
295 <div className="mt-2 space-y-3 font-mono text-sm text-[var(--color-text-muted)]">
296 {children}
297 </div>
298 </section>
299 );
300}
301
302function Snippet({ children }: { children: string }) {
303 return (
304 <pre className="overflow-x-auto rounded-md border border-[var(--color-border-subtle)] bg-[var(--color-surface)] p-4 font-mono text-xs">
305 <code>{children}</code>
306 </pre>
307 );
308}