konnosDashboard
test-helpers.ts499 lines · main
1// Shared harness for runtime integration tests. Materializes a real
2// on-disk bundle, spawns a real Deno isolate via the same `bunChildSpawn`
3// adapter the production bootstrap uses, runs ONE invocation, returns the
4// `InvokeResult` plus a teardown callback.
5//
6// Tasks 17–22 all share this helper. Keep it test-agnostic: no test-name
7// branching, no hard-coded fixtures beyond the "@briven/cli/server" stub.
8
9import { copyFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
10import { tmpdir } from 'node:os';
11import { join, resolve } from 'node:path';
12
13import { spawn as bunSpawn } from 'bun';
14
15import { IsolatePoolImpl, type SpawnFn, type SpawnedChild } from '../../src/pool-manager.js';
16import type { Bundle, InvokeRequest, InvokeResult } from '../../src/types.js';
17
18// Resolve the Deno binary at fixture time (not module load) so each test
19// can override via env. Falls through to the default install path on the
20// dev machine, then to bare `deno` on PATH (CI).
21function resolveDenoPath(): string {
22 return process.env.BRIVEN_RUNTIME_DENO_PATH ?? '/Users/flndrn/.deno/bin/deno';
23}
24
25export interface FixtureOpts {
26 /** Source code for the customer function. Written to briven/functions/<fnName>.ts */
27 fnSource: string;
28 /** Function name (matches the export and the file name). */
29 fnName: string;
30 /** Deployment ID echoed in the ready handshake. */
31 deploymentId: string;
32 /** Optional: extra config overrides on the pool. */
33 poolConfig?: Partial<{
34 invocationTimeoutMs: number;
35 idleKillMs: number;
36 maxInvocationsPerIsolate: number;
37 maxMemoryMb: number;
38 maxIsolates: number;
39 crashLoopThreshold: number;
40 crashLoopWindowMs: number;
41 }>;
42 /** Optional: surface isolate stderr lines (for debugging a failing test). */
43 onLog?: (line: unknown, projectId: string) => void;
44 /** Optional: project env vars exposed via Deno.env (allow-env list). */
45 projectEnv?: Record<string, string>;
46}
47
48export interface FixtureResult {
49 result: InvokeResult;
50 pool: IsolatePoolImpl;
51 cleanup: () => Promise<void>;
52}
53
54/**
55 * Materialize a fake bundle on disk, construct a pool against the real
56 * Deno binary, run one invoke, return the result + the pool (for tests
57 * that want to inspect state) + a cleanup callback.
58 *
59 * The caller MUST call `cleanup()` even on error (use try/finally).
60 */
61export async function runIntegrationFixture(opts: FixtureOpts): Promise<FixtureResult> {
62 const workDir = await mkdtemp(join(tmpdir(), 'briven-int-'));
63 const bundleDir = join(workDir, 'bundle');
64 const isolateBase = join(workDir, 'isolates');
65 const runtimeStubDir = join(workDir, 'stub');
66
67 // Vendor the real isolate-runtime stubs into the work dir. The
68 // materializer will copy them again into the per-isolate tmp dir, but
69 // it expects them to live under `runtimeStubDir`.
70 const realStubDir = resolve(import.meta.dir, '..', '..', 'src', 'isolate-runtime');
71 await mkdir(runtimeStubDir, { recursive: true });
72 for (const f of ['loop.ts', 'server.ts', 'types.ts']) {
73 await copyFile(join(realStubDir, f), join(runtimeStubDir, f));
74 }
75
76 // Customer function source.
77 await mkdir(join(bundleDir, 'functions'), { recursive: true });
78 await writeFile(join(bundleDir, 'functions', `${opts.fnName}.ts`), opts.fnSource);
79
80 await mkdir(isolateBase, { recursive: true });
81
82 const bundle: Bundle = {
83 projectId: 'p-int',
84 deploymentId: opts.deploymentId,
85 functionNames: [opts.fnName],
86 directory: bundleDir,
87 };
88 const request: InvokeRequest = {
89 projectId: 'p-int',
90 functionName: opts.fnName,
91 args: {},
92 deploymentId: opts.deploymentId,
93 requestId: `req-${Date.now()}`,
94 auth: null,
95 };
96
97 const denoPath = resolveDenoPath();
98 const spawn = makeBunChildSpawn(denoPath);
99
100 const pool = new IsolatePoolImpl({
101 spawn,
102 runtimeStubDir,
103 isolateBaseDir: isolateBase,
104 maxIsolates: opts.poolConfig?.maxIsolates ?? 50,
105 maxMemoryMb: opts.poolConfig?.maxMemoryMb ?? 128,
106 invocationTimeoutMs: opts.poolConfig?.invocationTimeoutMs ?? 30_000,
107 idleKillMs: opts.poolConfig?.idleKillMs ?? 10 * 60_000,
108 maxInvocationsPerIsolate: opts.poolConfig?.maxInvocationsPerIsolate ?? 1000,
109 crashLoopThreshold: opts.poolConfig?.crashLoopThreshold ?? 3,
110 crashLoopWindowMs: opts.poolConfig?.crashLoopWindowMs ?? 60_000,
111 runQueryProxy: async () => [],
112 onLog: (line, projectId) => {
113 if (opts.onLog) opts.onLog(line, projectId);
114 },
115 loadProjectEnv: async () => opts.projectEnv ?? {},
116 denoPath,
117 });
118
119 const result = await pool.invoke(bundle, request);
120
121 const cleanup = async () => {
122 await pool.shutdown();
123 await rm(workDir, { recursive: true, force: true });
124 };
125
126 return { result, pool, cleanup };
127}
128
129/**
130 * Adapter that maps Bun's `spawn` API onto the `SpawnFn` interface the
131 * pool expects. Mirrors `apps/runtime/src/runtime-bootstrap.ts` —
132 * intentionally duplicated for now (Phase 2 may extract a shared helper).
133 */
134function makeBunChildSpawn(denoPath: string): SpawnFn {
135 return async ({ args, env: childEnv, cwd }) => {
136 const proc = bunSpawn({
137 cmd: [denoPath, ...args],
138 cwd,
139 env: childEnv,
140 stdin: 'pipe',
141 stdout: 'pipe',
142 stderr: 'pipe',
143 });
144
145 const stdoutLines = lineIterator(proc.stdout as ReadableStream<Uint8Array>);
146 const stderrLines = lineIterator(proc.stderr as ReadableStream<Uint8Array>);
147
148 const child: SpawnedChild = {
149 pid: proc.pid,
150 stdin: {
151 write: async (line: string) => {
152 const n = proc.stdin.write(line);
153 await proc.stdin.flush();
154 return typeof n === 'number' ? n > 0 : Boolean(n);
155 },
156 end: () => proc.stdin.end(),
157 },
158 stdout: {
159 next: () => stdoutLines.next().then((r) => (r.done ? null : r.value)),
160 },
161 stderr: {
162 next: () => stderrLines.next().then((r) => (r.done ? null : r.value)),
163 },
164 wait: async () => {
165 const exitCode = await proc.exited;
166 return { exitCode, signal: null };
167 },
168 kill: (signal: string) => proc.kill(signal as never),
169 };
170 return child;
171 };
172}
173
174// ---------------------------------------------------------------------------
175// Multi-invocation helpers — used by Tasks 20–22.
176//
177// All three reuse the same per-test scratch dir layout `runIntegrationFixture`
178// builds (one workDir, one runtimeStubDir, one isolateBase). They diverge only
179// on how many bundles they materialize and how many invokes they fire against
180// the same pool.
181// ---------------------------------------------------------------------------
182
183/** Vendor stub files into runtimeStubDir. Mirrors `runIntegrationFixture`. */
184async function vendorRuntimeStubs(runtimeStubDir: string): Promise<void> {
185 await mkdir(runtimeStubDir, { recursive: true });
186 const realStubDir = resolve(import.meta.dir, '..', '..', 'src', 'isolate-runtime');
187 for (const f of ['loop.ts', 'server.ts', 'types.ts']) {
188 await copyFile(join(realStubDir, f), join(runtimeStubDir, f));
189 }
190}
191
192/**
193 * Wrap a SpawnFn so each spawn's `child.pid` is recorded into `sink`.
194 * Used by `runTwoSequentialInvocations` and `runIdleKillFixture` to prove
195 * a respawn happened (different PID).
196 */
197function withPidRecorder(inner: SpawnFn, sink: number[]): SpawnFn {
198 return async (opts) => {
199 const child = await inner(opts);
200 sink.push(child.pid);
201 return child;
202 };
203}
204
205export interface TwoInvocationsOpts {
206 first: { fnSource: string; fnName: string; deploymentId: string };
207 second: { fnSource: string; fnName: string; deploymentId: string };
208 poolConfig?: FixtureOpts['poolConfig'];
209}
210
211export interface TwoInvocationsResult {
212 first: InvokeResult;
213 second: InvokeResult;
214 /** PIDs observed across the two invocations. Should be 2 distinct values for deploy invalidation. */
215 pidsObserved: number[];
216 cleanup: () => Promise<void>;
217}
218
219/**
220 * Two sequential invocations against the SAME pool/projectId, with two
221 * different `deploymentId`s and two separately-materialized bundles. The
222 * second invocation triggers deploy-invalidation: the first isolate is
223 * retired and a fresh one cold-starts. PIDs are tracked via a wrapped
224 * SpawnFn so the test can assert two distinct PIDs.
225 */
226export async function runTwoSequentialInvocations(
227 opts: TwoInvocationsOpts,
228): Promise<TwoInvocationsResult> {
229 const workDir = await mkdtemp(join(tmpdir(), 'briven-int-'));
230 const bundleDirA = join(workDir, 'bundle-a');
231 const bundleDirB = join(workDir, 'bundle-b');
232 const isolateBase = join(workDir, 'isolates');
233 const runtimeStubDir = join(workDir, 'stub');
234 await vendorRuntimeStubs(runtimeStubDir);
235
236 await mkdir(join(bundleDirA, 'functions'), { recursive: true });
237 await writeFile(join(bundleDirA, 'functions', `${opts.first.fnName}.ts`), opts.first.fnSource);
238 await mkdir(join(bundleDirB, 'functions'), { recursive: true });
239 await writeFile(join(bundleDirB, 'functions', `${opts.second.fnName}.ts`), opts.second.fnSource);
240 await mkdir(isolateBase, { recursive: true });
241
242 const denoPath = resolveDenoPath();
243 const pidsObserved: number[] = [];
244 const spawn = withPidRecorder(makeBunChildSpawn(denoPath), pidsObserved);
245
246 const pool = new IsolatePoolImpl({
247 spawn,
248 runtimeStubDir,
249 isolateBaseDir: isolateBase,
250 maxIsolates: opts.poolConfig?.maxIsolates ?? 50,
251 maxMemoryMb: opts.poolConfig?.maxMemoryMb ?? 128,
252 invocationTimeoutMs: opts.poolConfig?.invocationTimeoutMs ?? 30_000,
253 idleKillMs: opts.poolConfig?.idleKillMs ?? 10 * 60_000,
254 maxInvocationsPerIsolate: opts.poolConfig?.maxInvocationsPerIsolate ?? 1000,
255 crashLoopThreshold: opts.poolConfig?.crashLoopThreshold ?? 3,
256 crashLoopWindowMs: opts.poolConfig?.crashLoopWindowMs ?? 60_000,
257 runQueryProxy: async () => [],
258 onLog: () => {},
259 loadProjectEnv: async () => ({}),
260 denoPath,
261 });
262
263 const bundleA: Bundle = {
264 projectId: 'p-int',
265 deploymentId: opts.first.deploymentId,
266 functionNames: [opts.first.fnName],
267 directory: bundleDirA,
268 };
269 const requestA: InvokeRequest = {
270 projectId: 'p-int',
271 functionName: opts.first.fnName,
272 args: {},
273 deploymentId: opts.first.deploymentId,
274 requestId: `req-${Date.now()}-a`,
275 auth: null,
276 };
277 const first = await pool.invoke(bundleA, requestA);
278
279 const bundleB: Bundle = {
280 projectId: 'p-int',
281 deploymentId: opts.second.deploymentId,
282 functionNames: [opts.second.fnName],
283 directory: bundleDirB,
284 };
285 const requestB: InvokeRequest = {
286 projectId: 'p-int',
287 functionName: opts.second.fnName,
288 args: {},
289 deploymentId: opts.second.deploymentId,
290 requestId: `req-${Date.now()}-b`,
291 auth: null,
292 };
293 const second = await pool.invoke(bundleB, requestB);
294
295 const cleanup = async () => {
296 await pool.shutdown();
297 await rm(workDir, { recursive: true, force: true });
298 };
299 return { first, second, pidsObserved, cleanup };
300}
301
302export interface RepeatedFixtureOpts {
303 fnName: string;
304 fnSource: string;
305 deploymentId: string;
306 count: number;
307 poolConfig?: FixtureOpts['poolConfig'];
308}
309
310export interface RepeatedFixtureResult {
311 results: InvokeResult[];
312 cleanup: () => Promise<void>;
313}
314
315/**
316 * Run `count` sequential invocations against the same pool, projectId,
317 * deploymentId, and bundle. Used by the crash-loop breaker test so the
318 * breaker history accumulates across all calls.
319 */
320export async function runFixtureRepeated(
321 opts: RepeatedFixtureOpts,
322): Promise<RepeatedFixtureResult> {
323 const workDir = await mkdtemp(join(tmpdir(), 'briven-int-'));
324 const bundleDir = join(workDir, 'bundle');
325 const isolateBase = join(workDir, 'isolates');
326 const runtimeStubDir = join(workDir, 'stub');
327 await vendorRuntimeStubs(runtimeStubDir);
328
329 await mkdir(join(bundleDir, 'functions'), { recursive: true });
330 await writeFile(join(bundleDir, 'functions', `${opts.fnName}.ts`), opts.fnSource);
331 await mkdir(isolateBase, { recursive: true });
332
333 const denoPath = resolveDenoPath();
334 const spawn = makeBunChildSpawn(denoPath);
335
336 const pool = new IsolatePoolImpl({
337 spawn,
338 runtimeStubDir,
339 isolateBaseDir: isolateBase,
340 maxIsolates: opts.poolConfig?.maxIsolates ?? 50,
341 maxMemoryMb: opts.poolConfig?.maxMemoryMb ?? 128,
342 invocationTimeoutMs: opts.poolConfig?.invocationTimeoutMs ?? 30_000,
343 idleKillMs: opts.poolConfig?.idleKillMs ?? 10 * 60_000,
344 maxInvocationsPerIsolate: opts.poolConfig?.maxInvocationsPerIsolate ?? 1000,
345 crashLoopThreshold: opts.poolConfig?.crashLoopThreshold ?? 3,
346 crashLoopWindowMs: opts.poolConfig?.crashLoopWindowMs ?? 60_000,
347 runQueryProxy: async () => [],
348 onLog: () => {},
349 loadProjectEnv: async () => ({}),
350 denoPath,
351 });
352
353 const bundle: Bundle = {
354 projectId: 'p-int',
355 deploymentId: opts.deploymentId,
356 functionNames: [opts.fnName],
357 directory: bundleDir,
358 };
359
360 const results: InvokeResult[] = [];
361 for (let i = 0; i < opts.count; i++) {
362 const request: InvokeRequest = {
363 projectId: 'p-int',
364 functionName: opts.fnName,
365 args: {},
366 deploymentId: opts.deploymentId,
367 requestId: `req-${Date.now()}-${i}`,
368 auth: null,
369 };
370 results.push(await pool.invoke(bundle, request));
371 }
372
373 const cleanup = async () => {
374 await pool.shutdown();
375 await rm(workDir, { recursive: true, force: true });
376 };
377 return { results, cleanup };
378}
379
380export interface IdleKillFixtureOpts {
381 fnSource: string;
382 fnName: string;
383 deploymentId: string;
384 /** Idle threshold in ms; the helper waits 2x this between invokes before triggering the sweeper. */
385 idleKillMs: number;
386}
387
388export interface IdleKillFixtureResult {
389 firstPid: number;
390 secondPid: number;
391 cleanup: () => Promise<void>;
392}
393
394/**
395 * Two sequential invocations against the same pool with the idle sweeper
396 * triggered between them, proving the first isolate gets retired and the
397 * second invocation cold-starts a fresh process.
398 */
399export async function runIdleKillFixture(
400 opts: IdleKillFixtureOpts,
401): Promise<IdleKillFixtureResult> {
402 const workDir = await mkdtemp(join(tmpdir(), 'briven-int-'));
403 const bundleDir = join(workDir, 'bundle');
404 const isolateBase = join(workDir, 'isolates');
405 const runtimeStubDir = join(workDir, 'stub');
406 await vendorRuntimeStubs(runtimeStubDir);
407
408 await mkdir(join(bundleDir, 'functions'), { recursive: true });
409 await writeFile(join(bundleDir, 'functions', `${opts.fnName}.ts`), opts.fnSource);
410 await mkdir(isolateBase, { recursive: true });
411
412 const denoPath = resolveDenoPath();
413 const pidsObserved: number[] = [];
414 const spawn = withPidRecorder(makeBunChildSpawn(denoPath), pidsObserved);
415
416 const pool = new IsolatePoolImpl({
417 spawn,
418 runtimeStubDir,
419 isolateBaseDir: isolateBase,
420 maxIsolates: 50,
421 maxMemoryMb: 128,
422 invocationTimeoutMs: 30_000,
423 idleKillMs: opts.idleKillMs,
424 maxInvocationsPerIsolate: 1000,
425 crashLoopThreshold: 3,
426 crashLoopWindowMs: 60_000,
427 runQueryProxy: async () => [],
428 onLog: () => {},
429 loadProjectEnv: async () => ({}),
430 denoPath,
431 });
432
433 const bundle: Bundle = {
434 projectId: 'p-int',
435 deploymentId: opts.deploymentId,
436 functionNames: [opts.fnName],
437 directory: bundleDir,
438 };
439 const makeRequest = (suffix: string): InvokeRequest => ({
440 projectId: 'p-int',
441 functionName: opts.fnName,
442 args: {},
443 deploymentId: opts.deploymentId,
444 requestId: `req-${Date.now()}-${suffix}`,
445 auth: null,
446 });
447
448 const first = await pool.invoke(bundle, makeRequest('a'));
449 if (!first.ok) {
450 await pool.shutdown();
451 await rm(workDir, { recursive: true, force: true });
452 throw new Error(
453 `idle-kill helper: first invoke failed with code=${first.code} message=${first.message}`,
454 );
455 }
456 const firstPid = pidsObserved[0] ?? -1;
457
458 // Wait long enough that the entry's lastActivityAt is older than idleKillMs.
459 await new Promise((r) => setTimeout(r, Math.max(opts.idleKillMs * 2, 50)));
460 await pool.triggerIdleCheck();
461
462 const second = await pool.invoke(bundle, makeRequest('b'));
463 if (!second.ok) {
464 await pool.shutdown();
465 await rm(workDir, { recursive: true, force: true });
466 throw new Error(
467 `idle-kill helper: second invoke failed with code=${second.code} message=${second.message}`,
468 );
469 }
470 const secondPid = pidsObserved[1] ?? -1;
471
472 const cleanup = async () => {
473 await pool.shutdown();
474 await rm(workDir, { recursive: true, force: true });
475 };
476 return { firstPid, secondPid, cleanup };
477}
478
479async function* lineIterator(
480 stream: ReadableStream<Uint8Array>,
481): AsyncGenerator<string, void, void> {
482 const reader = stream.getReader();
483 const decoder = new TextDecoder();
484 let buf = '';
485 while (true) {
486 const { value, done } = await reader.read();
487 if (done) {
488 if (buf.length > 0) yield buf;
489 return;
490 }
491 buf += decoder.decode(value, { stream: true });
492 let nl: number;
493 while ((nl = buf.indexOf('\n')) !== -1) {
494 const line = buf.slice(0, nl);
495 buf = buf.slice(nl + 1);
496 if (line) yield line;
497 }
498 }
499}