Testing Strategy

// Consumer-driven contracts in miniature: each consumer records the fields it actually reads from an API response,
// and the provider checks every candidate build against all recorded contracts before it deploys.
const contracts = {
  "web UI device page": {
    request: "GET /api/v1/devices/dev-1",
    expects: { id: "string", name: "string", status: ["online", "offline"], "firmware.version": "string" },
  },
  "room-booking service": {
    request: "GET /api/v1/devices/dev-1",
    expects: { id: "string", status: ["online", "offline"] },
  },
};
const get = (obj, path) => path.split(".").reduce((o, k) => (o == null ? undefined : o[k]), obj);
function verify(contract, response) {
  const problems = [];
  for (const [path, rule] of Object.entries(contract.expects)) {
    const value = get(response, path);
    if (value === undefined) problems.push(`missing ${path}`);
    else if (Array.isArray(rule) && !rule.includes(value)) problems.push(`${path} is "${value}", expected one of ${rule.join("|")}`);
    else if (!Array.isArray(rule) && typeof value !== rule) problems.push(`${path} is a ${typeof value}, expected ${rule}`);
  }
  return problems;
}
const providerBuilds = {
  "api v1.8 (current)": { id: "dev-1", name: "Boardroom 4", status: "online", firmware: { version: "10.2.1" } },
  "api v1.9 adds lastSeen": { id: "dev-1", name: "Boardroom 4", status: "online", firmware: { version: "10.2.1" }, lastSeen: "2026-10-27T09:00:00Z" },
  "api v2.0 flattens firmware": { id: "dev-1", name: "Boardroom 4", status: "online", firmwareVersion: "10.2.1" },
  "api v2.1 adds status 'updating'": { id: "dev-1", name: "Boardroom 4", status: "updating", firmware: { version: "10.3.0" } },
};
for (const [build, response] of Object.entries(providerBuilds)) {
  const results = Object.entries(contracts).map(([consumer, c]) => [consumer, verify(c, response)]);
  const ok = results.every(([, p]) => p.length === 0);
  console.log(`${build}: ${ok ? "safe to deploy" : "BLOCKED"}`);
  for (const [consumer, p] of results) if (p.length) console.log(`  breaks ${consumer}: ${p.join("; ")}`);
}

api v1.8 (current): safe to deploy
api v1.9 adds lastSeen: safe to deploy
api v2.0 flattens firmware: BLOCKED
  breaks web UI device page: missing firmware.version
api v2.1 adds status 'updating': BLOCKED
  breaks web UI device page: status is "updating", expected one of online|offline
  breaks room-booking service: status is "updating", expected one of online|offline

// Test doubles in one node:test suite: a fake works, a stub answers, a spy records and the test checks its calls afterwards.
import { test, mock, run } from "node:test";
import assert from "node:assert/strict";

// Code under test: tell every online device on an old firmware version that an update is available.
export async function announceUpdate({ devices, getLatestVersion, push }) {
  const latest = await getLatestVersion();
  const outdated = (await devices.list()).filter((d) => d.online && d.firmware !== latest);
  for (const d of outdated) await push(d.id, { type: "update-available", version: latest });
  return outdated.length;
}
class FakeDeviceStore { // a real, working implementation, just in memory
  #rows = new Map();
  async add(device) { this.#rows.set(device.id, { ...device }); }
  async list() { return [...this.#rows.values()]; }
}

if (process.env.NODE_TEST_CONTEXT) {
  test("pushes only to online devices on old firmware", async () => {
    const devices = new FakeDeviceStore(); // fake
    await devices.add({ id: "dev-1", online: true, firmware: "10.2.0" });
    await devices.add({ id: "dev-2", online: false, firmware: "10.2.0" });
    await devices.add({ id: "dev-3", online: true, firmware: "10.3.0" });
    const getLatestVersion = async () => "10.3.0"; // stub: canned answer, never checked
    const push = mock.fn(async () => {}); // spy: records every call
    assert.equal(await announceUpdate({ devices, getLatestVersion, push }), 1);
    assert.equal(push.mock.callCount(), 1);
    assert.deepEqual(push.mock.calls[0].arguments, ["dev-1", { type: "update-available", version: "10.3.0" }]);
  });
  test("a failing push stops the run (current behaviour, pinned)", async () => {
    const devices = new FakeDeviceStore();
    await devices.add({ id: "dev-1", online: true, firmware: "10.2.0" });
    await devices.add({ id: "dev-4", online: true, firmware: "10.1.0" });
    const push = mock.fn(async (id) => { if (id === "dev-1") throw new Error("push service 503"); });
    await assert.rejects(announceUpdate({ devices, getLatestVersion: async () => "10.3.0", push }), /503/);
    assert.equal(push.mock.callCount(), 1); // dev-4 was never told: a finding to raise, not to hide
  });
} else {
  for await (const { type, data } of run({ files: [import.meta.filename] })) {
    if (type === "test:pass" && data.nesting === 0) console.log(`✔ ${data.name}`);
    if (type === "test:fail" && data.nesting === 0) console.log(`✖ ${data.name}\n  ${data.details.error.cause?.message}`);
  }
}

✔ pushes only to online devices on old firmware
✔ a failing push stops the run (current behaviour, pinned)

// Diff coverage: of the lines a pull request ADDS, how many ran under the tests? Real git for the diff, Node's
// test runner for the coverage. The file's overall figure can look fine while the new code is untested.
import { run } from "node:test";
import { execFileSync } from "node:child_process";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

const dir = mkdtempSync(join(tmpdir(), "diffcov-"));
const env = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1", GIT_AUTHOR_NAME: "Dev",
  GIT_AUTHOR_EMAIL: "dev@example.com", GIT_COMMITTER_NAME: "Dev", GIT_COMMITTER_EMAIL: "dev@example.com",
  GIT_AUTHOR_DATE: "2026-10-20T10:00:00Z", GIT_COMMITTER_DATE: "2026-10-20T10:00:00Z" };
const git = (...args) => execFileSync("git", args, { cwd: dir, encoding: "utf8", env });
const base = [
  "export function label(device) {", "  const name = device.name.trim();", "  if (!device.online) return `${name} (offline)`;",
  "  return name;", "}",
  "export function byName(a, b) {", "  return a.name.localeCompare(b.name);", "}",
  "export function firmwareLabel(version) {", "  const [major, minor] = version.split(\".\");", "  return `v${major}.${minor}`;", "}",
];
const added = [ // the pull request: a new status for devices that are installing firmware
  "export function badge(device) {", "  if (device.status === \"updating\") {", "    const pct = Math.round(device.progress * 100);",
  "    return `Updating ${pct}%`;", "  }", "  return device.online ? \"Online\" : \"Offline\";", "}",
];
writeFileSync(join(dir, "devices.mjs"), base.join("\n") + "\n");
writeFileSync(join(dir, "devices.test.mjs"), `import { test } from "node:test"; import assert from "node:assert/strict";
import { label, byName, firmwareLabel, badge } from "./devices.mjs";
test("label", () => { assert.equal(label({ name: " Room 1 ", online: false }), "Room 1 (offline)"); assert.equal(label({ name: "Room 2", online: true }), "Room 2"); });
test("byName", () => assert.equal([{ name: "B" }, { name: "A" }].sort(byName)[0].name, "A"));
test("firmwareLabel", () => assert.equal(firmwareLabel("10.2.1"), "v10.2"));
test("badge", () => assert.equal(badge({ online: true, status: "idle" }), "Online"));`);
git("init", "-q", "-b", "main"); git("add", "."); git("commit", "-qm", "Device labels");
writeFileSync(join(dir, "devices.mjs"), [...base, ...added].join("\n") + "\n");
git("commit", "-qam", "Add device badge");

const newLines = new Set(); // line numbers added by the last commit, from the unified diff's hunk headers
for (const [, start, count = "1"] of git("diff", "-U0", "HEAD~1", "HEAD", "--", "devices.mjs").matchAll(/^@@ -\S+ \+(\d+)(?:,(\d+))? @@/gm))
  for (let i = 0; i < Number(count); i++) newLines.add(Number(start) + i);
let lines = [];
for await (const { type, data } of run({ files: [join(dir, "devices.test.mjs")], coverage: true, coverageIncludeGlobs: ["**/devices.mjs"] }))
  if (type === "test:coverage") lines = data.summary.files[0].lines; // [{ line, count }] for executable lines
const source = [...base, ...added];
const pct = (xs) => `${Math.round((xs.filter((l) => l.count > 0).length / xs.length) * 100)}%`;
const changed = lines.filter((l) => newLines.has(l.line));
console.log(`whole file: ${pct(lines)} of ${lines.length} lines covered`);
console.log(`added by the PR: ${pct(changed)} of ${changed.length} lines covered`);
for (const l of changed.filter((l) => l.count === 0)) console.log(`  not run: line ${l.line}: ${source[l.line - 1].trim()}`);
rmSync(dir, { recursive: true, force: true });

whole file: 84% of 19 lines covered
added by the PR: 57% of 7 lines covered
  not run: line 15: const pct = Math.round(device.progress * 100);
  not run: line 16: return `Updating ${pct}%`;
  not run: line 17: }

// An order-dependent test suite: three tests share one module-level Map. Node 26.1+ can shuffle test order with a
// seed (--test-random-seed, or randomSeed in run()), so the same seed always gives the same order and result.
import { test, beforeEach, run } from "node:test";
import assert from "node:assert/strict";

let devices = new Map(); // shared by every test in this file
if (process.env.NODE_TEST_CONTEXT) {
  if (process.env.FIXED) beforeEach(() => (devices = new Map([["dev-1", { name: "Room 1" }]]))); // each test gets its own state
  test("registers a device", () => {
    devices.set("dev-2", { name: "Room 2" });
    assert.ok(devices.has("dev-2"));
  });
  test("renames a device", () => {
    if (!process.env.FIXED) devices.set("dev-1", { name: "Room 1" }); // setup that only this test does
    devices.get("dev-1").name = "Boardroom";
    assert.equal(devices.get("dev-1").name, "Boardroom");
  });
  test("lists devices", () => assert.equal(devices.size, process.env.FIXED ? 1 : 2));
} else {
  for (const fixed of [false, true]) {
    console.log(fixed ? "With a fresh Map before each test:" : "Shared state:");
    for (const seed of [undefined, 1, 2, 5]) {
      const results = [];
      const env = { ...process.env, ...(fixed ? { FIXED: "1" } : {}) };
      for await (const { type, data } of run({ files: [import.meta.filename], randomSeed: seed, env })) {
        if (data?.nesting === 0 && type === "test:pass") results.push(`✔ ${data.name}`);
        if (data?.nesting === 0 && type === "test:fail") results.push(`✖ ${data.name}`);
      }
      console.log(`  ${seed === undefined ? "declared order" : `seed ${seed}`.padEnd(14)}  ${results.join(" → ")}`);
    }
  }
}

Shared state:
  declared order  ✔ registers a device → ✔ renames a device → ✔ lists devices
  seed 1          ✔ renames a device → ✔ registers a device → ✔ lists devices
  seed 2          ✖ lists devices → ✔ registers a device → ✔ renames a device
  seed 5          ✖ lists devices → ✔ renames a device → ✔ registers a device
With a fresh Map before each test:
  declared order  ✔ registers a device → ✔ renames a device → ✔ lists devices
  seed 1          ✔ renames a device → ✔ registers a device → ✔ lists devices
  seed 2          ✔ lists devices → ✔ registers a device → ✔ renames a device
  seed 5          ✔ lists devices → ✔ renames a device → ✔ registers a device

// A device is "offline" if no heartbeat arrived for 30 s. Testing that with real timers means sleeping 30 s, or
// shrinking the timeout and racing the scheduler (flaky). node:test's mock timers make time an input instead.
import { test, run } from "node:test";
import assert from "node:assert/strict";

export function createPresence({ timeoutMs, onOffline }) {
  const timers = new Map();
  const lastSeen = new Map();
  return {
    heartbeat(id) {
      clearTimeout(timers.get(id));
      lastSeen.set(id, Date.now());
      timers.set(id, setTimeout(() => onOffline(id, Date.now() - lastSeen.get(id)), timeoutMs));
    },
  };
}

if (process.env.NODE_TEST_CONTEXT) {
  test("goes offline exactly 30 s after the last heartbeat", (t) => {
    t.mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 });
    const offline = [];
    const presence = createPresence({ timeoutMs: 30_000, onOffline: (id, silentMs) => offline.push(`${id} after ${silentMs} ms`) });
    presence.heartbeat("dev-1");
    t.mock.timers.tick(29_999);
    assert.deepEqual(offline, []);
    t.mock.timers.tick(1);
    assert.deepEqual(offline, ["dev-1 after 30000 ms"]);
  });
  test("a heartbeat in time resets the countdown", (t) => {
    t.mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 });
    const offline = [];
    const presence = createPresence({ timeoutMs: 30_000, onOffline: (id) => offline.push(id) });
    presence.heartbeat("dev-1");
    t.mock.timers.tick(20_000);
    presence.heartbeat("dev-1"); // countdown restarts at t = 20 s
    t.mock.timers.tick(29_000);
    assert.deepEqual(offline, []);
    t.mock.timers.tick(1_000);
    assert.deepEqual(offline, ["dev-1"]);
  });
} else {
  for await (const { type, data } of run({ files: [import.meta.filename] })) {
    if (type === "test:pass" && data.nesting === 0) console.log(`✔ ${data.name}`);
    if (type === "test:fail" && data.nesting === 0) console.log(`✖ ${data.name}\n  ${data.details.error.cause?.message}`);
  }
}

✔ goes offline exactly 30 s after the last heartbeat
✔ a heartbeat in time resets the countdown

name: Quarantined tests

on:
  schedule:
    - cron: "40 2 * * *"
  workflow_dispatch:

permissions:
  contents: read

jobs:
  quarantined:
    name: Quarantined E2E tests, 20 runs each
    runs-on: ubuntu-24.04
    timeout-minutes: 45
    services:
      postgres:
        image: postgres:18
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: devices_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
    env:
      DATABASE_URL: postgres://postgres:postgres@localhost:5432/devices_test
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-node@v7
        with:
          node-version-file: .nvmrc
          cache: npm
      - run: npm ci
      - run: npm run build --workspace web
      - run: npm run migrate --workspace api
      - run: npx playwright install --with-deps chromium
      - name: Run every quarantined test 20 times
        id: repeat
        continue-on-error: true
        env:
          PLAYWRIGHT_JSON_OUTPUT_NAME: quarantine.json
        run: npx playwright test --grep @quarantine --repeat-each=20 --reporter=json
      - name: Summarise pass rates
        if: always()
        run: node scripts/flake-summary.mjs quarantine.json >> "$GITHUB_STEP_SUMMARY"
      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: quarantine-${{ github.run_id }}
          path: quarantine.json
          retention-days: 30

// A seam (Feathers): a place where you can change what code does without editing it there. The legacy function
// reached for the clock and the network itself; one small edit turns both into parameters with the old defaults.
import { test, run } from "node:test";
import assert from "node:assert/strict";

// Before: untestable without a real device API and waiting for 2 a.m.
//   export async function rebootIfIdle(device) {
//     const hour = new Date().getHours();
//     if (hour < 2 || hour > 4 || device.inCall) return "skipped";
//     await fetch(`https://devices.internal/api/v1/devices/${device.id}/reboot`, { method: "POST" });
//     return "rebooted";
//   }
// After: same behaviour for every existing caller, but tests can pass their own clock and HTTP client.
export async function rebootIfIdle(device, { now = () => new Date(), post = (url) => fetch(url, { method: "POST" }) } = {}) {
  const hour = now().getHours();
  if (hour < 2 || hour > 4 || device.inCall) return "skipped";
  await post(`https://devices.internal/api/v1/devices/${device.id}/reboot`);
  return "rebooted";
}

if (process.env.NODE_TEST_CONTEXT) {
  const at = (h) => () => new Date(2026, 9, 27, h, 30); // local time, so getHours() is h in any time zone
  test("reboots an idle device at 03:30", async () => {
    const calls = [];
    assert.equal(await rebootIfIdle({ id: "dev-1", inCall: false }, { now: at(3), post: async (url) => calls.push(url) }), "rebooted");
    assert.deepEqual(calls, ["https://devices.internal/api/v1/devices/dev-1/reboot"]);
  });
  test("never reboots a device that is in a call", async () => {
    const calls = [];
    assert.equal(await rebootIfIdle({ id: "dev-1", inCall: true }, { now: at(3), post: async (url) => calls.push(url) }), "skipped");
    assert.deepEqual(calls, []);
  });
  test("leaves devices alone during the working day", async () => {
    assert.equal(await rebootIfIdle({ id: "dev-1", inCall: false }, { now: at(10), post: async () => assert.fail("no POST") }), "skipped");
  });
} else {
  for await (const { type, data } of run({ files: [import.meta.filename] })) {
    if (type === "test:pass" && data.nesting === 0) console.log(`✔ ${data.name}`);
    if (type === "test:fail" && data.nesting === 0) console.log(`✖ ${data.name}\n  ${data.details.error.cause?.message}`);
  }
}

✔ reboots an idle device at 03:30
✔ never reboots a device that is in a call
✔ leaves devices alone during the working day

// Where to start testing a legacy codebase: files that change often AND are complicated (a "hotspot").
// Churn comes from git history (commits touching the file since a fixed date); complexity is a crude count of
// branches in the current file. A throwaway repo with fixed names and dates keeps the output deterministic.
import { execFileSync } from "node:child_process";
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, dirname } from "node:path";

const dir = mkdtempSync(join(tmpdir(), "hot-"));
const git = (args, date) => execFileSync("git", args, { cwd: dir, encoding: "utf8", env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1",
  GIT_AUTHOR_NAME: "Dev", GIT_AUTHOR_EMAIL: "dev@example.com", GIT_COMMITTER_NAME: "Dev", GIT_COMMITTER_EMAIL: "dev@example.com",
  ...(date && { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }) } }).trim();
const branchy = (n, tag) => Array.from({ length: n }, (_, i) => `if (x === ${i}) return "${tag}${i}";`).join("\n") + "\n";
const write = (file, text) => (mkdirSync(join(dir, dirname(file)), { recursive: true }), writeFileSync(join(dir, file), text));
git(["init", "-q", "-b", "main"]);

const sizes = { "api/devices.js": 25, "api/provisioning.js": 60, "web/Settings.tsx": 12, "lib/strings.js": 40, "api/health.js": 2 };
const touches = { // how many commits touched each file in the last quarter
  "api/devices.js": 14, "api/provisioning.js": 9, "web/Settings.tsx": 16, "lib/strings.js": 1, "api/health.js": 1,
};
for (const [f, n] of Object.entries(sizes)) write(f, branchy(n, "v0-"));
git(["add", "."]);
git(["commit", "-qm", "Import from the old repo"], "2025-11-03T09:00:00Z");
for (let day = 1; day <= 16; day++) {
  const date = `2026-08-${String(day).padStart(2, "0")}T10:00:00Z`;
  for (const [f, n] of Object.entries(touches)) if (day <= n) write(f, branchy(sizes[f], `v${day}-`));
  git(["add", "."]);
  git(["commit", "-qm", `Change ${day}`], date);
}

const log = git(["log", "--since=2026-07-01T00:00:00Z", "--name-only", "--format="]).split("\n").filter(Boolean);
const churn = Object.fromEntries(Object.keys(sizes).map((f) => [f, log.filter((x) => x === f).length]));
const complexity = (f) => (readFileSync(join(dir, f), "utf8").match(/\b(if|for|while|case)\b|&&|\|\||\?/g) || []).length;
const rows = Object.keys(sizes).map((f) => ({ f, churn: churn[f], cx: complexity(f), score: churn[f] * complexity(f) }))
  .sort((a, b) => b.score - a.score);
console.log("file                  commits  branches  hotspot score");
for (const r of rows) console.log(`${r.f.padEnd(21)} ${String(r.churn).padStart(7)}  ${String(r.cx).padStart(8)}  ${String(r.score).padStart(13)}`);
rmSync(dir, { recursive: true, force: true });

file                  commits  branches  hotspot score
api/provisioning.js         9        60            540
api/devices.js             14        25            350
web/Settings.tsx           16        12            192
lib/strings.js              1        40             40
api/health.js               1         2              2

// A golden-master test for a line-based control API (Telnet-style). The command set is invented for this example.
// A session recorded against the legacy implementation is replayed against the rewrite; any different reply fails.
const transcript = `
> get volume
< volume 50
> set volume 40
< ok
> get volume
< volume 40
> SET VOLUME 70
< ok
> set volume 140
< error 3: value out of range
> set volume loud
< error 2: bad argument
> reboot now
< error 1: unknown command
`;
function createRewrite() { // the new implementation someone wants to ship
  let volume = 50;
  const commands = {
    "get volume": () => `volume ${volume}`,
    "set volume": (arg) => {
      if (!/^\d+$/.test(arg ?? "")) return "error 2: bad argument";
      if (Number(arg) > 100) return "error 3: value out of range";
      volume = Number(arg);
      return "ok";
    },
  };
  return (line) => {
    const [verb, noun, arg] = line.trim().split(/\s+/);
    const handler = commands[`${verb} ${noun}`];
    return handler ? handler(arg) : "error 1: unknown command";
  };
}
const steps = [];
for (const [, cmd, reply] of transcript.matchAll(/^> (.*)\n< (.*)$/gm)) steps.push({ cmd, reply });
const send = createRewrite();
let failures = 0;
for (const { cmd, reply } of steps) {
  const got = send(cmd);
  if (got !== reply) failures++;
  console.log(`${got === reply ? "same" : "DIFF"}  > ${cmd.padEnd(16)} recorded "${reply}"${got === reply ? "" : `, rewrite "${got}"`}`);
}
console.log(`${steps.length} exchanges replayed, ${failures} different`);

same  > get volume       recorded "volume 50"
same  > set volume 40    recorded "ok"
same  > get volume       recorded "volume 40"
DIFF  > SET VOLUME 70    recorded "ok", rewrite "error 1: unknown command"
same  > set volume 140   recorded "error 3: value out of range"
same  > set volume loud  recorded "error 2: bad argument"
same  > reboot now       recorded "error 1: unknown command"
7 exchanges replayed, 1 different