CI runners

{
  "acme/web": {
    "snapshot": "web-main",
    "workdir": "/home/boxd/work/web",
    "on": { "push": ["main"], "pull_request": true },
    "setup": [
      "git diff --quiet \"$BASE\" HEAD -- yarn.lock || yarn install --frozen-lockfile"
    ],
    "checks": [
      "yarn lint --cache",
      "yarn typecheck --incremental"
    ],
    "fanout": { "command": "yarn test --shard={i}/{n}", "n": 8 },
    "promote": "main"
  }
}

// ci.run.ts — CI that runs on boxd, configured outside GitHub.
//
//   run ci.run.ts          (on any of your machines; the platform holds the
//                           GitHub trigger and wakes it on an event)
//
// GitHub only sends the event and shows the result. For every push or pull
// request on a configured repo:
//
//   1. restore a fresh, isolated machine from the repo's snapshot
//      (main's last green state: checkout, node_modules, caches, Docker layers)
//   2. fetch and check out the commit; run the setup steps
//   3. at the same moment, restore one more machine per test shard from the
//      same snapshot; each checks out and runs its slice of the tests while
//      the checks run on the first
//   4. report a GitHub check run on the commit
//   5. green on main: save the machine as the next version (two snapshot
//      names, alternating, so the last one stays restorable while one saves).
//      Anything else is destroyed.
//
// No runners, no pools: every job is a restore from a snapshot, in milliseconds.
import { boxd, githubApp, object } from "@boxd/run";
import config from "./ci.json";

type Repo = {
  snapshot: string;
  workdir: string;
  on: { push?: string[]; pull_request?: boolean };
  setup: string[];
  checks: string[];
  fanout?: { command: string; n: number };
  promote?: string;
};
const repos = config as Record<string, Repo>;

/** Which of the two snapshot names jobs restore from, per repo. Survives restarts. */
const state = object<{ active: Record<string, string> }>("boxd-ci");
state.active ??= {};
// A stored name only counts while it belongs to the configured snapshot, so
// pointing a repo at a new snapshot in ci.json takes effect.
const active = (repo: string) => {
  const a = state.active[repo];
  return a === repos[repo].snapshot || a === `${repos[repo].snapshot}-b` ? a : repos[repo].snapshot;
};
const other = (repo: string) => (active(repo) === repos[repo].snapshot ? `${repos[repo].snapshot}-b` : repos[repo].snapshot);
const promotions = new Map<string, Promise<void>>();

const secs = (t0: number) => ((performance.now() - t0) / 1000).toFixed(1);
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

type Step = { name: string; ok: boolean; s: string; tail?: string };

async function sh(id: string, dir: string, script: string, env?: Record<string, string>, timeout = 30 * 60_000) {
  return boxd.machines.exec(id, { command: ["bash", "-lc", `cd ${dir} && ${script}`], env, timeout });
}

async function step(steps: Step[], name: string, id: string, dir: string, script: string, env?: Record<string, string>, say?: (m: string) => void) {
  const t0 = performance.now();
  const r = await sh(id, dir, script, env);
  const out = `${r.stdout}\n${r.stderr}`.trim().split("\n");
  steps.push({ name, ok: r.success, s: secs(t0), tail: r.success ? undefined : out.slice(-40).join("\n") });
  say?.(`${r.success ? "✓" : "✗"} ${name}  (${secs(t0)}s)`);
  return r.success;
}

/** The short form of a step for the live log: `yarn test:code --cache …` → `test:code`,
    `pnpm run lint` → `lint`; anything else is the command itself, cut to 48 characters. */
const short = (cmd: string) =>
  cmd.match(/(?:yarn|pnpm|npm|bun)(?: run)? ([\w:.-]+)/)?.[1] ?? (cmd.length > 48 ? `${cmd.slice(0, 47)}…` : cmd);

async function ci(repoName: string, sha: string, ref: string, isMain: boolean, label: string, shards?: number) {
  const repo = repos[repoName];
  const [owner, name] = repoName.split("/");
  const gh = await githubApp.client();
  const t0 = performance.now();
  // The live log, `run logs -f`: one line per event, timed from the push.
  const say = (msg: string) => console.log(`${secs(t0).padStart(6)}s  ${msg}`);
  const check = await gh.rest.checks.create({ owner, repo: name, name: "boxd-ci", head_sha: sha, status: "in_progress", started_at: new Date().toISOString() });
  const steps: Step[] = [];
  const machines: string[] = [];
  let ok = false;
  let keep: string | undefined; // the machine to save as the next snapshot
  let restore = "";
  const source = active(repoName); // the snapshot this job restores from
  const n = repo.fanout ? (shards ?? repo.fanout.n) : 0;
  const boots: number[] = [];
  console.log(`\n▶ ${label} · ${sha.slice(0, 7)} · ${n + 1} machines from ${source}`);
  try {
    const token = await githubApp.getToken();
    const jobName = `ci-${sha.slice(0, 7)}-${Date.now() % 100000}`;

    // One machine, restored from main's last green state, with the commit on
    // top. Isolated: no boxd CLI, no integrations, so the code under test can't
    // reach the account. The App token is only in the fetch's environment.
    // `log` decides whether its checkout and setup show up as their own rows.
    const prepare = async (machineName: string, log: boolean) => {
      const tr = performance.now();
      const m = await boxd.machines.create({ name: machineName, fromSnapshot: source, isolated: true });
      machines.push(m.id);
      // A large or fresh snapshot can answer before the machine is up.
      await boxd.machines.waitUntilReady(m.id);
      const restored = `${secs(tr)}s`;
      // create() answers before the boot is recorded: read it back.
      const boot = m.bootTimeMs ?? (await boxd.machines.get(m.id).catch(() => null))?.bootTimeMs ?? null;
      if (boot != null) boots.push(boot);
      else boots.push(-1);
      if (boots.length === n + 1) {
        const known = boots.filter((b) => b >= 0);
        say(`✓ ${n + 1} machines restored from the snapshot${known.length ? `, each booted in ${Math.min(...known)}–${Math.max(...known)}ms` : ""}`);
      }
      const prep: Step[] = log ? steps : [];
      const base = (await sh(m.id, repo.workdir, "git rev-parse HEAD")).stdout.trim(); // the snapshot's commit
      const fetched = await step(prep, `checkout ${sha.slice(0, 7)}`, m.id, repo.workdir,
        `git -c http.extraheader="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GH_APP_TOKEN" | base64 -w0)" fetch -q origin ${sha} && git checkout -q -f ${sha}`,
        { GH_APP_TOKEN: token });
      let ready = fetched;
      for (const s of repo.setup) if (ready) ready = await step(prep, s, m.id, repo.workdir, s, { BASE: base });
      if (!ready && !log) steps.push(...prep.filter((x) => !x.ok).map((x) => ({ ...x, name: `${machineName}: ${x.name}` })));
      return { m, restored, ready };
    };

    // The main machine runs the checks; every test shard gets its own machine,
    // restored from the same snapshot at the same moment. Nothing waits on
    // anything else.
    // A shard that cannot get its machine (an org VM limit, say) fails the
    // run. Left uncaught, its rejection would take the whole automation down.
    const shards = Array.from({ length: n }, (_, i) => (async () => {
      try {
        const { m, ready } = await prepare(`${jobName}-s${i + 1}`, false);
        if (!ready) return false;
        const cmd = repo.fanout!.command.replaceAll("{i}", String(i + 1)).replaceAll("{n}", String(n));
        return await step(steps, `shard ${i + 1}/${n}`, m.id, repo.workdir, cmd, undefined, say);
      } catch (e) {
        steps.push({ name: `shard ${i + 1}/${n}`, ok: false, s: secs(t0), tail: e instanceof Error ? e.message : String(e) });
        return false;
      }
    })());
    const main = await prepare(jobName, true);
    restore = main.restored;
    let checksOk = main.ready;
    if (checksOk) for (const s of repo.checks) if (!(await step(steps, short(s), main.m.id, repo.workdir, s, undefined, say))) { checksOk = false; break; }
    ok = checksOk && (await Promise.all(shards)).every(Boolean);
    const m = main.m;

    // 5. Green on main: this machine becomes the next snapshot, after the
    //    result is reported (see finally). It is kept out of the cleanup.
    if (ok && isMain && repo.promote) keep = m.id;
  } catch (e) {
    steps.push({ name: "boxd-ci", ok: false, s: secs(t0), tail: e instanceof Error ? e.message : String(e) });
  } finally {
    await Promise.all(machines.filter((id) => id !== keep).map((id) => boxd.machines.delete(id).catch(() => {})));
    const total = secs(t0);
    const failed = steps.find((s) => !s.ok);
    const rows = steps.map((s) => `| ${s.ok ? "✓" : "✗"} | \`${s.name}\` | ${s.s}s |`).join("\n");
    await gh.rest.checks.update({
      owner, repo: name, check_run_id: check.data.id,
      status: "completed", conclusion: ok ? "success" : "failure", completed_at: new Date().toISOString(),
      output: {
        title: ok ? `green in ${total}s on boxd` : `failed: ${failed?.name ?? "?"}`,
        summary: `${label}. Restored from \`${source}\` in ${restore || "?"}; ${repo.fanout ? `${n} test shards, each on its own machine restored from the same snapshot; ` : ""}${total}s in total.\n\n| | step | time |\n|---|---|---|\n${rows}`,
        text: failed?.tail ? `\`\`\`\n${failed.tail.slice(-60_000)}\n\`\`\`` : undefined,
      },
    });
    say(`${ok ? "✓ green" : "✗ failed"}: check posted on ${sha.slice(0, 7)}`);
  }
  // Reported first, saved after: nobody waits on the snapshot.
  if (keep) {
    const machine = keep;
    console.log(`        green on main: saving this machine as the next snapshot…`);
    const prev = promotions.get(repoName) ?? Promise.resolve();
    const next = prev
      .then(() => promote(repoName, machine))
      .catch((e) => console.log(`✗ promote: ${e instanceof Error ? e.message : e}`))
      .finally(() => boxd.machines.delete(machine).catch(() => {}));
    promotions.set(repoName, next);
    await next;
  }
}

async function promote(repoName: string, machineId: string) {
  const t0 = performance.now();
  const target = other(repoName);
  const before = await boxd.snapshots.get(target).catch(() => null);
  await boxd.snapshots.create(machineId, target);
  // The copy reads from the live machine: keep it until the snapshot is ready.
  let snap = await boxd.snapshots.get(target);
  while (snap.status !== "ready" || (snap.version ?? 0) <= (before?.version ?? 0)) {
    await sleep(1000);
    snap = await boxd.snapshots.get(target);
  }
  state.active = { ...state.active, [repoName]: target };
  console.log(`        ★ saved as ${target}@${snap.version} in ${secs(t0)}s: the next job starts here`);
}

for (const [repoName, repo] of Object.entries(repos)) {
  if (repo.on.push?.length) {
    githubApp.on("push", { repo: repoName }, async (e: any) => {
      const branch = String(e.ref).replace("refs/heads/", "");
      if (!repo.on.push!.includes(branch) || e.deleted) return;
      // "[shards=16]" in the commit message overrides the shard count for that run.
      const asked = /\[shards=(\d+)\]/.exec(e.head_commit?.message ?? "");
      await ci(repoName, e.after, branch, branch === repo.promote, `push to ${branch}`, asked ? Number(asked[1]) : undefined);
    });
  }
  if (repo.on.pull_request) {
    githubApp.on("pull_request", { repo: repoName }, async (e: any) => {
      if (!["opened", "synchronize", "reopened"].includes(e.action)) return;
      await ci(repoName, e.pull_request.head.sha, e.pull_request.head.ref, false, `pull request #${e.number}`);
    });
  }
}
console.log(`boxd-ci watching ${Object.keys(repos).join(", ")}`);