Drop7 Research
src/bench/cli.tsTypeScript174 lines · 5.9 KB
/**
 * Runs the scripted-round benchmark and writes leaderboard data for the web
 * app. Usage:
 *
 *   npm run bench                                  # default policies x all rounds
 *   npm run bench -- --policies greedy,expectimax-d3 --rounds gauntlet-01
 *   npm run bench -- --all                         # include slow policies (D4)
 *
 * Output: web/data/leaderboard.json plus one replay file per game under
 * web/data/replays/. Scripted rounds are a public playground; results are not
 * research tier evidence (docs/benchmarks.md).
 */
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
  BENCH_POLICIES,
  DEFAULT_POLICY_IDS,
  getPolicy,
} from "./policies.ts";
import { playScriptedGame, type BenchGameResult } from "./runner.ts";
import { validateScriptedRound, type ScriptedRound } from "./rounds.ts";

const HERE = dirname(fileURLToPath(import.meta.url));
const ROUNDS_DIR = join(HERE, "rounds");
const DEFAULT_OUT = join(HERE, "..", "..", "web", "data");

interface CliOptions {
  policyIds: string[];
  roundIds: string[] | null;
  outDir: string;
}

function parseArgs(argv: readonly string[]): CliOptions {
  const options: CliOptions = {
    policyIds: [...DEFAULT_POLICY_IDS],
    roundIds: null,
    outDir: DEFAULT_OUT,
  };
  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    if (arg === "--policies") {
      options.policyIds = argv[++index].split(",").map((id) => id.trim());
    } else if (arg === "--rounds") {
      options.roundIds = argv[++index].split(",").map((id) => id.trim());
    } else if (arg === "--all") {
      options.policyIds = BENCH_POLICIES.map((policy) => policy.id);
    } else if (arg === "--out") {
      options.outDir = argv[++index];
    } else {
      throw new Error(`Unknown argument: ${arg}`);
    }
  }
  return options;
}

function loadRounds(): ScriptedRound[] {
  return readdirSync(ROUNDS_DIR)
    .filter((file) => file.endsWith(".json"))
    .sort()
    .map((file) =>
      validateScriptedRound(
        JSON.parse(readFileSync(join(ROUNDS_DIR, file), "utf8")),
      ),
    );
}

function summarize(games: readonly BenchGameResult[]) {
  const scores = games.map((game) => game.score).sort((a, b) => a - b);
  const moves = games.map((game) => game.moves);
  const quantile = (sorted: readonly number[], fraction: number) => {
    if (sorted.length === 0) return null;
    const position = Math.max(
      0,
      Math.min(sorted.length - 1, (sorted.length - 1) * fraction),
    );
    const lower = Math.floor(position);
    const upper = Math.ceil(position);
    return sorted[lower] * (upper - position) + sorted[upper] * (position - lower);
  };
  const mean = (values: readonly number[]) =>
    values.length === 0
      ? null
      : values.reduce((sum, value) => sum + value, 0) / values.length;
  return {
    games: games.length,
    meanScore: mean(scores),
    medianScore: quantile(scores, 0.5),
    minimumScore: scores[0] ?? null,
    maximumScore: scores.at(-1) ?? null,
    meanMoves: mean(moves),
    censoredGames: games.filter((game) => game.censored).length,
    illegalMoves: games.reduce((sum, game) => sum + game.illegalMoves, 0),
    meanClearsPerMove:
      games.reduce((sum, game) => sum + game.discsCleared, 0) /
      Math.max(1, games.reduce((sum, game) => sum + game.moves, 0)),
    meanRevealsPerMove:
      games.reduce((sum, game) => sum + game.coveredRevealed, 0) /
      Math.max(1, games.reduce((sum, game) => sum + game.moves, 0)),
    maxChain: Math.max(0, ...games.map((game) => game.maxChain)),
    elapsedMs: games.reduce((sum, game) => sum + game.elapsedMs, 0),
  };
}

function main() {
  const options = parseArgs(process.argv.slice(2));
  const allRounds = loadRounds();
  const rounds = options.roundIds
    ? options.roundIds.map((id) => {
        const round = allRounds.find((candidate) => candidate.id === id);
        if (!round) {
          throw new Error(
            `Unknown round "${id}". Available: ${allRounds.map((r) => r.id).join(", ")}`,
          );
        }
        return round;
      })
    : allRounds;
  const policies = options.policyIds.map(getPolicy);

  const replayDir = join(options.outDir, "replays");
  mkdirSync(replayDir, { recursive: true });

  const games: BenchGameResult[] = [];
  for (const policy of policies) {
    for (const round of rounds) {
      const startedAt = Date.now();
      const game = playScriptedGame(policy, round, { captureAnimation: true });
      games.push(game);
      writeFileSync(
        join(replayDir, `${policy.id}--${round.id}.json`),
        `${JSON.stringify({ ...game, frames: game.frames })}\n`,
      );
      console.log(
        `${policy.id.padEnd(16)} ${round.id}  score=${String(
          game.score,
        ).padStart(8)}  moves=${String(game.moves).padStart(4)}  ${
          game.censored ? "censored" : "game over"
        }  (${Math.round(game.elapsedMs)}ms / ${Date.now() - startedAt}ms)`,
      );
    }
  }

  const leaderboard = {
    format: "drop7-leaderboard-v1",
    generatedAt: new Date().toISOString(),
    moveCap: rounds[0]?.maximumMoves ?? null,
    rounds: rounds.map((round) => ({
      id: round.id,
      name: round.name,
      generatorSeedHex: round.generatorSeedHex,
    })),
    policies: policies.map((policy) => ({
      id: policy.id,
      name: policy.name,
      family: policy.family,
      description: policy.description,
      publicInformation: policy.publicInformation,
      researchPath: policy.researchPath,
    })),
    games: games.map(({ frames: _frames, ...game }) => game),
    summaries: policies.map((policy) => ({
      policyId: policy.id,
      ...summarize(games.filter((game) => game.policyId === policy.id)),
    })),
  };
  writeFileSync(
    join(options.outDir, "leaderboard.json"),
    `${JSON.stringify(leaderboard, null, 2)}\n`,
  );
  console.log(`\nwrote ${join(options.outDir, "leaderboard.json")}`);
}

main();