Drop7 Research
approaches/baselines-diagnostics/heuristic-benchmark/main.tsTypeScript313 lines · 9.4 KB
import { pathToFileURL } from "node:url";
import {
  HEURISTIC_PROFILES,
  type HeuristicProfileName,
} from "../../../src/core/typescript/heuristic.ts";
import {
  runHeadlessTournament,
  type HeadlessProfileSummary,
} from "../../../src/core/typescript/headless.ts";

export interface BenchmarkArguments {
  profiles: HeuristicProfileName[];
  seeds: number[];
  depth: number;
  timeLimitMs?: number;
  maxWork?: number;
  maxMoves: number;
  format: "table" | "json";
  quiet: boolean;
}

const MAX_SEED_COUNT = 100_000;

export function runCli(arguments_: readonly string[]) {
  const parsed = parseArguments(arguments_);
  if (parsed === null) {
    process.stdout.write(helpText());
    return;
  }

  const tournament = runHeadlessTournament({
    profiles: parsed.profiles,
    seeds: parsed.seeds,
    search: {
      maxDepth: parsed.depth,
      ...(parsed.timeLimitMs === undefined
        ? {}
        : { timeLimitMs: parsed.timeLimitMs }),
      ...(parsed.maxWork === undefined ? {} : { maxWork: parsed.maxWork }),
    },
    maxMoves: parsed.maxMoves,
    onGameComplete: (game, completed, total) => {
      if (parsed.quiet || parsed.format === "json") return;
      process.stderr.write(
        `\r${completed}/${total} · ${game.heuristicProfile} · seed ${game.seed} · ${formatInteger(game.score)} points`,
      );
      if (completed === total) process.stderr.write("\n");
    },
  });

  if (parsed.format === "json") {
    process.stdout.write(
      `${JSON.stringify(
        tournament,
        (key, value) => (key === "elapsedMs" ? undefined : value),
        2,
      )}\n`,
    );
    return;
  }

  process.stdout.write(`${formatSummaryTable(tournament.summaries)}\n`);
  process.stdout.write(
    `\nreference: ${tournament.referenceProfile} · ${parsed.seeds.length} paired seeds · depth ${parsed.depth} · ${
      parsed.maxWork === undefined
        ? `${formatInteger(parsed.timeLimitMs! / 1_000)}s per move`
        : `${formatInteger(parsed.maxWork)} work per move${
            parsed.timeLimitMs === undefined
              ? ""
              : ` + ${formatInteger(parsed.timeLimitMs / 1_000)}s guard`
          }`
    } · cap ${parsed.maxMoves} moves\n`,
  );
}

if (
  process.argv[1] &&
  import.meta.url === pathToFileURL(process.argv[1]).href
) {
  runCli(process.argv.slice(2));
}

export function parseArguments(
  arguments_: readonly string[],
): BenchmarkArguments | null {
  let profiles: HeuristicProfileName[] = [];
  let seedStart = 1;
  let gameCount = 8;
  let explicitSeeds: number[] | undefined;
  let depth = 1;
  let timeLimitMs: number | undefined;
  let maxWork: number | undefined;
  let maxMoves = 300;
  let format: BenchmarkArguments["format"] = "table";
  let quiet = false;

  for (let index = 0; index < arguments_.length; index += 1) {
    const flag = arguments_[index];
    if (flag === "--help" || flag === "-h") return null;
    if (flag === "--quiet") {
      quiet = true;
      continue;
    }
    const value = arguments_[index + 1];
    if (value === undefined) throw new Error(`Missing value after ${flag}`);
    index += 1;

    switch (flag) {
      case "--profile":
      case "--profiles":
        profiles.push(...parseProfiles(value));
        break;
      case "--games":
        gameCount = parsePositiveInteger(value, flag);
        if (gameCount > MAX_SEED_COUNT) {
          throw new Error(`--games cannot exceed ${MAX_SEED_COUNT}`);
        }
        break;
      case "--seed":
        seedStart = parseInteger(value, flag);
        break;
      case "--seeds":
        explicitSeeds = parseSeeds(value);
        break;
      case "--depth":
        depth = parsePositiveInteger(value, flag);
        if (depth > 8) throw new Error("--depth must be between 1 and 8");
        break;
      case "--time-limit-s":
        timeLimitMs = parsePositiveInteger(value, flag) * 1_000;
        break;
      case "--max-work":
        maxWork = parsePositiveInteger(value, flag);
        break;
      case "--max-moves":
        maxMoves = parsePositiveInteger(value, flag);
        break;
      case "--format":
        if (value !== "table" && value !== "json") {
          throw new Error("--format must be table or json");
        }
        format = value;
        break;
      default:
        throw new Error(`Unknown option ${flag}`);
    }
  }

  if (profiles.length === 0) profiles = ["legacy", "combined"];
  profiles = [...new Set(profiles)];
  if (maxWork === undefined && timeLimitMs === undefined) maxWork = 200_000;
  const seeds = [
    ...new Set(
      explicitSeeds ??
        Array.from({ length: gameCount }, (_, offset) => seedStart + offset),
    ),
  ];
  for (const seed of seeds) validateSeed(seed);

  return {
    profiles,
    seeds,
    depth,
    timeLimitMs,
    maxWork,
    maxMoves,
    format,
    quiet,
  };
}

function parseProfiles(value: string) {
  return value.split(",").map((profile) => {
    if (!Object.hasOwn(HEURISTIC_PROFILES, profile)) {
      throw new Error(
        `Unknown profile ${profile}. Choose ${Object.keys(HEURISTIC_PROFILES).join(", ")}`,
      );
    }
    return profile as HeuristicProfileName;
  });
}

function parseSeeds(value: string) {
  const range = /^(?<start>-?\d+)\.\.(?<end>-?\d+)$/.exec(value);
  if (!range?.groups) {
    return value.split(",").map((seed) => parseInteger(seed, "--seeds"));
  }
  const start = parseInteger(range.groups.start, "--seeds");
  const end = parseInteger(range.groups.end, "--seeds");
  const direction = start <= end ? 1 : -1;
  const length = Math.abs(end - start) + 1;
  if (length > MAX_SEED_COUNT) {
    throw new Error(`--seeds cannot contain more than ${MAX_SEED_COUNT} games`);
  }
  return Array.from(
    { length },
    (_, offset) => start + offset * direction,
  );
}

function parsePositiveInteger(value: string, flag: string) {
  const parsed = parseInteger(value, flag);
  if (parsed < 1) throw new Error(`${flag} must be at least 1`);
  return parsed;
}

function parseInteger(value: string, flag: string) {
  const parsed = Number(value);
  if (!Number.isSafeInteger(parsed)) {
    throw new Error(`${flag} must be an integer`);
  }
  return parsed;
}

function validateSeed(seed: number) {
  if (seed < 0 || seed > 0xffff_ffff) {
    throw new Error("Seeds must be uint32 integers");
  }
}

function formatSummaryTable(summaries: readonly HeadlessProfileSummary[]) {
  const rows = summaries.map((summary) => ({
    profile: summary.heuristicProfile,
    final: `${summary.completedGames}/${summary.games}`,
    mean: formatOptionalInteger(summary.meanScore),
    median: formatOptionalInteger(summary.medianScore),
    p10: formatOptionalInteger(summary.p10Score),
    p90: formatOptionalInteger(summary.p90Score),
    moves: summary.meanMoves.toFixed(1),
    depth: summary.meanCompletedDepth.toFixed(2),
    depth0: formatPercent(
      summary.depthZeroSearches,
      summary.meanMoves * summary.games,
    ),
    incomplete: formatPercent(
      summary.incompleteSearches,
      summary.meanMoves * summary.games,
    ),
    work: formatInteger(summary.meanSearchWorkPerMove),
    paired: summary.pairedGames.toString(),
    delta: signedInteger(summary.pairedMeanDelta),
    "95% delta": formatInterval(summary.pairedDelta95),
    "W/T/L": `${summary.wins}/${summary.ties}/${summary.losses}`,
  }));
  const headers = Object.keys(rows[0] ?? { profile: "profile" });
  const widths = headers.map((header) =>
    Math.max(
      header.length,
      ...rows.map((row) => String(row[header as keyof typeof row]).length),
    ),
  );
  const render = (values: readonly string[]) =>
    values.map((value, index) => value.padEnd(widths[index])).join("  ");
  return [
    render(headers),
    render(widths.map((width) => "-".repeat(width))),
    ...rows.map((row) =>
      render(headers.map((header) => String(row[header as keyof typeof row]))),
    ),
  ].join("\n");
}

function signedInteger(value: number | null) {
  if (value === null) return "—";
  const rounded = Math.round(value);
  return `${rounded > 0 ? "+" : ""}${formatInteger(rounded)}`;
}

function formatOptionalInteger(value: number | null) {
  return value === null ? "—" : formatInteger(value);
}

function formatInterval(value: readonly [number, number] | null) {
  return value === null
    ? "—"
    : `${signedInteger(value[0])}…${signedInteger(value[1])}`;
}

function formatPercent(numerator: number, denominator: number) {
  return denominator === 0
    ? "—"
    : `${((numerator / denominator) * 100).toFixed(1)}%`;
}

function formatInteger(value: number) {
  return Math.round(value).toLocaleString("en-US");
}

function helpText() {
  return `Drop7 headless heuristic tournament

Usage:
  npm run benchmark -- [options]

Options:
  --profile NAME[,NAME]  Profiles to compare; repeatable
  --games N              Consecutive seeds to run (default: 8)
  --seed N               First seed (default: 1)
  --seeds A..B           Explicit inclusive seed range
  --depth N              Maximum search depth, 1-8 (default: 1)
  --max-work N           Deterministic work budget per move (default: 200000)
  --time-limit-s N       Optional integer wall-clock limit per move
  --max-moves N          Mark a surviving game censored at this cap (default: 300)
  --format table|json    Summary table or full machine-readable results
  --quiet                Hide per-game progress
  --help                 Show this help

Profiles: ${Object.keys(HEURISTIC_PROFILES).join(", ")}

Use --max-work without a time limit when comparing move quality. A time-limited
run also measures heuristic cost, JIT warm-up, and current machine load.
`;
}