Drop7 Research
approaches/lifetime-objective/learned-leaf/analyze.pyPython156 lines · 5.7 KB
"""Tables for the learned-leaf 2x2.

Whole games are the independent unit throughout (docs/benchmarks.md).  Paired
deltas and the difference-in-differences resample SEEDS, not arms, so every
resample keeps a game's four arm outcomes together.
"""

from __future__ import annotations

import argparse
import json
import os
import sys

import numpy as np

BOOTSTRAP_SEED = 0xA52A_1EAF
RESAMPLES = 20000


def load(path: str) -> dict:
    with open(path) as handle:
        blob = json.load(handle)
    games = {g["seedHex"]: g for g in blob["gamesDetail"]}
    blob["bySeed"] = games
    blob["order"] = [g["seedHex"] for g in blob["gamesDetail"]]
    return blob


def summary(blob: dict) -> dict:
    games = blob["gamesDetail"]
    moves = sum(g["moves"] for g in games)
    return {
        "games": len(games),
        "scoreMean": blob["score"]["mean"],
        "scoreMedian": blob["score"]["median"],
        "scoreQ25": blob["score"]["q25"],
        "scoreMin": blob["score"]["min"],
        "scoreMax": blob["score"]["max"],
        "scoreSd": blob["score"]["sd"],
        "movesMean": blob["moves"]["mean"],
        "movesMedian": blob["moves"]["median"],
        "clearsPerMove": blob["numberedClearsPerMove"],
        "revealsPerMove": blob["coverRevealsPerMove"],
        "meanOccupiedCells": blob["meanOccupiedCells"],
        "pointsPerMove": blob["pointsPerMove"],
        "workPerMove": blob["workPerMove"],
        "censored": blob["censoredGames"],
        "identityFailures": blob["scoreIdentityFailures"],
        "wallSeconds": blob["wallSeconds"],
        "threads": blob["threads"],
        "config": blob["config"],
        "totalMoves": moves,
    }


def aligned(a: dict, b: dict, field: str):
    seeds = a["order"]
    if seeds != b["order"]:
        raise SystemExit("cohorts are not the same ordered seeds")
    return (np.array([a["bySeed"][s][field] for s in seeds], dtype=float),
            np.array([b["bySeed"][s][field] for s in seeds], dtype=float))


def lower_bound(values: np.ndarray, alpha: float = 0.05) -> float:
    rng = np.random.default_rng(BOOTSTRAP_SEED)
    n = len(values)
    picks = rng.integers(0, n, size=(RESAMPLES, n))
    return float(np.quantile(values[picks].mean(axis=1), alpha))


def paired(a: dict, b: dict, field: str = "score") -> dict:
    """b minus a."""
    x, y = aligned(a, b, field)
    d = y - x
    wins = int((d > 0).sum())
    losses = int((d < 0).sum())
    return {"delta": float(d.mean()),
            "lower95": lower_bound(d),
            "median": float(np.median(d)),
            "wins": wins, "ties": len(d) - wins - losses, "losses": losses}


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--label", action="append", default=[],
                        help="name=path, repeatable")
    parser.add_argument("--pair", action="append", default=[],
                        help="baseline:candidate, repeatable")
    parser.add_argument("--did", default="",
                        help="refA:candA:refB:candB -- (candB-refB) - (candA-refA)")
    parser.add_argument("--json", default="")
    args = parser.parse_args()

    arms = {}
    for entry in args.label:
        name, path = entry.split("=", 1)
        if not os.path.exists(path):
            print(f"missing {path}", file=sys.stderr)
            continue
        arms[name] = load(path)

    report = {"arms": {name: summary(blob) for name, blob in arms.items()},
              "pairs": {}, "bootstrapResamples": RESAMPLES,
              "bootstrapSeed": hex(BOOTSTRAP_SEED)}

    print(f"{'arm':<22}{'score mean':>12}{'median':>10}{'moves':>9}"
          f"{'clr/mv':>9}{'rev/mv':>9}{'cells':>8}{'work/mv':>12}{'cens':>6}{'idfail':>7}")
    for name, blob in arms.items():
        s = report["arms"][name]
        print(f"{name:<22}{s['scoreMean']:>12,.0f}{s['scoreMedian']:>10,.0f}"
              f"{s['movesMean']:>9.2f}{s['clearsPerMove']:>9.4f}"
              f"{s['revealsPerMove']:>9.4f}{s['meanOccupiedCells']:>8.2f}"
              f"{s['workPerMove']:>12,.0f}{s['censored']:>6}{s['identityFailures']:>7}")

    for entry in args.pair:
        base, cand = entry.split(":", 1)
        if base not in arms or cand not in arms:
            continue
        block = {field: paired(arms[base], arms[cand], field)
                 for field in ("score", "moves", "clearsPerMove", "revealsPerMove")}
        report["pairs"][f"{cand}-{base}"] = block
        s = block["score"]
        m = block["moves"]
        print(f"\n{cand} - {base}: score {s['delta']:+,.0f} "
              f"(95% lower {s['lower95']:+,.0f})  moves {m['delta']:+.2f}  "
              f"W-T-L {s['wins']}-{s['ties']}-{s['losses']}")

    if args.did:
        ra, ca, rb, cb = args.did.split(":")
        if all(n in arms for n in (ra, ca, rb, cb)):
            xa, ya = aligned(arms[ra], arms[ca], "score")
            xb, yb = aligned(arms[rb], arms[cb], "score")
            did = (yb - xb) - (ya - xa)
            report["differenceInDifferences"] = {
                "definition": f"({cb}-{rb}) - ({ca}-{ra})",
                "delta": float(did.mean()),
                "lower95": lower_bound(did),
                "deltaA": float((ya - xa).mean()),
                "deltaB": float((yb - xb).mean()),
                "positiveGames": int((did > 0).sum()),
                "games": int(len(did)),
            }
            d = report["differenceInDifferences"]
            print(f"\nDiD {d['definition']}: {d['delta']:+,.0f} "
                  f"(95% lower {d['lower95']:+,.0f})   "
                  f"dA {d['deltaA']:+,.0f}  dB {d['deltaB']:+,.0f}")

    if args.json:
        os.makedirs(os.path.dirname(args.json) or ".", exist_ok=True)
        with open(args.json, "w") as handle:
            json.dump(report, handle, indent=2)


if __name__ == "__main__":
    main()