Drop7 Research
approaches/lifetime-objective/flow-ceiling/paired.pyPython123 lines · 4.3 KB
#!/usr/bin/env python3
"""Paired whole-game comparison of two `flow-run` cohorts on shared master tapes.

Whole games are the statistical unit (`docs/methodology.md`).  Both cohorts must
have been run on the same master-tape seeds and the same `--max-moves`, since a
master tape is a function of both.  Pairing is preserved through the bootstrap:
one replicate resamples *seeds* and recomputes both arms from that same seed
multiset, so common randomness is never broken.

Reported per metric: the candidate mean, the comparator mean, the paired delta,
and a **one-sided 95% bootstrap lower bound** on that delta.  A lower bound at
or below zero means the cohort does not establish an advantage.

Usage:
    paired.py <candidate.jsonl> <comparator.jsonl> [--label-a A] [--label-b B]
"""
import json
import random
import sys


def load(path):
    with open(path) as handle:
        return {g["seed"]: g for g in
                (json.loads(line) for line in handle if line.strip())}


def slope(values, skip=1):
    n = len(values) - skip
    if n < 3:
        return 0.0
    xs = list(range(n))
    ys = values[skip:]
    sx, sy = sum(xs), sum(ys)
    sxy = sum(x * y for x, y in zip(xs, ys))
    sxx = sum(x * x for x in xs)
    det = n * sxx - sx * sx
    return (n * sxy - sx * sy) / det if det else 0.0


def stats(games, seeds):
    moves = sum(games[s]["moves"] for s in seeds)
    cleared = sum(games[s]["cleared"] for s in seeds)
    revealed = sum(games[s]["revealed"] for s in seeds)
    n = len(seeds)
    return {
        "meanMoves": moves / n,
        "meanScore": sum(games[s]["score"] for s in seeds) / n,
        "clearsPerMove": cleared / moves if moves else 0.0,
        "revealsPerMove": revealed / moves if moves else 0.0,
        "occSlope": sum(slope(games[s]["cycleOccupancy"]) for s in seeds) / n,
        "censored": sum(1 for s in seeds if games[s]["censored"]) / n,
    }


METRICS = [
    ("mean moves", "meanMoves"),
    ("mean score", "meanScore"),
    ("clears/move (pooled)", "clearsPerMove"),
    ("reveals/move (pooled)", "revealsPerMove"),
    ("occupancy slope", "occSlope"),
    ("censored fraction", "censored"),
]


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("--")]
    labels = {}
    argv = sys.argv[1:]
    for flag, key in (("--label-a", "a"), ("--label-b", "b")):
        if flag in argv:
            labels[key] = argv[argv.index(flag) + 1]
    a_path, b_path = args[0], args[1]
    A, B = load(a_path), load(b_path)
    seeds = sorted(set(A) & set(B))
    if not seeds:
        print("no shared master-tape seeds; these cohorts are not paired")
        return 2
    la = labels.get("a", a_path.rsplit("/", 1)[-1])
    lb = labels.get("b", b_path.rsplit("/", 1)[-1])
    print(f"paired on {len(seeds)} shared master tapes "
          f"({len(A)} vs {len(B)} games available)")
    sa, sb = stats(A, seeds), stats(B, seeds)

    random.seed(20260820)
    replicates = 4000
    draws = {key: [] for _n, key in METRICS}
    for _ in range(replicates):
        order = [random.choice(seeds) for _ in seeds]
        ra, rb = stats(A, order), stats(B, order)
        for _n, key in METRICS:
            draws[key].append(ra[key] - rb[key])

    print(f"\n{'metric':<24}{la[:14]:>15}{lb[:14]:>15}{'delta':>13}"
          f"{'95% lower':>13}   verdict")
    for name, key in METRICS:
        d = sorted(draws[key])
        lo = d[int(0.05 * replicates)]
        delta = sa[key] - sb[key]
        # For occupancy slope a *lower* value is better: it is the rate at
        # which the board fills, so a reliably negative delta is an advantage.
        lower_is_better = key == "occSlope"
        hi = d[int(0.95 * replicates) - 1]
        if (lo > 0) != lower_is_better and (lo > 0 or hi < 0):
            verdict = "advantage established"
        elif (lo > 0 or hi < 0):
            verdict = "DISADVANTAGE established"
        else:
            verdict = "straddles zero"
        if lower_is_better:
            verdict += " (lower is better)"
        print(f"{name:<24}{sa[key]:>15.4f}{sb[key]:>15.4f}{delta:>+13.4f}"
              f"{lo:>+13.4f}   {verdict}")

    wins = sum(1 for s in seeds if A[s]["moves"] > B[s]["moves"])
    ties = sum(1 for s in seeds if A[s]["moves"] == B[s]["moves"])
    print(f"\nper-tape lifetime record: {wins} win / {ties} tie / "
          f"{len(seeds)-wins-ties} loss")
    return 0


if __name__ == "__main__":
    sys.exit(main())