#!/usr/bin/env python3 """Summarise the adaptive-step-bounds benchmark raw CSV logs. Usage: summarize.py [OUT_DIR] Reads ``OUT_DIR/raw/*.csv`` (written by ``run_limited.sh``) and writes ``OUT_DIR/logs/summary.txt``. It has no dependency on any untracked script or on older local data. Defaults: OUT_DIR = /local/adaptive_step_bounds_2026-10-05, derived from this file's location (benchmarks/adaptive_step_bounds_2026-10-05/). """ import csv import math import os import sys HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(os.path.dirname(HERE)) DEFAULT_OUT = os.path.join(ROOT, "local", "adaptive_step_bounds_2026-10-05") OUTCOME = {0: "ESC", 1: "DARK", 2: "UNRES", 3: "INC"} REASON = { 0: "NONE", 1: "REDSHIFT", 2: "BUDGET", 3: "TIMERANGE", 4: "DOMAIN", 5: "INVMETRIC", 6: "INTEGERR", 7: "UNSUPPORTED", 8: "PROTOCOL", 9: "IO", } def fnum(v): if v is None or v == "" or v == "nan": return math.nan return float(v) def fmt(v): if isinstance(v, float): if math.isnan(v): return "nan" if v == 0: return "0" return f"{v:.4g}" return str(v) def table(title, header, recs): lines = [f"## {title}", ""] lines.append("| " + " | ".join(header) + " |") lines.append("|" + "|".join(["---"] * len(header)) + "|") for r in recs: lines.append("| " + " | ".join(fmt(x) for x in r) + " |") lines.append("") return lines def ang_between(a, b): va = [fnum(a["nx"]), fnum(a["ny"]), fnum(a["nz"])] vb = [fnum(b["nx"]), fnum(b["ny"]), fnum(b["nz"])] na = math.sqrt(sum(x * x for x in va)) nb = math.sqrt(sum(x * x for x in vb)) if not (na > 0 and nb > 0): return math.nan cross = (va[1]*vb[2]-va[2]*vb[1], va[2]*vb[0]-va[0]*vb[2], va[0]*vb[1]-va[1]*vb[0]) dot = sum(x * y for x, y in zip(va, vb)) return math.atan2(math.sqrt(sum(x*x for x in cross)), dot) class Raw: def __init__(self, out_dir): self.raw = os.path.join(out_dir, "raw") self.logs = os.path.join(out_dir, "logs") def rows(self, name): with open(os.path.join(self.raw, name)) as f: return list(csv.DictReader(f)) def summarise_agg(raw, name, title, keycol): recs = raw.rows(name) out = [] for r in recs: out.append((r["case"], float(r[keycol]), int(r["class_mismatch"]), fnum(r["max_dn_ang"]), fnum(r["max_dgrel"]), fnum(r["max_dstopT"]), int(r["sum_rhs"]), int(r["max_rejected"]), int(r["escaped"]), int(r["dark"]), int(r["unresolved"]), int(r["incomplete"]))) out.sort(key=lambda x: (x[0], x[1])) return table(title, ["case", keycol, "mismatch", "max_dn_ang", "max_dgrel", "max_dstopT", "sum_rhs", "max_rej", "ESC", "DARK", "UNRES", "INC"], out) def build(raw): lines = ["# Adaptive-step-bounds benchmark summary tables", ""] ref = raw.rows("a_sch_reference.csv") recs = [] for r in ref: recs.append((r["case"], int(r["dir"]), fnum(r["theta"]), OUTCOME[int(r["outcome"])], REASON[int(r["reason"])], fnum(r["stop_t"]), int(r["steps"]), int(r["rejected"]), int(r["rhs"]), fnum(r["g"]), fnum(r["thr"]))) lines += table("A Schwarzschild references (DP tol=1e-12, max_step=0.25)", ["case", "dir", "theta", "outcome", "reason", "stop_t", "steps", "rej", "rhs", "g", "thr"], recs) lines += summarise_agg(raw, "a_sch_upper_summary.csv", "A Schwarzschild upper scan (min_step=1e-12, " "tol=1e-9)", "upper") lines += summarise_agg(raw, "a_sch_min_summary.csv", "A Schwarzschild min scan (max_step=2, tol=1e-9)", "min_step") rk = raw.rows("a_sch_rk4_sensitive.csv") groups = {} for r in rk: groups.setdefault((r["case"], r["dir"]), {})[r["phase"]] = r recs = [] for (case, d), gg in sorted(groups.items()): a = gg.get("rk4_0.01") b = gg.get("rk4_0.005") if not a or not b: continue dn = math.nan if a["outcome"] != b["outcome"] else ang_between(a, b) dg = (math.nan if a["outcome"] != b["outcome"] else abs(fnum(a["g"]) - fnum(b["g"]))) recs.append((case, d, fnum(a["theta"]), OUTCOME[int(a["outcome"])], OUTCOME[int(b["outcome"])], dn, dg, int(a["rhs"]), int(b["rhs"]))) lines += table("A Schwarzschild sensitive RK4 .01 vs .005", ["case", "dir", "theta", "out.01", "out.005", "hhalve_dn_ang", "|dg|", "rhs.01", "rhs.005"], recs) mk = raw.rows("a_mink_analytic.csv") mkx = raw.rows("a_mink_analytic_x.csv") groups = {} for r in mk: groups.setdefault((r["upper"], r["min_step"]), []).append(r) recs = [] for (u, f), g in sorted(groups.items(), key=lambda kv: (float(kv[0][0]), float(kv[0][1]))): recs.append((u, f, len(g), max(fnum(r["dn_ang"]) for r in g), max(fnum(r["dgrel"]) for r in g), max(fnum(r["t_err"]) for r in g))) lines += table("A Minkowski flat analytic check (n_inf and t)", ["upper", "min_step", "n", "max_dn_ang", "max_dgrel", "max_t_err"], recs) groups = {} for r in mkx: groups.setdefault((r["upper"], r["min_step"]), []).append(r) recs = [] for (u, f), g in sorted(groups.items(), key=lambda kv: (float(kv[0][0]), float(kv[0][1]))): maxx = max(max(fnum(r["x_err_x"]), fnum(r["x_err_y"]), fnum(r["x_err_z"])) for r in g) recs.append((u, f, len(g), maxx, max(fnum(r["t_err"]) for r in g))) lines += table("A Minkowski flat analytic check (crossing x and t)", ["upper", "min_step", "n", "max_x_err", "max_t_err"], recs) for case in ["alc_v3_s1", "alc_v3_s10", "alc_v9_s1", "alc_v9_s10"]: lines += summarise_agg(raw, f"a_alc_{case}_upper_summary.csv", f"A Alcubierre {case} upper scan", "upper") lines += summarise_agg(raw, "a_alc_v9_s100_min_summary.csv", "A Alcubierre v9 s100 lower-bound stress " "(upper=0.004)", "min_step") b = raw.rows("b_summary.csv") recs = [] for r in b: recs.append((r["case"], int(r["dir"]), OUTCOME[int(r["ref_outcome"])], fnum(r["span"]), fnum(r["target"]), int(r["observed_steps"]), int(r["reached_target"]), REASON.get(int(r["terminated_reason"]), r["terminated_reason"]), fnum(r["h_min"]), fnum(r["h_max"]), fnum(r["h_first"]), int(r["boundary_steps"]), int(r["sum_rhs"]), int(r["sum_reject"]), fnum(r["null_residual_max"]))) lines += table("B observed accepted step h (finite T=min(20, ref span))", ["case", "dir", "ref", "span", "target", "steps", "reached", "term", "h_min", "h_max", "h_first", "bnd", "rhs", "rej", "null_max"], recs) if os.path.exists(os.path.join(raw.raw, "critical_ref.csv")): lines += critical_section(raw) return lines def critical_section(raw): rows = raw.rows("critical_ref.csv") key = {} for r in rows: key[(r["camera"], int(r["dir"]), r["cfg"])] = r lines = table("critical reference check (raw rows)", ["camera", "dir", "theta", "cfg", "outcome", "reason", "end_id", "stop_t", "steps", "rejected", "rhs", "g", "thr", "L", "L0"], [(r["camera"], int(r["dir"]), fnum(r["theta"]), r["cfg"], OUTCOME[int(r["outcome"])], REASON[int(r["reason"])], r["end_id"], fnum(r["stop_t"]), int(r["steps"]), int(r["rejected"]), int(r["rhs"]), fnum(r["g"]), fnum(r["thr"]), fnum(r.get("L")), fnum(r.get("L0"))) for r in rows]) # pair comparisons pairs = [("ref_tol1e-12_h0.25", "ref_tol1e-13_h0.125"), ("tol1e-11_h2", "tol1e-11_h8"), ("ref_tol1e-12_h0.25", "tol1e-11_h2"), ("ref_tol1e-12_h0.25", "tol1e-11_h8")] for a_cfg, b_cfg in pairs: recs = [] for (cam, di, cfg), a in sorted(key.items()): if cfg != a_cfg: continue b = key.get((cam, di, b_cfg)) if not b: continue same = a["outcome"] == b["outcome"] dn = ang_between(a, b) if same else math.nan dg = (math.nan if not same else abs(fnum(a["g"]) / fnum(b["g"]) - 1.0) if fnum(a["g"]) > 0 and fnum(b["g"]) > 0 else math.nan) dstop = (fnum(a["stop_t"]) - fnum(b["stop_t"]) if fnum(a["stop_t"]) == fnum(a["stop_t"]) and fnum(b["stop_t"]) == fnum(b["stop_t"]) else math.nan) dmar = (fnum(a["thr"]) - fnum(b["thr"]) if fnum(a["thr"]) == fnum(a["thr"]) and fnum(b["thr"]) == fnum(b["thr"]) else math.nan) recs.append((cam, di, fnum(a["theta"]), OUTCOME[int(a["outcome"])], OUTCOME[int(b["outcome"])], same, dn, dg, dstop, dmar)) lines += table(f"critical pair {a_cfg} vs {b_cfg}", ["camera", "dir", "theta", "a", "b", "same", "dn_ang", "dgrel", "dstop_t", "dthr"], recs) return lines def main(argv): out = argv[1] if len(argv) > 1 else DEFAULT_OUT raw = Raw(out) os.makedirs(raw.logs, exist_ok=True) lines = build(raw) path = os.path.join(raw.logs, "summary.txt") with open(path, "w") as f: f.write("\n".join(lines) + "\n") print(f"wrote {path} ({len(lines)} lines)") return 0 if __name__ == "__main__": sys.exit(main(sys.argv))