bench: add optional CP-SAT reference

Provide a separately installed OR-Tools model to compare a generic constraint solver with the native first-solution path without adding a production or default-test dependency. Use no-overlap, exact-fill, edge and equal-copy symmetry constraints, validate placements independently, and record model size, memory, worker count and timings for orders 8 and 9. Keep the tool only as a reference and defer DLX absent new evidence. Tests: Release CTest (9 passed) Tests: Python reference tests and compilation checks Tests: independently validated CP-SAT orders 8 and 9 Refs: #5
2026-07-30 17:40:17 +01:00
parent 3e667d6de0
commit 0a7ce1e49e
11 changed files with 660 additions and 0 deletions
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Repeat the optional CP-SAT reference and summarize its JSON reports."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--solver", type=Path, required=True)
+    parser.add_argument("--order", type=int, default=8)
+    parser.add_argument("--workers", type=int, required=True)
+    parser.add_argument("--repetitions", type=int, default=3)
+    parser.add_argument("--time-limit", type=float, default=600)
+    args = parser.parse_args()
+    if args.repetitions < 1:
+        parser.error("repetitions must be positive")
+
+    runs = []
+    process_errors = []
+    for _ in range(args.repetitions):
+        command = [
+            sys.executable,
+            str(args.solver),
+            str(args.order),
+            "--workers",
+            str(args.workers),
+            "--time-limit",
+            str(args.time_limit),
+        ]
+        process = subprocess.run(command, text=True, capture_output=True)
+        try:
+            report = json.loads(process.stdout)
+        except json.JSONDecodeError as error:
+            parser.exit(1, f"reference produced invalid JSON: {error}\n{process.stderr}")
+        runs.append(report)
+        if process.returncode != 0:
+            process_errors.append(
+                {
+                    "returncode": process.returncode,
+                    "stderr": process.stderr,
+                    "status": report.get("result", {}).get("status"),
+                }
+            )
+
+    completed = [run for run in runs if run["result"]["valid"]]
+    solve_times = [run["timing_seconds"]["solve"] for run in completed]
+    first_model = runs[0]["model"]
+    model_stable = all(run["model"] == first_model for run in runs)
+    output = {
+        "schema": "partridge-cp-sat-benchmark-v1",
+        "order": args.order,
+        "workers": args.workers,
+        "repetitions": args.repetitions,
+        "completed": len(completed),
+        "model": first_model,
+        "model_stable": model_stable,
+        "process_errors": process_errors,
+        "peak_rss_bytes": [run["memory"]["peak_rss_bytes"] for run in runs],
+        "solve_seconds": solve_times,
+        "solve_median_seconds": statistics.median(solve_times) if solve_times else None,
+        "runs": runs,
+    }
+    json.dump(output, sys.stdout, indent=2, sort_keys=True)
+    sys.stdout.write("\n")
+    return (
+        0
+        if len(completed) == args.repetitions
+        and not process_errors
+        and model_stable
+        else 1
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())