#!/usr/bin/env python3 """Repeat the optional CP-SAT reference and summarize its JSON reports.""" from __future__ import annotations import argparse import json import statistics import subprocess import sys from pathlib import Path def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--solver", type=Path, required=True) parser.add_argument("--order", type=int, default=8) parser.add_argument("--workers", type=int, required=True) parser.add_argument("--repetitions", type=int, default=3) parser.add_argument("--time-limit", type=float, default=600) args = parser.parse_args() if args.repetitions < 1: parser.error("repetitions must be positive") runs = [] process_errors = [] for _ in range(args.repetitions): command = [ sys.executable, str(args.solver), str(args.order), "--workers", str(args.workers), "--time-limit", str(args.time_limit), ] process = subprocess.run(command, text=True, capture_output=True) try: report = json.loads(process.stdout) except json.JSONDecodeError as error: parser.exit(1, f"reference produced invalid JSON: {error}\n{process.stderr}") runs.append(report) if process.returncode != 0: process_errors.append( { "returncode": process.returncode, "stderr": process.stderr, "status": report.get("result", {}).get("status"), } ) completed = [run for run in runs if run["result"]["valid"]] solve_times = [run["timing_seconds"]["solve"] for run in completed] first_model = runs[0]["model"] model_stable = all(run["model"] == first_model for run in runs) output = { "schema": "partridge-cp-sat-benchmark-v1", "order": args.order, "workers": args.workers, "repetitions": args.repetitions, "completed": len(completed), "model": first_model, "model_stable": model_stable, "process_errors": process_errors, "peak_rss_bytes": [run["memory"]["peak_rss_bytes"] for run in runs], "solve_seconds": solve_times, "solve_median_seconds": statistics.median(solve_times) if solve_times else None, "runs": runs, } json.dump(output, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") return ( 0 if len(completed) == args.repetitions and not process_errors and model_stable else 1 ) if __name__ == "__main__": raise SystemExit(main())