bench: add repeatable solver measurements

Add an opt-in benchmark probe and JSON runner that separate solve, construction, validation, and rendering time. Record stable search counters, environment metadata, warm-up and repetition policy, timeouts, errors, median spread, and instrumentation overhead.

Compile production solving without counters and interleave counted and plain trials when measuring overhead. Keep heavyweight cases outside the default correctness path while testing counter and report behavior cheaply.

Tests: Debug and Release CTest suites (7 passed each)

Refs: #8

BENCHMARKING.md

@@ -0,0 +1,76 @@
+# Solver benchmarks
+
+The benchmark suite records repeatable performance data independently of the
+default correctness tests.  It exercises the exhaustive infeasible order 7 and
+the first-solution orders 8 and 9.  The benchmark executable is opt-in:
+
+```sh
+cmake -S . -B build-benchmark -DCMAKE_BUILD_TYPE=Release \
+  -DPARTRIDGE_BUILD_BENCHMARKS=ON
+cmake --build build-benchmark
+python3 benchmarks/run.py --binary build-benchmark/partridge_benchmark \
+  > benchmark.json
+```
+
+The default policy is one unrecorded warm-up followed by five repetitions per
+case, with a 600-second timeout for each process.  Solver stdout is captured;
+the probe renders into an in-memory stream so grids do not perturb terminal I/O.
+Override the policy with `--orders`, `--warmup`, `--repetitions`, and
+`--timeout`.  Order 9 is intentionally supported but may be omitted during
+local iteration because the current solver takes minutes:
+
+```sh
+python3 benchmarks/run.py --binary build-benchmark/partridge_benchmark \
+  --orders 7 8 --warmup 1 --repetitions 5 --timeout 60 > benchmark.json
+```
+
+The JSON contains every run and median, range, and median absolute deviation
+for solve, construction, independent validation, and rendering.  The current
+direct-search solver reports zero construction time: its setup and allocation
+remain part of solve time.  The separate construction field is reserved for
+future constructive solution paths.  The document also records compiler,
+flags, build type, commit, OS/CPU metadata, worker count, search policy, seed,
+timeouts, errors, invalid outputs, and the stdout policy.  Search counts must
+be stable across repeated runs.  Prune and task counters are zero for the
+current unpruned, single-threaded solver and reserve stable schema fields for
+later work.
+
+The runner writes its JSON report before returning a failure status if any mode
+has no completed runs or produces an error or invalid output.  Timeouts are
+reported but do not fail a case when another repetition completed.
+
+Normal `partridge_cpp` calls instantiate a compile-time counter-free solver.
+Use `--measure-overhead` to run both counter-free and counted variants and
+report their median difference:
+
+```sh
+python3 benchmarks/run.py --binary build-benchmark/partridge_benchmark \
+  --orders 8 --warmup 2 --repetitions 7 --timeout 60 --measure-overhead \
+  > overhead.json
+```
+
+Counter-free and counted warm-ups and measurements are interleaved.  The first
+mode alternates on each repetition, limiting systematic bias from temperature,
+frequency scaling, and run order.  Reported overhead is the difference between
+the two independently summarized medians.
+
+Do not use wall-clock thresholds as correctness checks.  Keep the generated
+JSON outside version control unless it is being deliberately added as a named
+comparison baseline.
+
+## Post-correctness baseline
+
+This framework starts from commit `ce39d0a` after the rendering assertion fix
+in #7 and completion check fix in #14.  The earlier Apple M1 Release results in
+`results.md` are approximately 1.76 seconds for order 8 and 158.69 seconds
+elapsed for order 9; they predate the structured runner and do not contain
+search counters.
+
+A clean structured baseline will be recorded here after the benchmark
+framework itself is committed.  Order 9 may use its documented historical
+result initially because its current runtime is several minutes; the default
+suite includes it with a per-run timeout.
+
+New optimization issues should quote the exact JSON
+environment, policy, median/spread, stable counters, and counted overhead from
+this runner for both before and after revisions.

CMakeLists.txt

@@ -6,6 +6,12 @@ set(CMAKE_CXX_STANDARD 20)
 add_executable(partridge_cpp
         main.cc)
 
+option(PARTRIDGE_BUILD_BENCHMARKS "Build the heavyweight benchmark probe" OFF)
+if(PARTRIDGE_BUILD_BENCHMARKS)
+  add_executable(partridge_benchmark
+          benchmarks/benchmark.cc)
+endif()
+
 include(CTest)
 
 if(BUILD_TESTING)
@@ -16,4 +22,11 @@ if(BUILD_TESTING)
   add_test(NAME rendering COMMAND partridge_tests rendering)
   add_test(NAME solver-small COMMAND partridge_tests solver-small)
   add_test(NAME solver-completion COMMAND partridge_tests solver-completion)
+  add_test(NAME search-counters COMMAND partridge_tests search-counters)
+  find_package(Python3 COMPONENTS Interpreter)
+  if(Python3_Interpreter_FOUND)
+    add_test(NAME benchmark-format
+      COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/run.py"
+              --self-test)
+  endif()
 endif()

README.md

@@ -63,3 +63,6 @@ N=9 # Set N to largest size of square.
 
 See [TESTING.md](./TESTING.md) for CTest, Debug, and sanitizer
 instructions.
+
+Performance measurements use the separate opt-in suite described in
+[BENCHMARKING.md](./BENCHMARKING.md).

benchmarks/benchmark.cc

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2026, Matthew-Gretton-Dann
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#define PARTRIDGE_TESTING
+#include "../main.cc"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <sstream>
+#include <string>
+
+namespace {
+  using Clock = std::chrono::steady_clock;
+
+  auto seconds(Clock::time_point begin, Clock::time_point end) -> double {
+    return std::chrono::duration<double>(end - begin).count();
+  }
+
+  auto valid(std::uint64_t order, Results const &result) -> bool {
+    auto const expected = triangle_num(order);
+    if (result.length() != expected) {
+      return false;
+    }
+
+    std::vector<std::uint64_t> multiplicities(order + 1);
+    std::vector<bool> occupied(expected * expected);
+    for (auto const &square: result.squares()) {
+      auto const side = square.length();
+      auto const x0 = square.pos() % expected;
+      auto const y0 = square.pos() / expected;
+      if (side == 0 || side > order || x0 + side > expected ||
+          y0 + side > expected) {
+        return false;
+      }
+      ++multiplicities[side];
+      for (auto y = y0; y < y0 + side; ++y) {
+        for (auto x = x0; x < x0 + side; ++x) {
+          auto &&cell = occupied[x + y * expected];
+          if (cell) {
+            return false;
+          }
+          cell = true;
+        }
+      }
+    }
+    for (std::uint64_t side = 1; side <= order; ++side) {
+      if (multiplicities[side] != side) {
+        return result.squares().empty() && order < 8;
+      }
+    }
+    return std::ranges::find(occupied, false) == occupied.end();
+  }
+
+  auto boolean(bool value) -> char const * { return value ? "true" : "false"; }
+}
+
+int main(int argc, char **argv) {
+  if (argc != 3) {
+    std::cerr << "usage: partridge_benchmark ORDER counters|plain\n";
+    return 2;
+  }
+  auto const order = static_cast<std::uint64_t>(std::strtoull(argv[1], nullptr, 10));
+  auto const instrument = std::string_view(argv[2]) == "counters";
+  if (!instrument && std::string_view(argv[2]) != "plain") {
+    std::cerr << "counter mode must be counters or plain\n";
+    return 2;
+  }
+
+  SearchCounters counters;
+  auto const solve_begin = Clock::now();
+  auto result = instrument ? find_solution_instrumented(order, counters)
+                           : find_solution(order);
+  auto const solve_end = Clock::now();
+
+  auto const validation_begin = Clock::now();
+  auto const validation_ok = valid(order, result);
+  auto const validation_end = Clock::now();
+
+  auto const render_begin = Clock::now();
+  std::ostringstream rendered;
+  auto *const old_buffer = std::cout.rdbuf(rendered.rdbuf());
+  result.output();
+  std::cout.rdbuf(old_buffer);
+  auto const render_end = Clock::now();
+
+  std::cout.precision(17);
+  std::cout
+      << "{\"schema_version\":1"
+      << ",\"order\":" << order
+      << ",\"instrumented\":" << boolean(instrument)
+      << ",\"solved\":" << boolean(!result.squares().empty())
+      << ",\"valid\":" << boolean(validation_ok)
+      << ",\"timing_seconds\":{\"solve\":" << seconds(solve_begin, solve_end)
+      << ",\"construction\":0"
+      << ",\"validation\":" << seconds(validation_begin, validation_end)
+      << ",\"render\":" << seconds(render_begin, render_end) << "}"
+      << ",\"counters\":{\"search_nodes\":" << counters.search_nodes
+      << ",\"loop_iterations\":" << counters.loop_iterations
+      << ",\"attempted_placements\":" << counters.attempted_placements
+      << ",\"backtracks\":" << counters.backtracks
+      << ",\"prune_checks\":" << counters.prune_checks
+      << ",\"prune_hits\":" << counters.prune_hits
+      << ",\"generated_tasks\":" << counters.generated_tasks
+      << ",\"completed_tasks\":" << counters.completed_tasks << "}}\n";
+  return validation_ok ? 0 : 1;
+}

benchmarks/run.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python3
+"""Repeatable Partridge solver benchmark runner."""
+
+import argparse
+import json
+import os
+import platform
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+def cache_values(build_dir):
+    values = {}
+    cache = build_dir / "CMakeCache.txt"
+    if not cache.exists():
+        return values
+    for line in cache.read_text(encoding="utf-8").splitlines():
+        if line.startswith(("//", "#")) or "=" not in line:
+            continue
+        key_type, value = line.split("=", 1)
+        key = key_type.split(":", 1)[0]
+        values[key] = value
+    return values
+
+
+def command_output(command, cwd=None):
+    try:
+        return subprocess.run(
+            command, cwd=cwd, check=True, capture_output=True, text=True
+        ).stdout.strip()
+    except (OSError, subprocess.CalledProcessError):
+        return "unknown"
+
+
+def environment(binary):
+    build_dir = binary.resolve().parent
+    cache = cache_values(build_dir)
+    compiler = cache.get("CMAKE_CXX_COMPILER", "unknown")
+    build_type = cache.get("CMAKE_BUILD_TYPE", "unknown")
+    type_flags = cache.get(f"CMAKE_CXX_FLAGS_{build_type.upper()}", "")
+    flags = " ".join(
+        part for part in (cache.get("CMAKE_CXX_FLAGS", ""), type_flags) if part
+    )
+    source_dir = cache.get("CMAKE_HOME_DIRECTORY")
+    commit = command_output(
+        ["git", "rev-parse", "HEAD"], cwd=source_dir if source_dir else None
+    )
+    status = command_output(
+        ["git", "status", "--porcelain"], cwd=source_dir if source_dir else None
+    )
+    compiler_version = (
+        command_output([compiler, "--version"]).splitlines()[:1]
+        if compiler != "unknown"
+        else ["unknown"]
+    )[0]
+    return {
+        "os": platform.platform(),
+        "compiler": compiler,
+        "compiler_version": compiler_version,
+        "compiler_flags": flags,
+        "linker_flags": cache.get("CMAKE_EXE_LINKER_FLAGS", ""),
+        "build_type": build_type,
+        "commit": commit,
+        "working_tree_dirty": status not in ("", "unknown"),
+        "hardware": {
+            "model": command_output(["sysctl", "-n", "hw.model"])
+            if sys.platform == "darwin"
+            else platform.node(),
+            "machine": platform.machine() or "unknown",
+            "processor": platform.processor() or "unknown",
+            "logical_cpus": os.cpu_count(),
+        },
+        "workers": 1,
+        "search_policy": "single-threaded, deterministic, largest-fitting-first",
+        "seed": None,
+    }
+
+
+def run_once(binary, order, mode, timeout):
+    started = time.monotonic()
+    try:
+        process = subprocess.run(
+            [str(binary), str(order), mode],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return {
+            "status": "timeout",
+            "wall_seconds": time.monotonic() - started,
+        }
+    if process.returncode != 0:
+        return {
+            "status": "error",
+            "returncode": process.returncode,
+            "stderr": process.stderr[-1000:],
+            "wall_seconds": time.monotonic() - started,
+        }
+    try:
+        result = json.loads(process.stdout)
+    except json.JSONDecodeError as error:
+        return {
+            "status": "invalid-output",
+            "detail": str(error),
+            "stdout": process.stdout[-1000:],
+        }
+    result["status"] = "ok"
+    result["wall_seconds"] = time.monotonic() - started
+    return result
+
+
+def distribution(values):
+    if not values:
+        return None
+    median = statistics.median(values)
+    return {
+        "median": median,
+        "min": min(values),
+        "max": max(values),
+        "spread": max(values) - min(values),
+        "median_absolute_deviation": statistics.median(
+            abs(value - median) for value in values
+        ),
+    }
+
+
+def summarize(runs):
+    completed = [run for run in runs if run["status"] == "ok"]
+    return {
+        "attempted": len(runs),
+        "completed": len(completed),
+        "timeouts": sum(run["status"] == "timeout" for run in runs),
+        "errors": sum(run["status"] == "error" for run in runs),
+        "invalid_outputs": sum(
+            run["status"] == "invalid-output" for run in runs
+        ),
+        "solve_seconds": distribution(
+            [run["timing_seconds"]["solve"] for run in completed]
+        ),
+        "construction_seconds": distribution(
+            [run["timing_seconds"]["construction"] for run in completed]
+        ),
+        "validation_seconds": distribution(
+            [run["timing_seconds"]["validation"] for run in completed]
+        ),
+        "render_seconds": distribution(
+            [run["timing_seconds"]["render"] for run in completed]
+        ),
+        "counter_values": completed[0]["counters"] if completed else None,
+        "counters_stable": bool(completed)
+        and all(run["counters"] == completed[0]["counters"] for run in completed),
+    }
+
+
+def summary_failed(summary):
+    return (
+        summary["completed"] == 0
+        or summary["errors"] > 0
+        or summary["invalid_outputs"] > 0
+    )
+
+
+def self_test():
+    sample = distribution([1.0, 3.0, 2.0])
+    assert sample["median"] == 2.0
+    assert sample["spread"] == 2.0
+    assert sample["median_absolute_deviation"] == 1.0
+    parsed = json.loads(
+        '{"timing_seconds":{"solve":1,"construction":0},"counters":{}}'
+    )
+    assert parsed["timing_seconds"]["solve"] == 1
+    runs = [
+        {
+            "status": "ok",
+            "timing_seconds": {
+                "solve": 1.0,
+                "construction": 0.0,
+                "validation": 0.1,
+                "render": 0.2,
+            },
+            "counters": {"search_nodes": 3},
+        },
+        {"status": "timeout"},
+        {"status": "error"},
+        {"status": "invalid-output"},
+    ]
+    summary = summarize(runs)
+    assert summary["completed"] == 1
+    assert summary["timeouts"] == 1
+    assert summary["errors"] == 1
+    assert summary["invalid_outputs"] == 1
+    assert summary["construction_seconds"]["median"] == 0.0
+    assert summary_failed(summary)
+    assert summary_failed(summarize([{"status": "timeout"}]))
+    completed_with_timeout = summarize([runs[0], {"status": "timeout"}])
+    assert not summary_failed(completed_with_timeout)
+    assert trial_modes(["counters", "plain"], 0) == ["counters", "plain"]
+    assert trial_modes(["counters", "plain"], 1) == ["plain", "counters"]
+
+
+def trial_modes(modes, trial):
+    if len(modes) < 2 or trial % 2 == 0:
+        return modes
+    return list(reversed(modes))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--binary", type=Path)
+    parser.add_argument("--orders", nargs="+", type=int, default=[7, 8, 9])
+    parser.add_argument("--warmup", type=int, default=1)
+    parser.add_argument("--repetitions", type=int, default=5)
+    parser.add_argument("--timeout", type=float, default=600.0)
+    parser.add_argument(
+        "--measure-overhead",
+        action="store_true",
+        help="also run the compile-time counter-free solver",
+    )
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        self_test()
+        return 0
+    if args.binary is None:
+        parser.error("--binary is required")
+    if args.warmup < 0 or args.repetitions < 1 or args.timeout <= 0:
+        parser.error("warmup must be non-negative; repetitions and timeout positive")
+
+    modes = ["counters", "plain"]
+    if not args.measure_overhead:
+        modes = ["counters"]
+    cases = []
+    failed = False
+    for order in args.orders:
+        mode_results = {mode: {"runs": []} for mode in modes}
+        for trial in range(args.warmup):
+            for mode in trial_modes(modes, trial):
+                run_once(args.binary, order, mode, args.timeout)
+        for trial in range(args.repetitions):
+            for mode in trial_modes(modes, trial):
+                mode_results[mode]["runs"].append(
+                    run_once(args.binary, order, mode, args.timeout)
+                )
+        for result in mode_results.values():
+            result["summary"] = summarize(result["runs"])
+            summary = result["summary"]
+            failed = failed or summary_failed(summary)
+        overhead = None
+        counted = mode_results["counters"]["summary"]["solve_seconds"]
+        plain = mode_results.get("plain", {}).get("summary", {}).get("solve_seconds")
+        if counted and plain and plain["median"]:
+            overhead = {
+                "median_seconds": counted["median"] - plain["median"],
+                "median_percent": 100.0
+                * (counted["median"] / plain["median"] - 1.0),
+            }
+        cases.append({"order": order, "modes": mode_results, "counter_overhead": overhead})
+
+    document = {
+        "schema": "partridge-benchmark-v1",
+        "environment": environment(args.binary),
+        "policy": {
+            "orders": args.orders,
+            "warmup_runs": args.warmup,
+            "measured_repetitions": args.repetitions,
+            "per_run_timeout_seconds": args.timeout,
+            "stdout": "captured; rendered grid suppressed by probe",
+        },
+        "cases": cases,
+    }
+    json.dump(document, sys.stdout, indent=2, sort_keys=True)
+    sys.stdout.write("\n")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

main.cc

@@ -228,11 +228,30 @@ namespace {
   /** Vector used to identify the available squares. */
   using Avail = std::vector<size_t>;
 
+  /** Optional search instrumentation.
+   *
+   * Counters for search features which are not implemented by the current
+   * single-threaded solver remain zero.  Keeping them in the stable output
+   * schema lets later solver implementations remain comparable.
+   */
+  struct SearchCounters {
+    size_t search_nodes = 0;
+    size_t loop_iterations = 0;
+    size_t attempted_placements = 0;
+    size_t backtracks = 0;
+    size_t prune_checks = 0;
+    size_t prune_hits = 0;
+    size_t generated_tasks = 0;
+    size_t completed_tasks = 0;
+  };
+
   /** Find a solution to the \a n th Partridge problem.
    *
    * Returns the grid of the solution.
    */
-  auto find_solution(size_t const n) noexcept -> Results {
+  template<bool Instrument>
+  auto find_solution_impl(size_t const n, SearchCounters *const counters) noexcept
+      -> Results {
     /* Implementation is iterative, as opposed to recursive.
      *
      * The recursive implementation is easier to understand - but is
@@ -264,7 +283,15 @@ namespace {
     Pos pos = 0;
     size_t idx = n;
 
+    if constexpr (Instrument) {
+      assert(counters != nullptr);
+      ++counters->search_nodes;
+    }
+
     while (true) {
+      if constexpr (Instrument) {
+        ++counters->loop_iterations;
+      }
       /* If the idx is 0 we've looked at all possible square lengths for this
        * position, and they've failed.  Pop the last square of the stack, remove
        * it from the grid and try the next smaller size in the same position.
@@ -277,6 +304,9 @@ namespace {
         sqs.pop_back();
         grid.clear(sq);
         ++avail_sqs[sq.length()];
+        if constexpr (Instrument) {
+          ++counters->backtracks;
+        }
         pos = sq.pos();
         idx = sq.length() - 1;
         continue;
@@ -292,6 +322,9 @@ namespace {
        * set up to look at the next position.
        */
       auto const sq = Square(pos, idx);
+      if constexpr (Instrument) {
+        ++counters->attempted_placements;
+      }
       --avail_sqs[idx];
       grid.add(sq);
       sqs.push_back(sq);
@@ -301,11 +334,24 @@ namespace {
       // Have we reached the end?  If so success!
       if (pos == grid.end()) { break; }
 
+      if constexpr (Instrument) {
+        ++counters->search_nodes;
+      }
       idx = grid.largest_square(pos, n);
     }
 
     return {length, sqs};
   }
+
+  auto find_solution(size_t const n) noexcept -> Results {
+    return find_solution_impl<false>(n, nullptr);
+  }
+
+  auto find_solution_instrumented(size_t const n,
+                                  SearchCounters &counters) noexcept -> Results {
+    counters = {};
+    return find_solution_impl<true>(n, &counters);
+  }
 } // anon namespace
 
 #ifndef PARTRIDGE_TESTING

tests/tests.cc

@@ -281,6 +281,26 @@ namespace {
     return failures;
   }
 
+  auto test_search_counters() -> int {
+    SearchCounters counters;
+    auto const solution = find_solution_instrumented(2, counters);
+    int failures = 0;
+    failures += expect(solution.squares().empty(),
+                       "instrumented solver changed an infeasible result");
+    failures += expect(counters.search_nodes > 0 &&
+                           counters.loop_iterations >= counters.search_nodes,
+                       "instrumented solver did not count search work");
+    failures += expect(counters.attempted_placements > 0 &&
+                           counters.backtracks > 0,
+                       "instrumented solver did not count placements/backtracks");
+    failures += expect(counters.prune_checks == 0 &&
+                           counters.prune_hits == 0 &&
+                           counters.generated_tasks == 0 &&
+                           counters.completed_tasks == 0,
+                       "unimplemented solver counters were not zero");
+    return failures;
+  }
+
   auto test_solver_completion() -> int {
     int failures = 0;
     for (auto const order: std::array<std::uint64_t, 2>{1, 8}) {
@@ -317,6 +337,9 @@ int main(int argc, char **argv) {
   if (test == "solver-completion") {
     return test_solver_completion();
   }
+  if (test == "search-counters") {
+    return test_search_counters();
+  }
   std::cerr << "unknown test: " << test << '\n';
   return 2;
 }