diff --git a/BENCHMARKING.md b/BENCHMARKING.md new file mode 100644 index 0000000..3ee0bf2 --- /dev/null +++ b/BENCHMARKING.md @@ -0,0 +1,88 @@ +# Solver benchmarks + +The benchmark suite records repeatable performance data independently of the +default correctness tests. It exercises the exhaustive infeasible order 7 and +the first-solution orders 8 and 9. The benchmark executable is opt-in: + +```sh +cmake -S . -B build-benchmark -DCMAKE_BUILD_TYPE=Release \ + -DPARTRIDGE_BUILD_BENCHMARKS=ON +cmake --build build-benchmark +python3 benchmarks/run.py --binary build-benchmark/partridge_benchmark \ + > benchmark.json +``` + +The default policy is one unrecorded warm-up followed by five repetitions per +case, with a 600-second timeout for each process. Solver stdout is captured; +the probe renders into an in-memory stream so grids do not perturb terminal I/O. +Override the policy with `--orders`, `--warmup`, `--repetitions`, and +`--timeout`. Order 9 is intentionally supported but may be omitted during +local iteration because the current solver takes minutes: + +```sh +python3 benchmarks/run.py --binary build-benchmark/partridge_benchmark \ + --orders 7 8 --warmup 1 --repetitions 5 --timeout 60 > benchmark.json +``` + +The JSON contains every run and median, range, and median absolute deviation +for solve, construction, independent validation, and rendering. The current +direct-search solver reports zero construction time: its setup and allocation +remain part of solve time. The separate construction field is reserved for +future constructive solution paths. The document also records compiler, +flags, build type, commit, OS/CPU metadata, worker count, search policy, seed, +timeouts, errors, invalid outputs, and the stdout policy. Search counts must +be stable across repeated runs. Prune and task counters are zero for the +current unpruned, single-threaded solver and reserve stable schema fields for +later work. + +The runner writes its JSON report before returning a failure status if any mode +has no completed runs or produces an error or invalid output. Timeouts are +reported but do not fail a case when another repetition completed. + +Normal `partridge_cpp` calls instantiate a compile-time counter-free solver. +Use `--measure-overhead` to run both counter-free and counted variants and +report their median difference: + +```sh +python3 benchmarks/run.py --binary build-benchmark/partridge_benchmark \ + --orders 8 --warmup 2 --repetitions 7 --timeout 60 --measure-overhead \ + > overhead.json +``` + +Counter-free and counted warm-ups and measurements are interleaved. The first +mode alternates on each repetition, limiting systematic bias from temperature, +frequency scaling, and run order. Reported overhead is the difference between +the two independently summarized medians. + +Do not use wall-clock thresholds as correctness checks. Keep the generated +JSON outside version control unless it is being deliberately added as a named +comparison baseline. + +## Post-correctness baseline + +This framework starts from commit `ce39d0a` after the rendering assertion fix +in #7 and completion check fix in #14. The earlier Apple M1 Release results in +`results.md` are approximately 1.76 seconds for order 8 and 158.69 seconds +elapsed for order 9; they predate the structured runner and do not contain +search counters. + +The first clean structured baseline used commit `598667b`, Apple Clang 21.0.0 +with `-O3 -DNDEBUG`, Apple arm64, one worker, one warm-up, and three measured +repetitions. The runner reported a clean working tree: + +| Order | Result | Counted solve median (range) | Nodes | Placements | Backtracks | +| --- | --- | --- | ---: | ---: | ---: | +| 7 | infeasible | 3.453 s (3.444–3.455 s) | 110,483,315 | 110,483,314 | 110,483,314 | +| 8 | solution | 1.817 s (1.814–1.817 s) | 60,485,176 | 60,485,176 | 60,485,140 | + +Counts were stable across repetitions. Interleaved counter-free medians were +3.435 seconds for order 7 and 1.805 seconds for order 8, giving counted +overheads of 0.53% and 0.64% respectively. Construction time was zero; median +independent validation and rendering times were each below 0.02 milliseconds. + +Order 9 was not rerun for this initial baseline because its documented runtime +is several minutes. The default suite includes it with a per-run timeout. + +New optimization issues should quote the exact JSON +environment, policy, median/spread, stable counters, and counted overhead from +this runner for both before and after revisions. diff --git a/CMakeLists.txt b/CMakeLists.txt index a87e234..1302ea4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,12 @@ set(CMAKE_CXX_STANDARD 20) add_executable(partridge_cpp main.cc) +option(PARTRIDGE_BUILD_BENCHMARKS "Build the heavyweight benchmark probe" OFF) +if(PARTRIDGE_BUILD_BENCHMARKS) + add_executable(partridge_benchmark + benchmarks/benchmark.cc) +endif() + include(CTest) if(BUILD_TESTING) @@ -16,4 +22,11 @@ if(BUILD_TESTING) add_test(NAME rendering COMMAND partridge_tests rendering) add_test(NAME solver-small COMMAND partridge_tests solver-small) add_test(NAME solver-completion COMMAND partridge_tests solver-completion) + add_test(NAME search-counters COMMAND partridge_tests search-counters) + find_package(Python3 COMPONENTS Interpreter) + if(Python3_Interpreter_FOUND) + add_test(NAME benchmark-format + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/benchmarks/run.py" + --self-test) + endif() endif() diff --git a/README.md b/README.md index 3d49145..4ff9fce 100644 --- a/README.md +++ b/README.md @@ -63,3 +63,6 @@ N=9 # Set N to largest size of square. See [TESTING.md](./TESTING.md) for CTest, Debug, and sanitizer instructions. + +Performance measurements use the separate opt-in suite described in +[BENCHMARKING.md](./BENCHMARKING.md). diff --git a/benchmarks/benchmark.cc b/benchmarks/benchmark.cc new file mode 100644 index 0000000..c19a61e --- /dev/null +++ b/benchmarks/benchmark.cc @@ -0,0 +1,110 @@ +/* + * Copyright 2026, Matthew-Gretton-Dann + * SPDX-License-Identifier: Apache-2.0 + */ + +#define PARTRIDGE_TESTING +#include "../main.cc" + +#include +#include +#include +#include +#include +#include + +namespace { + using Clock = std::chrono::steady_clock; + + auto seconds(Clock::time_point begin, Clock::time_point end) -> double { + return std::chrono::duration(end - begin).count(); + } + + auto valid(std::uint64_t order, Results const &result) -> bool { + auto const expected = triangle_num(order); + if (result.length() != expected) { + return false; + } + + std::vector multiplicities(order + 1); + std::vector occupied(expected * expected); + for (auto const &square: result.squares()) { + auto const side = square.length(); + auto const x0 = square.pos() % expected; + auto const y0 = square.pos() / expected; + if (side == 0 || side > order || x0 + side > expected || + y0 + side > expected) { + return false; + } + ++multiplicities[side]; + for (auto y = y0; y < y0 + side; ++y) { + for (auto x = x0; x < x0 + side; ++x) { + auto &&cell = occupied[x + y * expected]; + if (cell) { + return false; + } + cell = true; + } + } + } + for (std::uint64_t side = 1; side <= order; ++side) { + if (multiplicities[side] != side) { + return result.squares().empty() && order < 8; + } + } + return std::ranges::find(occupied, false) == occupied.end(); + } + + auto boolean(bool value) -> char const * { return value ? "true" : "false"; } +} + +int main(int argc, char **argv) { + if (argc != 3) { + std::cerr << "usage: partridge_benchmark ORDER counters|plain\n"; + return 2; + } + auto const order = static_cast(std::strtoull(argv[1], nullptr, 10)); + auto const instrument = std::string_view(argv[2]) == "counters"; + if (!instrument && std::string_view(argv[2]) != "plain") { + std::cerr << "counter mode must be counters or plain\n"; + return 2; + } + + SearchCounters counters; + auto const solve_begin = Clock::now(); + auto result = instrument ? find_solution_instrumented(order, counters) + : find_solution(order); + auto const solve_end = Clock::now(); + + auto const validation_begin = Clock::now(); + auto const validation_ok = valid(order, result); + auto const validation_end = Clock::now(); + + auto const render_begin = Clock::now(); + std::ostringstream rendered; + auto *const old_buffer = std::cout.rdbuf(rendered.rdbuf()); + result.output(); + std::cout.rdbuf(old_buffer); + auto const render_end = Clock::now(); + + std::cout.precision(17); + std::cout + << "{\"schema_version\":1" + << ",\"order\":" << order + << ",\"instrumented\":" << boolean(instrument) + << ",\"solved\":" << boolean(!result.squares().empty()) + << ",\"valid\":" << boolean(validation_ok) + << ",\"timing_seconds\":{\"solve\":" << seconds(solve_begin, solve_end) + << ",\"construction\":0" + << ",\"validation\":" << seconds(validation_begin, validation_end) + << ",\"render\":" << seconds(render_begin, render_end) << "}" + << ",\"counters\":{\"search_nodes\":" << counters.search_nodes + << ",\"loop_iterations\":" << counters.loop_iterations + << ",\"attempted_placements\":" << counters.attempted_placements + << ",\"backtracks\":" << counters.backtracks + << ",\"prune_checks\":" << counters.prune_checks + << ",\"prune_hits\":" << counters.prune_hits + << ",\"generated_tasks\":" << counters.generated_tasks + << ",\"completed_tasks\":" << counters.completed_tasks << "}}\n"; + return validation_ok ? 0 : 1; +} diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100755 index 0000000..5484e23 --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Repeatable Partridge solver benchmark runner.""" + +import argparse +import json +import os +import platform +import statistics +import subprocess +import sys +import time +from pathlib import Path + + +def cache_values(build_dir): + values = {} + cache = build_dir / "CMakeCache.txt" + if not cache.exists(): + return values + for line in cache.read_text(encoding="utf-8").splitlines(): + if line.startswith(("//", "#")) or "=" not in line: + continue + key_type, value = line.split("=", 1) + key = key_type.split(":", 1)[0] + values[key] = value + return values + + +def command_output(command, cwd=None): + try: + return subprocess.run( + command, cwd=cwd, check=True, capture_output=True, text=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def environment(binary): + build_dir = binary.resolve().parent + cache = cache_values(build_dir) + compiler = cache.get("CMAKE_CXX_COMPILER", "unknown") + build_type = cache.get("CMAKE_BUILD_TYPE", "unknown") + type_flags = cache.get(f"CMAKE_CXX_FLAGS_{build_type.upper()}", "") + flags = " ".join( + part for part in (cache.get("CMAKE_CXX_FLAGS", ""), type_flags) if part + ) + source_dir = cache.get("CMAKE_HOME_DIRECTORY") + commit = command_output( + ["git", "rev-parse", "HEAD"], cwd=source_dir if source_dir else None + ) + status = command_output( + ["git", "status", "--porcelain"], cwd=source_dir if source_dir else None + ) + compiler_version = ( + command_output([compiler, "--version"]).splitlines()[:1] + if compiler != "unknown" + else ["unknown"] + )[0] + return { + "os": platform.platform(), + "compiler": compiler, + "compiler_version": compiler_version, + "compiler_flags": flags, + "linker_flags": cache.get("CMAKE_EXE_LINKER_FLAGS", ""), + "build_type": build_type, + "commit": commit, + "working_tree_dirty": status not in ("", "unknown"), + "hardware": { + "model": command_output(["sysctl", "-n", "hw.model"]) + if sys.platform == "darwin" + else platform.node(), + "machine": platform.machine() or "unknown", + "processor": platform.processor() or "unknown", + "logical_cpus": os.cpu_count(), + }, + "workers": 1, + "search_policy": "single-threaded, deterministic, largest-fitting-first", + "seed": None, + } + + +def run_once(binary, order, mode, timeout): + started = time.monotonic() + try: + process = subprocess.run( + [str(binary), str(order), mode], + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return { + "status": "timeout", + "wall_seconds": time.monotonic() - started, + } + if process.returncode != 0: + return { + "status": "error", + "returncode": process.returncode, + "stderr": process.stderr[-1000:], + "wall_seconds": time.monotonic() - started, + } + try: + result = json.loads(process.stdout) + except json.JSONDecodeError as error: + return { + "status": "invalid-output", + "detail": str(error), + "stdout": process.stdout[-1000:], + } + result["status"] = "ok" + result["wall_seconds"] = time.monotonic() - started + return result + + +def distribution(values): + if not values: + return None + median = statistics.median(values) + return { + "median": median, + "min": min(values), + "max": max(values), + "spread": max(values) - min(values), + "median_absolute_deviation": statistics.median( + abs(value - median) for value in values + ), + } + + +def summarize(runs): + completed = [run for run in runs if run["status"] == "ok"] + return { + "attempted": len(runs), + "completed": len(completed), + "timeouts": sum(run["status"] == "timeout" for run in runs), + "errors": sum(run["status"] == "error" for run in runs), + "invalid_outputs": sum( + run["status"] == "invalid-output" for run in runs + ), + "solve_seconds": distribution( + [run["timing_seconds"]["solve"] for run in completed] + ), + "construction_seconds": distribution( + [run["timing_seconds"]["construction"] for run in completed] + ), + "validation_seconds": distribution( + [run["timing_seconds"]["validation"] for run in completed] + ), + "render_seconds": distribution( + [run["timing_seconds"]["render"] for run in completed] + ), + "counter_values": completed[0]["counters"] if completed else None, + "counters_stable": bool(completed) + and all(run["counters"] == completed[0]["counters"] for run in completed), + } + + +def summary_failed(summary): + return ( + summary["completed"] == 0 + or summary["errors"] > 0 + or summary["invalid_outputs"] > 0 + ) + + +def self_test(): + sample = distribution([1.0, 3.0, 2.0]) + assert sample["median"] == 2.0 + assert sample["spread"] == 2.0 + assert sample["median_absolute_deviation"] == 1.0 + parsed = json.loads( + '{"timing_seconds":{"solve":1,"construction":0},"counters":{}}' + ) + assert parsed["timing_seconds"]["solve"] == 1 + runs = [ + { + "status": "ok", + "timing_seconds": { + "solve": 1.0, + "construction": 0.0, + "validation": 0.1, + "render": 0.2, + }, + "counters": {"search_nodes": 3}, + }, + {"status": "timeout"}, + {"status": "error"}, + {"status": "invalid-output"}, + ] + summary = summarize(runs) + assert summary["completed"] == 1 + assert summary["timeouts"] == 1 + assert summary["errors"] == 1 + assert summary["invalid_outputs"] == 1 + assert summary["construction_seconds"]["median"] == 0.0 + assert summary_failed(summary) + assert summary_failed(summarize([{"status": "timeout"}])) + completed_with_timeout = summarize([runs[0], {"status": "timeout"}]) + assert not summary_failed(completed_with_timeout) + assert trial_modes(["counters", "plain"], 0) == ["counters", "plain"] + assert trial_modes(["counters", "plain"], 1) == ["plain", "counters"] + + +def trial_modes(modes, trial): + if len(modes) < 2 or trial % 2 == 0: + return modes + return list(reversed(modes)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", type=Path) + parser.add_argument("--orders", nargs="+", type=int, default=[7, 8, 9]) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repetitions", type=int, default=5) + parser.add_argument("--timeout", type=float, default=600.0) + parser.add_argument( + "--measure-overhead", + action="store_true", + help="also run the compile-time counter-free solver", + ) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + self_test() + return 0 + if args.binary is None: + parser.error("--binary is required") + if args.warmup < 0 or args.repetitions < 1 or args.timeout <= 0: + parser.error("warmup must be non-negative; repetitions and timeout positive") + + modes = ["counters", "plain"] + if not args.measure_overhead: + modes = ["counters"] + cases = [] + failed = False + for order in args.orders: + mode_results = {mode: {"runs": []} for mode in modes} + for trial in range(args.warmup): + for mode in trial_modes(modes, trial): + run_once(args.binary, order, mode, args.timeout) + for trial in range(args.repetitions): + for mode in trial_modes(modes, trial): + mode_results[mode]["runs"].append( + run_once(args.binary, order, mode, args.timeout) + ) + for result in mode_results.values(): + result["summary"] = summarize(result["runs"]) + summary = result["summary"] + failed = failed or summary_failed(summary) + overhead = None + counted = mode_results["counters"]["summary"]["solve_seconds"] + plain = mode_results.get("plain", {}).get("summary", {}).get("solve_seconds") + if counted and plain and plain["median"]: + overhead = { + "median_seconds": counted["median"] - plain["median"], + "median_percent": 100.0 + * (counted["median"] / plain["median"] - 1.0), + } + cases.append({"order": order, "modes": mode_results, "counter_overhead": overhead}) + + document = { + "schema": "partridge-benchmark-v1", + "environment": environment(args.binary), + "policy": { + "orders": args.orders, + "warmup_runs": args.warmup, + "measured_repetitions": args.repetitions, + "per_run_timeout_seconds": args.timeout, + "stdout": "captured; rendered grid suppressed by probe", + }, + "cases": cases, + } + json.dump(document, sys.stdout, indent=2, sort_keys=True) + sys.stdout.write("\n") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/main.cc b/main.cc index 55e69f0..e92fb7d 100644 --- a/main.cc +++ b/main.cc @@ -228,11 +228,30 @@ namespace { /** Vector used to identify the available squares. */ using Avail = std::vector; + /** Optional search instrumentation. + * + * Counters for search features which are not implemented by the current + * single-threaded solver remain zero. Keeping them in the stable output + * schema lets later solver implementations remain comparable. + */ + struct SearchCounters { + size_t search_nodes = 0; + size_t loop_iterations = 0; + size_t attempted_placements = 0; + size_t backtracks = 0; + size_t prune_checks = 0; + size_t prune_hits = 0; + size_t generated_tasks = 0; + size_t completed_tasks = 0; + }; + /** Find a solution to the \a n th Partridge problem. * * Returns the grid of the solution. */ - auto find_solution(size_t const n) noexcept -> Results { + template + auto find_solution_impl(size_t const n, SearchCounters *const counters) noexcept + -> Results { /* Implementation is iterative, as opposed to recursive. * * The recursive implementation is easier to understand - but is @@ -264,7 +283,15 @@ namespace { Pos pos = 0; size_t idx = n; + if constexpr (Instrument) { + assert(counters != nullptr); + ++counters->search_nodes; + } + while (true) { + if constexpr (Instrument) { + ++counters->loop_iterations; + } /* If the idx is 0 we've looked at all possible square lengths for this * position, and they've failed. Pop the last square of the stack, remove * it from the grid and try the next smaller size in the same position. @@ -277,6 +304,9 @@ namespace { sqs.pop_back(); grid.clear(sq); ++avail_sqs[sq.length()]; + if constexpr (Instrument) { + ++counters->backtracks; + } pos = sq.pos(); idx = sq.length() - 1; continue; @@ -292,6 +322,9 @@ namespace { * set up to look at the next position. */ auto const sq = Square(pos, idx); + if constexpr (Instrument) { + ++counters->attempted_placements; + } --avail_sqs[idx]; grid.add(sq); sqs.push_back(sq); @@ -301,11 +334,24 @@ namespace { // Have we reached the end? If so success! if (pos == grid.end()) { break; } + if constexpr (Instrument) { + ++counters->search_nodes; + } idx = grid.largest_square(pos, n); } return {length, sqs}; } + + auto find_solution(size_t const n) noexcept -> Results { + return find_solution_impl(n, nullptr); + } + + auto find_solution_instrumented(size_t const n, + SearchCounters &counters) noexcept -> Results { + counters = {}; + return find_solution_impl(n, &counters); + } } // anon namespace #ifndef PARTRIDGE_TESTING diff --git a/tests/tests.cc b/tests/tests.cc index 939fc78..3bc6fe4 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -281,6 +281,26 @@ namespace { return failures; } + auto test_search_counters() -> int { + SearchCounters counters; + auto const solution = find_solution_instrumented(2, counters); + int failures = 0; + failures += expect(solution.squares().empty(), + "instrumented solver changed an infeasible result"); + failures += expect(counters.search_nodes > 0 && + counters.loop_iterations >= counters.search_nodes, + "instrumented solver did not count search work"); + failures += expect(counters.attempted_placements > 0 && + counters.backtracks > 0, + "instrumented solver did not count placements/backtracks"); + failures += expect(counters.prune_checks == 0 && + counters.prune_hits == 0 && + counters.generated_tasks == 0 && + counters.completed_tasks == 0, + "unimplemented solver counters were not zero"); + return failures; + } + auto test_solver_completion() -> int { int failures = 0; for (auto const order: std::array{1, 8}) { @@ -317,6 +337,9 @@ int main(int argc, char **argv) { if (test == "solver-completion") { return test_solver_completion(); } + if (test == "search-counters") { + return test_search_counters(); + } std::cerr << "unknown test: " << test << '\n'; return 2; }