Source tests/test_bench.py
"""Sampling benchmarks reject incomplete executions without treating observations as failures."""import argparsefrom contextlib import redirect_stdoutimport iofrom pathlib import Pathimport sysimport unittestfrom unittest.mock import patchROOT = Path(__file__).resolve().parents[1]sys.path.insert(0, str(ROOT / "tools"))import benchclass SamplingBenchmarkTests(unittest.TestCase):    def sample(self, source="", target="", run=None):        args = argparse.Namespace(binary=Path("determinize"), samples=10, seed=0,                                  fuel=100, timeout=30)        output = io.StringIO()        if run is None:            run = bench.ToolRun(0, source + target, "", 1.25)        with patch.object(bench, "count_sample_sites", return_value=bench.SampleSites(1, 1, 1, 0)), \             patch.object(bench, "run_tool", return_value=run), redirect_stdout(output):            result = bench.det_approx(Path("input.det"), args)        return result, output.getvalue()    def test_execution_failure(self):        for label in ("Source", "Determinized"):            with self.subTest(label=label):                summaries = {                    name: (f"{name}: {9 if name == label else 10}/10 runs returned a value\n"                           "  empirical mean among returned values: 5; variance: 2\n"                           + ("  first failure: step limit reached\n" if name == label else ""))                    for name in ("Source", "Determinized")                }                result, output = self.sample(summaries["Source"], summaries["Determinized"])                self.assertEqual(result["status"], "failed")                self.assertEqual(result["stage"], "sampling")                self.assertIn(f"{label}: 1/10 executions failed", result["error"])                self.assertIn("first failure: step limit reached", result["error"])                self.assertIsNone(result["variance_reduction_factor"])                self.assertIn(f"ERROR: {label}: 1/10 executions failed", output)                self.assertIn("Variance reduction: n/a (sampling failed)", output)                failed = result["source" if label == "Source" else "determinized"]                self.assertEqual(failed["returned"], 9)                self.assertEqual(failed["requested"], 10)                self.assertEqual(failed["failed"], 1)                self.assertEqual(failed["first_failure"], "step limit reached")    def test_sampling_timeout(self):        result, output = self.sample(run=bench.ToolRun(None, "", "", 30))        self.assertEqual(result["status"], "failed")        self.assertEqual(result["stage"], "sampling")        self.assertIsNone(result.get("variance_reduction_factor"))        self.assertIn("ERROR: sampling timed out after 30s", output)    def test_incomplete_counts(self):        for count in ("9/10", "9/9"):            with self.subTest(count=count):                result, _ = self.sample(                    f"Source: {count} runs returned a value\n"                    "  empirical mean among returned values: 5; variance: 2\n",                    "Determinized: 10/10 runs returned a value\n"                    "  empirical mean among returned values: 5; variance: 1\n")                self.assertEqual(result["status"], "failed")                self.assertIsNone(result["variance_reduction_factor"])    def test_rejected_observations(self):        result, _ = self.sample(            "Source: 7/10 runs returned a value\n"            "  empirical mean among returned values: 5; variance: 2\n"            "  rejected observations: 3\n",            "Determinized: 8/10 runs returned a value\n"            "  empirical mean among returned values: 5; variance: 1\n"            "  rejected observations: 2\n")        self.assertEqual(result["status"], "completed")        self.assertEqual(result["source"]["rejected"], 3)        self.assertEqual(result["determinized"]["rejected"], 2)        self.assertEqual(result["variance_reduction_factor"], 2)    def test_all_runs_return(self):        result, output = self.sample(            "Source: 10/10 runs returned a value\n"            "  empirical mean among returned values: 5; variance: 2\n",            "Determinized: 10/10 runs returned a value\n"            "  empirical mean among returned values: 5; variance: 0\n")        self.assertEqual(result["status"], "completed")        self.assertEqual(result["variance_reduction_factor"], "infinite")        self.assertIn("Variance reduction: infinite", output)        self.assertEqual(result["sampling_seconds"], 1.25)if __name__ == "__main__":    unittest.main()