"""Exact result certificates, independent kernel replay, and the Storm adapter without Storm;test_results_storm.py runs Storm itself."""from fractions import Fractionimport jsonimport importlib.utilfrom types import SimpleNamespacefrom unittest.mock import patchfrom pathlib import Pathimport subprocessimport sysimport tempfileimport unittestfrom test_export import ROOT, LEAN, BIN, solve_exportCASES = [ ("let p = bernoulli[E](0.5) in discrete[E](p/2,0.25,*)", "source", Fraction(5, 4)), ("let p = bernoulli[E](0.5) in discrete[E](p/2,0.25,*)", "determinized", Fraction(5, 4)), ("let ps = if flip(0.5) then [] else 0::[] in discrete_list[E](ps)", "source", Fraction(1, 2)), ("if flip(0.25) then 8 else -4", "source", Fraction(-1)), ("let _ = observe(flip(0.5)) in 3", "source", Fraction(3, 2)), ("let f = rec f x => if flip(0.5) then -3 else f x in f 0", "source", Fraction(-3)), ("discrete[G](0,0.25,0,0.75)", "source", Fraction(5, 2)), ("uniform[E](0,3)", "determinized", Fraction(3, 2)), ("-7/3", "source", Fraction(-7, 3)), ("let _ = observe(false) in 3", "source", Fraction(0)), ("if flip(0.25) then 1 else 0", "source", Fraction(1, 4)),]def generate(directory, program, subject="source", *options, additive=False): source = directory / "input.det" source.write_text(program) prefix = directory / "model" mode = [] if additive else ["--no-additive"] result = subprocess.run([BIN, "--check", "--result", prefix, "--subject", subject, *mode, *options, source], text=True, capture_output=True, timeout=60) return result, prefixdef kernel(path): return subprocess.run(["lake", "env", "lean", path], cwd=LEAN, text=True, capture_output=True, timeout=180)def storm_adapter(): """tools/storm.py as a module.""" spec = importlib.util.spec_from_file_location("storm_adapter", ROOT / "tools/storm.py") adapter = importlib.util.module_from_spec(spec) spec.loader.exec_module(adapter) return adapterclass ResultTests(unittest.TestCase): def test_paper_noisy_iteration(self): program = (ROOT / "examples/paper/noisy-iteration.det").read_text() with tempfile.TemporaryDirectory() as tmp: result, prefix = generate(Path(tmp), program, "determinized") self.assertEqual(result.returncode, 0, result.stderr) data = json.loads(Path(str(prefix) + ".result.json").read_text()) self.assertEqual(data["return_mass"], "1") self.assertEqual(data["conditional_mean"], "1/3") self.assertEqual(data["second_moment"], "1/3") self.assertEqual(data["conditional_variance"], "2/9") self.assertEqual(solve_export(prefix), Fraction(1, 3)) checked = kernel(Path(str(prefix) + ".result.lean")) self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) for forbidden in ("sorryAx", "ofReduceBool", "trustCompiler"): self.assertNotIn(forbidden, checked.stdout + checked.stderr) def test_transfer_hypotheses(self): note = "The source has the same expectation if it is domain-safe" program = CASES[0][0] for subject in ("determinized", "source"): with self.subTest(subject=subject), tempfile.TemporaryDirectory() as tmp: result, _ = generate(Path(tmp), program, subject) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(note in result.stdout, subject == "determinized", result.stdout) def test_exact_ground_truth(self): for program, subject, expected in CASES: with self.subTest(program=program), tempfile.TemporaryDirectory() as tmp: result, prefix = generate(Path(tmp), program, subject) self.assertEqual(result.returncode, 0, result.stderr) data = json.loads(Path(str(prefix) + ".result.json").read_text()) self.assertEqual(Fraction(data["answer"]), expected) self.assertEqual(solve_export(prefix), expected) self.assertEqual(data["subject"], subject) self.assertFalse(data["kernel_checked"]) self.assertEqual(data["certificate_status"], "generated") self.assertEqual(data["termination_statistics_scope"], "graph") self.assertGreaterEqual(data["rank_bound"], 0) def test_kernel_results_and_tampering(self): for program, subject, _ in CASES[:8]: with self.subTest(program=program), tempfile.TemporaryDirectory() as tmp: result, prefix = generate(Path(tmp), program, subject) self.assertEqual(result.returncode, 0, result.stderr) certificate = Path(str(prefix) + ".result.lean") checked = kernel(certificate) self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) self.assertIn("expectedReward", checked.stdout) for forbidden in ("sorryAx", "ofReduceBool", "trustCompiler"): self.assertNotIn(forbidden, checked.stdout) text = certificate.read_text() start = text.index(" values :=", text.index("def result :")) end = text.index("\n\n", start) certificate.write_text(text[:start] + " values := fun _ _ => 12345" + text[end:]) self.assertNotEqual(kernel(certificate).returncode, 0) start = text.index(" rank :=", text.index("def result :")) end = text.index("\n", start) certificate.write_text(text[:start] + " rank := fun _ => 0" + text[end:]) self.assertNotEqual(kernel(certificate).returncode, 0) def test_divergence_and_conditional_moments(self): cases = [ ("let f = rec f x => f x in f 0", ("0", "0", "0", None, None)), ("let f = rec f x => f x in if flip(0.5) then f 0 else if flip(0.5) then -2 else 4", ("1/2", "1/2", "5", "1", "9")), ("let _ = observe(flip(0.5)) in if flip(0.5) then -2 else 4", ("1/2", "1/2", "5", "1", "9")), ] for program, expected in cases: with self.subTest(program=program), tempfile.TemporaryDirectory() as tmp: result, prefix = generate(Path(tmp), program) self.assertEqual(result.returncode, 0, result.stderr) data = json.loads(Path(str(prefix) + ".result.json").read_text()) self.assertEqual(tuple(data[key] for key in ("return_mass", "answer", "second_moment", "conditional_mean", "conditional_variance")), expected) checked = kernel(Path(str(prefix) + ".result.lean")) self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) def test_termination_probabilities(self): cases = [ ("let f = rec f x => f x in f 0", ("0", "0", "1")), ("let _ = observe(false) in 3", ("0", "1", "0")), ("let f = rec f x => f x in if flip(0.5) then f 0 else let _ = observe(flip(0.5)) in 1", ("1/4", "1/4", "1/2")), ] for program, expected in cases: with self.subTest(program=program), tempfile.TemporaryDirectory() as tmp: result, prefix = generate(Path(tmp), program) self.assertEqual(result.returncode, 0, result.stderr) data = json.loads(Path(str(prefix) + ".result.json").read_text()) self.assertEqual(tuple(data[key] for key in ("return_mass", "rejection_probability", "divergence_probability")), expected) checked = kernel(Path(str(prefix) + ".result.lean")) self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) def test_failure_preserves_outputs(self): for program, options, error in [ ("1/0", [], "division"), ("if flip(0.5) then 2 else 3", ["--max-result-states", "1"], "state limit"), ("uniform[G](0,1)", [], "unsupported"), ]: with self.subTest(program=program), tempfile.TemporaryDirectory() as tmp: directory = Path(tmp) old = directory / "model.result.lean" old.write_text("existing certificate") result, prefix = generate(directory, program, "source", *options) self.assertNotEqual(result.returncode, 0) self.assertIn(error, result.stderr.lower()) self.assertEqual(old.read_text(), "existing certificate") self.assertFalse(Path(str(prefix) + ".result.json").exists()) self.assertFalse(Path(str(prefix) + ".tra").exists()) def test_storm_failure_reports(self): adapter = storm_adapter() for failure in ("timeout", "storm", "kernel", "disagreement"): with self.subTest(failure=failure), tempfile.TemporaryDirectory() as tmp: prefix = Path(tmp) / "model" args = SimpleNamespace(prefix=prefix, subject="source", timeout=1, binary=BIN, compare=True, additive=False, file=Path(tmp) / "input.det") calls = [] def command(argv, **kwargs): calls.append(argv) if failure == "timeout": raise subprocess.TimeoutExpired(argv, 1) stage = len(calls) if (stage == 2 and failure == "storm") or (stage == 3 and failure == "kernel"): return subprocess.CompletedProcess(argv, 1, failure + " rejected", "") if stage == 2: Path(str(prefix) + ".storm-values.json").write_text( '{"storm_version": "test", "storm_build_type": "test"}') if stage == 4: Path(str(prefix) + ".result.json").write_text('{"answer": "1/3"}') output = "\n".join(f"'{name}' depends on axioms: [propext, Classical.choice, Quot.sound]" for name in ("outputStatistics", "terminationProbabilities", "conditionalVariance", "reportedStatistics", "printedStatistics", "printedConditionalMoments")) return subprocess.CompletedProcess(argv, 0, output, "") with patch.object(adapter.importlib.metadata, "version", return_value="test"), \ patch.object(adapter, "run_command", side_effect=command), \ patch.object(adapter, "certificate_text", return_value=("proof", { "mass": Fraction(1), "first": Fraction(1,2), "second": Fraction(1,4), "rejection": Fraction(0)})), patch("builtins.print"): self.assertEqual(adapter.run(args), 1) report = json.loads(Path(str(prefix) + ".storm.json").read_text()) self.assertEqual(report["status"], "failed") self.assertEqual(len(calls), {"timeout": 1, "storm": 2, "kernel": 3, "disagreement": 4}[failure]) if failure == "disagreement": self.assertIn("disagrees", report["error"]) self.assertTrue(report["kernel_checked"]) else: self.assertNotIn("kernel_checked", report) def test_certificate_axioms(self): adapter = storm_adapter() names = ("outputStatistics", "terminationProbabilities", "conditionalVariance", "reportedStatistics", "printedStatistics", "printedConditionalMoments") lines = [f"'{name}' depends on axioms: [propext, Classical.choice, Quot.sound]" for name in names] output = "\n".join(lines) self.assertEqual(len(adapter.checked_axioms(output)), 6) self.assertEqual(len(adapter.checked_axioms("\n".join(lines[:-1]), conditional=False)), 5) for invalid in ("", output.replace("Quot.sound", "sorryAx"), output.replace("Quot.sound", "Lean.ofReduceBool"), output.replace("Quot.sound", "unprovedClaim"), "\n".join(lines[:-1]), "\n".join(lines[:-2] + lines[-1:])): with self.assertRaises(ValueError): adapter.checked_axioms(invalid) def test_storm_timeout_stops_children(self): adapter = storm_adapter() with tempfile.TemporaryDirectory() as tmp: pidfile = Path(tmp) / "child.pid" script = ("import subprocess, sys, time; " "from pathlib import Path; " "p=subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)']); " f"Path({str(pidfile)!r}).write_text(str(p.pid)); time.sleep(30)") with self.assertRaises(subprocess.TimeoutExpired): adapter.run_command([sys.executable, "-c", script], cwd=ROOT, timeout=1) child = int(pidfile.read_text()) state = subprocess.run(["ps", "-p", str(child), "-o", "stat="], text=True, capture_output=True).stdout.strip() self.assertTrue(not state or state.startswith("Z"), state) def test_storm_certificate_validation(self): adapter = storm_adapter() with tempfile.TemporaryDirectory() as tmp: result, prefix = generate(Path(tmp), "if flip(0.5) then 1 else 2") self.assertEqual(result.returncode, 0, result.stderr) size, edges, labels, _ = adapter.read_model(prefix) dead, rank, following = adapter.boundary(size, edges, labels) valid = {"dead": [i in dead for i in range(size-1)], "rank": rank, "next": following, "values": {name: ["0"]*(size-1) for name in ("mass", "first", "second", "rejection")}} for key, invalid in [("dead", [False]), ("rank", [-1]*(size-1)), ("next", [size-1]*(size-1)), ("values", {})]: with self.subTest(key=key), self.assertRaises(ValueError): adapter.certificate_text(prefix, {**valid, key: invalid}) for bad in ("inf", "nan", "1/0"): with self.subTest(bad=bad), self.assertRaises((ValueError, ZeroDivisionError)): adapter.certificate_text(prefix, {**valid, "values": { **valid["values"], "mass": [bad]*(size-1)}}) certificate, _ = adapter.certificate_text(prefix, valid) path = Path(tmp) / "wrong.lean" path.write_text(certificate) self.assertNotEqual(kernel(path).returncode, 0)if __name__ == "__main__": unittest.main()