#!/usr/bin/env python3 """Tests of the judge, the format checker and the starting snakes. Run: python3 -I checks/test_judge.py""" import json import os import shutil import subprocess import sys import tempfile from fractions import Fraction from pathlib import Path PKG = Path(__file__).resolve().parent.parent JUDGE, REPO = PKG / "checks" / "judge", PKG sys.dont_write_bytecode = True sys.path.insert(0, str(JUDGE)) from snake import INSTANCES, RECORD # noqa: E402 def run(script, *args, cwd, route_dir="routes/x"): # as the check runs it (python3 checks/judge/check.py), without the environment's PYTHON* variables or user site env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "SS_ROUTE_DIR": route_dir} p = subprocess.run([sys.executable, "-E", "-s", "-B", str(script), *map(str, args)], cwd=cwd, capture_output=True, text=True, env=env, timeout=120) return p.returncode, p.stdout, p.stderr def length(c): """Independent of the judge: the path's vertices as bit strings, then every pair at Hamming distance 1 must be consecutive on it. Returns the number of edges, or None when the path is not a snake.""" d = c["d"] path = [0] for ch in c["transitions"]: path.append(path[-1] ^ (1 << int(ch, 36))) if len(set(path)) != len(path): return None index = {v: i for i, v in enumerate(path)} for v, i in index.items(): for b in range(d): j = index.get(v ^ (1 << b)) if j is not None and abs(i - j) != 1: return None return len(path) - 1 def start(k): return json.loads((REPO / "constructions" / f"{k}.json").read_text()) def write(folder, files): for name, content in files.items(): path = folder / name path.parent.mkdir(parents=True, exist_ok=True) if isinstance(content, bytes): path.write_bytes(content) else: path.write_text(content if isinstance(content, str) else json.dumps(content)) def check(files, judge=JUDGE, route_dir="routes/x", main=None): """Runs verify on a route folder holding these files (and main's constructions/ when given); returns (exit code, stdout, stderr, metrics or None).""" with tempfile.TemporaryDirectory() as tmp: folder = Path(tmp, *route_dir.split("/")) folder.mkdir(parents=True) write(folder, files) if main: write(Path(tmp, "constructions"), main) code, out, err = run(judge / "check.py", folder, cwd=tmp, route_dir=route_dir) metrics = Path(tmp, "metrics.json") return code, out, err, json.loads(metrics.read_text()) if metrics.exists() else None def judge_with(records, tmp): """A copy of the judge whose records are changed: {instance: length}.""" out = Path(tmp, "judge") shutil.copytree(JUDGE, out) table = json.loads((JUDGE / "records.json").read_text()) for k, v in records.items(): table[k]["value"] = v (out / "records.json").write_text(json.dumps(table)) return out def test_starts_are_the_records(): assert RECORD == {"d13": 2946, "d14": 5850, "d15": 11673, "d16": 23318} for k in INSTANCES: c = start(k) assert f"d{c['d']}" == k and length(c) == RECORD[k], k code, out, err, m = check({}, route_dir="routes", main={f"{k}.json": start(k) for k in INSTANCES}) assert code == 0 and not err, out + err assert m == {"score": 1.0, "score_exact": "1", "subtasks": RECORD, "new_records": []}, m assert "d16: 23318 edges (constructions/d16.json), +0 against the record\n" in out, out assert out.strip().endswith("score 1.000000000; no new record"), out def test_score_and_new_records(): short = {**start("d15"), "transitions": start("d15")["transitions"][:1000]} # a piece of a snake is a snake with tempfile.TemporaryDirectory() as tmp: code, out, err, m = check({"a.json": start("d13"), "short.json": short}, judge=judge_with({"d13": 2940}, tmp)) assert code == 0 and not err, out + err s = (Fraction(2946, 2940) + 3) / 4 # the short d15 counts as the record assert m["score_exact"] == str(s) and m["subtasks"] == {"d13": 2946, "d15": 1000} and m["new_records"] == ["d13"], m assert "d13: 2946 edges (x/a.json), +6 against the record; a new record\n" in out and "d15: 1000 edges (x/short.json), -10673" in out, out def test_paths_that_are_not_snakes_fail(): for t, message in (("00", "returns to vertex 0 at step 2"), ("010", "steps 0 and 3 are adjacent"), ([0, 1, 0], "steps 0 and 3 are adjacent")): code, out, err, m = check({"bad.json": {"d": 13, "transitions": t}}) assert code == 1 and "not a snake" in out and message in out and not err and m is None, (t, out) def test_both_forms_and_whitespace(): t = start("d13")["transitions"] spaced = " ".join(t[i:i + 10] for i in range(0, len(t), 10)).upper() as_list = [int(ch, 36) for ch in t] code, out, err, m = check({"a.json": {"d": 13, "transitions": spaced}, "b.json": {"d": 13, "transitions": as_list}}) assert code == 0 and out.count("d13: 2946 edges") == 2 and m["subtasks"] == {"d13": 2946}, out def test_invalid_candidates_fail_cleanly(): good = start("d14") cases = { "dimension": ({**good, "d": 12}, '"d" must be one of'), "bool": ({**good, "d": True}, '"d" must be one of'), "step": ({"d": 13, "transitions": "0d"}, "coordinate from 0 to 12"), "sign": ({"d": 13, "transitions": "0-1"}, "coordinate from 0 to 12"), "float": ({"d": 13, "transitions": [0, 1.0]}, "coordinate from 0 to 12"), "true": ({"d": 13, "transitions": [0, True]}, "coordinate from 0 to 12"), "empty": ({"d": 13, "transitions": " "}, "nonempty string or list"), "long": ({"d": 13, "transitions": "0" * 8192}, "fewer than 8192 steps"), "object": ({"d": 13, "transitions": {"0": 1}}, "nonempty string or list"), } for name, (candidate, message) in cases.items(): code, out, err, m = check({"ok.json": good, f"{name}.json": candidate}) assert code == 1 and message in out and not err, (name, out, err) assert m["subtasks"] == {"d14": 5850}, (name, m) def test_malformed_files_fail_cleanly(): for content in ("{not json", "", b"\xff\xfe{"): code, out, err, m = check({"bad.json": content}) assert code == 1 and "unreadable" in out and not err and m is None, (out, err) assert out.strip().splitlines()[-1] == "no valid candidates: 1 failed", out def test_no_candidates(): for files in ({}, {"notes.json": [1, 2]}, {"README.md": "# x"}): code, out, err, m = check(files) assert code == 0 and out.strip() == "no candidates to check" and m is None, out with tempfile.TemporaryDirectory() as tmp: code, out, err = run(JUDGE / "check.py", Path(tmp, "missing"), cwd=tmp) assert code == 0 and out.strip() == "no candidates to check", out def test_improves(): with tempfile.TemporaryDirectory() as tmp: main = Path(tmp, "main") shutil.copytree(REPO / "constructions", main / "constructions") route = Path(tmp, "route") write(route, {"a.json": start("d13")}) code, out, _ = run(JUDGE / "improves.py", route, main, cwd=main) assert code == 1 and "d13: 2946 edges (route/a.json) against the default branch's 2946, +0; the record is 2946 (William Toner, 2026-10-11)" in out, out assert out.strip().endswith("no improvement on the default branch"), out write(main / "constructions", {"d13.json": {**start("d13"), "transitions": start("d13")["transitions"][:2900]}}) code, out, _ = run(JUDGE / "improves.py", route, main, cwd=main) assert code == 0 and "+46: improves" in out and out.strip().endswith("improves on the default branch: d13; no new record"), out code, out, _ = run(judge_with({"d13": 2940}, tmp) / "improves.py", route, main, cwd=main) assert code == 0 and "a new record, above 2940" in out and out.strip().endswith("new records: d13"), out code, out, _ = run(JUDGE / "improves.py", Path(tmp, "empty"), main, cwd=main) assert code == 1 and out.startswith("no snake"), out code, out, _ = run(JUDGE / "improves.py", main / "routes", main, cwd=main, route_dir="routes") assert code == 0 and out.strip() == "the default branch holds d13: 2900, d14: 5850, d15: 11673, d16: 23318; no new record", out def test_check_format(): tool = REPO / "tools" / "check_format.py" code, out, err = run(tool, REPO / "constructions", cwd=REPO) assert code == 0 and out.strip().endswith("4 candidates, 0 files with format errors"), out + err with tempfile.TemporaryDirectory() as tmp: write(Path(tmp), {"bad.json": {"d": 13, "transitions": "0d1x"}, "dim.json": {"d": 17, "transitions": "0"}, "broken.json": "{", "other.json": "[]"}) code, out, err = run(tool, tmp, cwd=tmp) assert code == 1 and "2 steps are not coordinates from 0 to 12" in out and '"d" must be one of' in out, out assert "not valid JSON" in out and "ignored" in out and out.strip().endswith("2 candidates, 3 files with format errors"), out if __name__ == "__main__": failed = 0 for name, test in list(globals().items()): if name.startswith("test_"): try: test() print(f"ok {name}") except Exception as exc: failed += 1 print(f"FAIL {name}: {exc!r}") sys.exit(1 if failed else 0)