#!/usr/bin/env python3 """Tests of the judge, the format checker and the starting codes. Run: python3 -I checks/test_judge.py""" import json import os import shutil import subprocess import sys import tempfile from fractions import Fraction from itertools import combinations from pathlib import Path PKG = Path(__file__).resolve().parent.parent JUDGE, REPO = PKG / "checks" / "judge", PKG sys.dont_write_bytecode = True sys.path.insert(0, str(JUDGE)) from codes import INSTANCES, RECORD, START # noqa: E402 def run(script, *args, cwd, route_dir="routes/x"): # as the check runs it (python3 checks/judge/check.py), without the environment's PYTHON* variables or user site env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "SS_ROUTE_DIR": route_dir} p = subprocess.run([sys.executable, "-E", "-s", "-B", str(script), *map(str, args)], cwd=cwd, capture_output=True, text=True, env=env, timeout=120) return p.returncode, p.stdout, p.stderr def is_code(words): """Independent of the judge: every pair of words, every pair of deleted positions in each.""" n = len(words[0]) sub = lambda w, i, j: w[:i] + w[i + 1:j] + w[j + 1:] D = [{sub(w, i, j) for i, j in combinations(range(n), 2)} for w in words] return len(set(words)) == len(words) and all(D[a].isdisjoint(D[b]) for a, b in combinations(range(len(words)), 2)) def start(k): return json.loads((REPO / "constructions" / f"{k}.json").read_text()) def write(folder, files): for name, content in files.items(): path = folder / name path.parent.mkdir(parents=True, exist_ok=True) if isinstance(content, bytes): path.write_bytes(content) else: path.write_text(content if isinstance(content, str) else json.dumps(content)) def check(files, judge=JUDGE, route_dir="routes/x", main=None): """Runs verify on a route folder holding these files (and main's constructions/ when given); returns (exit code, stdout, stderr, metrics or None).""" with tempfile.TemporaryDirectory() as tmp: folder = Path(tmp, *route_dir.split("/")) folder.mkdir(parents=True) write(folder, files) if main: write(Path(tmp, "constructions"), main) code, out, err = run(judge / "check.py", folder, cwd=tmp, route_dir=route_dir) metrics = Path(tmp, "metrics.json") return code, out, err, json.loads(metrics.read_text()) if metrics.exists() else None def judge_with(records, tmp): """A copy of the judge whose records are changed: {instance: size}.""" out = Path(tmp, "judge") shutil.copytree(JUDGE, out) table = json.loads((JUDGE / "records.json").read_text()) for k, v in records.items(): table[k]["value"] = v (out / "records.json").write_text(json.dumps(table)) return out def test_starts(): assert RECORD == {"n16": 220, "n17": 349, "n18": 567} for k in INSTANCES: c = start(k) assert f"n{c['n']}" == k and len(c["words"]) == START[k] and all(len(w) == c["n"] for w in c["words"]), k assert is_code(c["words"]), k code, out, err, m = check({}, route_dir="routes", main={f"{k}.json": start(k) for k in INSTANCES}) assert code == 0 and not err, out + err s = (Fraction(220, 220) + Fraction(316, 349) + Fraction(525, 567)) / 3 assert m == {"score": float(s), "score_exact": "26680/28269", "subtasks": {"n16": 220, "n17": 316, "n18": 525}, "new_records": []}, m assert str(s) == "26680/28269" assert "n16: 220 words (constructions/n16.json), +0 against the record\n" in out, out assert out.strip().endswith("score 0.943790; no new record"), out def test_score_and_new_records(): small = {"n": 17, "words": start("n17")["words"][:10]} with tempfile.TemporaryDirectory() as tmp: code, out, err, m = check({"a.json": start("n18"), "small.json": small}, judge=judge_with({"n18": 500}, tmp)) assert code == 0 and not err, out + err s = (1 + Fraction(316, 349) + Fraction(525, 500)) / 3 # n16 and n17 count as their starts assert m["score_exact"] == str(s) and m["subtasks"] == {"n17": 10, "n18": 525} and m["new_records"] == ["n18"], m assert "n18: 525 words (x/a.json), +25 against the record; a new record\n" in out, out assert out.strip().endswith("; new records: n18"), out def test_codes_that_share_a_descendant_fail(): clash = {"n": 16, "words": ["0" * 16, "0" * 15 + "1"]} # deleting the 1 and a 0 gives fourteen 0s, as from 0...0 code, out, err, m = check({"clash.json": clash}) assert code == 1 and "share the descendant 00000000000000" in out and not err and m is None, out worse = {"n": 18, "words": start("n18")["words"] + ["1" * 17 + "0"]} assert not is_code(worse["words"]) code, out, err, m = check({"worse.json": worse}) assert code == 1 and "not a code correcting two deletions" in out, out def test_invalid_candidates_fail_cleanly(): good = start("n16") w = good["words"] cases = { "length": ({**good, "words": w[:-1] + [w[-1] + "0"]}, "16 binary digits"), "digit": ({**good, "words": w[:-1] + ["2" + w[-1][1:]]}, "16 binary digits"), "number": ({**good, "words": w[:-1] + [1010]}, "16 binary digits"), "repeat": ({**good, "words": w + [w[0]]}, "distinct"), "empty": ({**good, "words": []}, "nonempty list"), "string": ({**good, "words": "0101"}, "nonempty list"), "size": ({**good, "n": 15}, '"n" must be one of'), "bool": ({**good, "n": True}, '"n" must be one of'), } for name, (candidate, message) in cases.items(): code, out, err, m = check({"ok.json": good, f"{name}.json": candidate}) assert code == 1 and message in out and not err, (name, out, err) assert m["subtasks"] == {"n16": 220}, (name, m) def test_malformed_files_fail_cleanly(): for content in ("{not json", "", b"\xff\xfe{"): code, out, err, m = check({"bad.json": content}) assert code == 1 and "unreadable" in out and not err and m is None, (out, err) assert out.strip().splitlines()[-1] == "no valid candidates: 1 failed", out def test_no_candidates(): for files in ({}, {"notes.json": [1, 2]}, {"README.md": "# x"}): code, out, err, m = check(files) assert code == 0 and out.strip() == "no candidates to check" and m is None, out with tempfile.TemporaryDirectory() as tmp: code, out, err = run(JUDGE / "check.py", Path(tmp, "missing"), cwd=tmp) assert code == 0 and out.strip() == "no candidates to check", out def test_improves(): with tempfile.TemporaryDirectory() as tmp: main = Path(tmp, "main") shutil.copytree(REPO / "constructions", main / "constructions") route = Path(tmp, "route") write(route, {"a.json": start("n17")}) code, out, _ = run(JUDGE / "improves.py", route, main, cwd=main) assert code == 1 and "n17: 316 words (route/a.json) against the default branch's 316, +0; the record is 349" in out, out assert out.strip().endswith("no improvement on the default branch"), out write(main / "constructions", {"n17.json": {"n": 17, "words": start("n17")["words"][:300]}}) code, out, _ = run(JUDGE / "improves.py", route, main, cwd=main) assert code == 0 and "+16: improves" in out and out.strip().endswith("improves on the default branch: n17; no new record"), out code, out, _ = run(judge_with({"n17": 315}, tmp) / "improves.py", route, main, cwd=main) assert code == 0 and "a new record, above 315" in out and out.strip().endswith("new records: n17"), out code, out, _ = run(JUDGE / "improves.py", Path(tmp, "empty"), main, cwd=main) assert code == 1 and out.startswith("no code"), out code, out, _ = run(JUDGE / "improves.py", main / "routes", main, cwd=main, route_dir="routes") assert code == 0 and out.strip() == "the default branch holds n16: 220, n17: 300, n18: 525; no new record", out def test_check_format(): tool = REPO / "tools" / "check_format.py" code, out, err = run(tool, REPO / "constructions", cwd=REPO) assert code == 0 and out.strip().endswith("3 candidates, 0 files with format errors"), out + err bad = {"n": 17, "words": ["0" * 17, "0" * 16, "0" * 17]} with tempfile.TemporaryDirectory() as tmp: write(Path(tmp), {"bad.json": bad, "twice.json": {"n": 16, "words": ["0" * 16] * 2}, "broken.json": "{", "other.json": "[]"}) code, out, err = run(tool, tmp, cwd=tmp) assert code == 1 and "1 words are not strings of 17 binary digits" in out and "1 words are repeated" in out, out assert "not valid JSON" in out and "ignored" in out and out.strip().endswith("2 candidates, 3 files with format errors"), out if __name__ == "__main__": failed = 0 for name, test in list(globals().items()): if name.startswith("test_"): try: test() print(f"ok {name}") except Exception as exc: failed += 1 print(f"FAIL {name}: {exc!r}") sys.exit(1 if failed else 0)