| 1 | """Bounded, paired local strength audit; never reads or writes the leaderboard. |
| 2 | |
| 3 | The real pinned engine runs each game. Per-seat RNG state is isolated to match |
| 4 | the child adapter's random.seed(0), so one policy cannot change referee random |
| 5 | draws or another player's random stream. No model/network calls are made. |
| 6 | |
| 7 | .venvs/werewolf/bin/python games/werewolf/referee/compare.py --rounds 20 |
| 8 | """ |
| 9 | import argparse |
| 10 | from collections import defaultdict |
| 11 | import contextlib |
| 12 | import json |
| 13 | import random |
| 14 | import sys |
| 15 | import time |
| 16 | |
| 17 | with contextlib.redirect_stdout(sys.stderr): |
| 18 | from kaggle_environments import make |
| 19 | from kaggle_environments.envs.werewolf.werewolf import deterministic_agent, random_agent |
| 20 | from play import configuration |
| 21 | from rule_agent import RuleAgent |
| 22 | |
| 23 | |
| 24 | class Isolated: |
| 25 | def __init__(self, fn): |
| 26 | self.fn, self.state = fn, random.Random(0).getstate() |
| 27 | |
| 28 | def __call__(self, obs, config=None): |
| 29 | outer = random.getstate() |
| 30 | random.setstate(self.state) |
| 31 | try: |
| 32 | return self.fn(obs) |
| 33 | finally: |
| 34 | self.state = random.getstate() |
| 35 | random.setstate(outer) |
| 36 | |
| 37 | |
| 38 | def main(): |
| 39 | parser = argparse.ArgumentParser() |
| 40 | parser.add_argument('--rounds', type=int, default=20, choices=range(1, 101), metavar='1..100') |
| 41 | parser.add_argument('--policy', choices=['both', 'old', 'evidence'], default='both') |
| 42 | parser.add_argument('--opponent', choices=['all', 'deterministic', 'random', 'evidence'], default='all') |
| 43 | args = parser.parse_args() |
| 44 | started = time.monotonic() |
| 45 | for opponent, fn in [('deterministic', deterministic_agent), ('random', random_agent), ('evidence', None)]: |
| 46 | if args.opponent not in ('all', opponent): |
| 47 | continue |
| 48 | for name, factory in [('old', lambda: Isolated(deterministic_agent)), ('evidence', RuleAgent)]: |
| 49 | if args.policy not in ('both', name): |
| 50 | continue |
| 51 | by = defaultdict(list) |
| 52 | for repeat in range(args.rounds): |
| 53 | for mine in range(7): |
| 54 | seed = repeat + 900 |
| 55 | random.seed(seed) |
| 56 | conf = configuration(seed, mine, rng=random.Random(9000 + repeat * 7 + mine)) |
| 57 | env = make('werewolf', configuration=conf) |
| 58 | players = [Isolated(fn) if fn else RuleAgent() for _ in range(7)] |
| 59 | players[mine] = factory() |
| 60 | env.run(players) |
| 61 | if not env.done or env.info['GAME_END']['terminated_with_agent_error']: |
| 62 | raise RuntimeError(f'Invalid game: opponent={opponent}, policy={name}, round={repeat}, seat={mine}') |
| 63 | by[conf['agents'][mine]['role']].append(int((env.state[mine].reward or 0) > 0)) |
| 64 | print(json.dumps({'opponent': opponent, 'policy': name, 'wins': sum(map(sum, by.values())), |
| 65 | 'games': sum(map(len, by.values())), |
| 66 | 'roles': {r: {'wins': sum(v), 'games': len(v)} for r, v in by.items()}}), flush=True) |
| 67 | print(json.dumps({'elapsed_seconds': round(time.monotonic() - started, 3)})) |
| 68 | |
| 69 | |
| 70 | if __name__ == '__main__': |
| 71 | main() |
