diff --git a/games/imagegame/master.py b/games/imagegame/master.py index 5c62d9aeac87333a7c992e5b158ab5e52469688e..870734d439a1999db7c249f2d06733000835a8b9 100644 --- a/games/imagegame/master.py +++ b/games/imagegame/master.py @@ -1,7 +1,7 @@ from typing import List, Tuple, Dict from clemgame import metrics -from clemgame.clemgame import GameMaster, GameBenchmark +from clemgame.clemgame import GameMaster, GameBenchmark, GameScorer from games.imagegame.game import ImageGame from games.imagegame.evaluator import evaluate, calculate_flipped_pixels from clemgame import get_logger @@ -10,6 +10,10 @@ import math GAME_NAME = "imagegame" +PLAYER_B_PATTERN = r'^\n*([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n*$' +PLAYER_A_PATTERN = r'^Instruction:\s*(.+)\n*(.+)*$' +TERMINATE_PATTERN = r'^Instruction:\s*(DONE|Done|done)' + logger = get_logger(__name__) @@ -24,9 +28,7 @@ class ImageGameMaster(GameMaster): self.parsed_request_count = 0 self.violated_request_count = 0 self.aborted_ratio = 0 - self.player_b_pattern = r'^\n*([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n([A-Zâ–¢]\s){4}[A-Zâ–¢]\n*$' - self.player_a_pattern = r'^Instruction:\s*(.+)\n*(.+)*$' - self.terminate_pattern = r'^Instruction:\s*(DONE|Done|done)' + self.turn_request_stats = {} def get_description(self) -> str: @@ -83,7 +85,7 @@ class ImageGameMaster(GameMaster): self.game.given_instruction.add_system_message(player_1_response_text) # check if it reached the end on 1 side - match = re.match(self.terminate_pattern, player_1_response_text) + match = re.match(TERMINATE_PATTERN, player_1_response_text) if match: self.parsed_request_count += 1 self.turn_request_stats[self.game.current_turn]['parsed_count'] += 1 @@ -148,7 +150,7 @@ class ImageGameMaster(GameMaster): self.request_count += 1 # check if Player 2 message has the required format: grid - match = re.compile(self.player_b_pattern).match(player_2_response_text) + match = re.compile(PLAYER_B_PATTERN).match(player_2_response_text) if match: self.parsed_request_count += 1 self.turn_request_stats[self.game.current_turn]['parsed_count'] += 1 @@ -171,6 +173,13 @@ class ImageGameMaster(GameMaster): self.game.current_turn += 1 + +class ImageGameScorer(GameScorer): + + def __init__(self, experiment: Dict, game_instance: Dict): + super().__init__(GAME_NAME, experiment, game_instance) + self.target_grid = game_instance["target_grid"] + def compute_scores(self, episode_interactions: Dict) -> None: precision, recall, f1 = 0, 0, 0 @@ -200,7 +209,7 @@ class ImageGameMaster(GameMaster): player_1_message = turn[1]['action']['content'] # Player generates "DONE" - match = re.match(self.terminate_pattern, player_1_message) + match = re.match(TERMINATE_PATTERN, player_1_message) if match: break @@ -238,7 +247,7 @@ class ImageGameMaster(GameMaster): episode_request_count += 1 # check Player 2 message if it matches the instruction => grid - match = re.compile(self.player_b_pattern).match(player_2_message) + match = re.compile(PLAYER_B_PATTERN).match(player_2_message) if match: turn_parsed_request_count += 1 episode_parsed_request_count += 1 @@ -251,7 +260,7 @@ class ImageGameMaster(GameMaster): # calculate player-specific and turn-specific metrics try: - precision, recall, f1 = evaluate(self.game.target_grid, player_2_message) + precision, recall, f1 = evaluate(self.target_grid, player_2_message) except: pass @@ -359,10 +368,6 @@ class ImageGameMaster(GameMaster): self.log_episode_score(metrics.METRIC_REQUEST_SUCCESS, request_success_ratio) - def _get_recorded_turns(self, records: Dict) -> List[int]: - return list(range(len(records["turns"]))) - - class ImageGameBenchmark(GameBenchmark): def __init__(self): @@ -372,4 +377,7 @@ class ImageGameBenchmark(GameBenchmark): return "Image Game simulation to generate referring expressions and fill a grid accordingly" def create_game_master(self, experiment: Dict, player_backends: List[str]) -> GameMaster: - return ImageGameMaster(experiment, player_backends) \ No newline at end of file + return ImageGameMaster(experiment, player_backends) + + def create_game_scorer(self, experiment: Dict, game_instance: Dict) -> GameScorer: + return ImageGameScorer(experiment, game_instance) \ No newline at end of file