From f075da93e8d9215d026ca7eeb0dfd60381928786 Mon Sep 17 00:00:00 2001 From: Ali Toosi Date: Sat, 16 Apr 2022 12:22:40 +1000 Subject: [PATCH] Game Runner - Worker --- .gitignore | 4 ++ Dockerfile | 14 ++++++ README.md | 11 +++++ app/config.py | 13 ++++++ app/game.py | 66 +++++++++++++++++++++++++++ app/logs.py | 2 + app/main.py | 85 +++++++++++++++++++++++++++++++++++ build_and_push.sh | 5 +++ public_image/Dockerfile | 16 +++++++ public_image/README.md | 1 + public_image/get_bot_names.py | 13 ++++++ public_image/requirements.txt | 1 + public_image/run_game.sh | 6 +++ public_image/run_image.sh | 1 + requirements.txt | 2 + 15 files changed, 240 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app/config.py create mode 100644 app/game.py create mode 100644 app/logs.py create mode 100644 app/main.py create mode 100755 build_and_push.sh create mode 100644 public_image/Dockerfile create mode 100644 public_image/README.md create mode 100644 public_image/get_bot_names.py create mode 100644 public_image/requirements.txt create mode 100644 public_image/run_game.sh create mode 100644 public_image/run_image.sh create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6ed91a6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.python-version +.vscode +.idea +__pycache__ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f31c264 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.9 + +RUN apt update && apt install -y vim cmake pkg-config +RUN apt install -y mesa-utils libglu1-mesa-dev freeglut3-dev mesa-common-dev +RUN apt install -y libglew-dev libglfw3-dev libglm-dev +RUN apt install -y libao-dev libmpg123-dev + +COPY ./requirements.txt /codequest/requirements.txt +COPY ./app /codequest/app + +WORKDIR /codequest +RUN pip install -r requirements.txt + +CMD ["python", "./app/main.py"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..35a4080 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +This is what happens: +``` +app/ + games/ + match_/ + info.json + results.json + bots/ + / + ... submission content ... +``` \ No newline at end of file diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..da24103 --- /dev/null +++ b/app/config.py @@ -0,0 +1,13 @@ +import os + +server_url = os.environ.get('server_url', 'http://localhost') +if server_url[-1] == '/': + server_url = server_url[:-1] + +worker_id = os.environ.get('worker_id', 1) + +wait_time_before_shutdown = 60 # wait this many minutes for server to become responsive +number_of_threads = 1 + +game_retries = 5 +game_timeout_time = 500 # seconds diff --git a/app/game.py b/app/game.py new file mode 100644 index 0000000..8dc061d --- /dev/null +++ b/app/game.py @@ -0,0 +1,66 @@ +import os +import requests as r +import shutil +import json +import subprocess +import config +from logs import log + + +def download_match_submissions(match_index, teams): + match_folder = f'games/match_{match_index}' + + if os.path.isdir(match_folder): + shutil.rmtree(match_folder) + os.makedirs(f'{match_folder}/bots') + + with open(f'{match_folder}/info.json', 'w') as f: + f.write(json.dumps({ + 'match_index': match_index, + 'teams': teams + })) + + for team in teams: + team_directory = f'{match_folder}/bots/{team["name"]}' + os.makedirs(team_directory, exist_ok=True) + team_submission = r.get(team['submission'], allow_redirects=True) + with open(f'{team_directory}/submission.zip', 'wb') as f: + f.write(team_submission.content) + + +def unzip_match_submissions(match_index, teams): + match_folder = f'games/match_{match_index}' + for team in teams: + team_directory = f'{match_folder}/bots/{team["name"]}' + shutil.unpack_archive(f'{team_directory}/submission.zip', f'{team_directory}') + os.remove(f'{team_directory}/submission.zip') + + +def run_game(match_folder, map_name, teams): + teams = [team['name'] for team in teams] + results = None + retries_left = config.game_retries + 1 + command = ' '.join(['timeout', str(config.game_timeout_time), 'codequest22', '--no-visual', '-m', map_name] + \ + [f'bots/{team_name}/'.replace(" ", r"\ ") for team_name in teams]) + log(f'Running the game for {str(teams)}') + while results is None and retries_left > 0: + retries_left -= 1 + + try: + log(command) + subprocess.run(command, shell=True, cwd=match_folder) + except Exception: + pass + + replay_file = f'{match_folder}/replay.txt' + if os.path.isfile(replay_file): + result_line = subprocess.check_output(['tail', '-1', replay_file]).strip() + try: + raw_results = json.loads(result_line) + if raw_results['type'] != 'winner': + raise Exception('Game failed somewhere') + results = {teams[i]: raw_results['score'][i] for i in range(len(teams))} + except: + log(f'Game between {str(teams)} failed or crashed. Retrying soon...') + + return results is not None, results diff --git a/app/logs.py b/app/logs.py new file mode 100644 index 0000000..ea81a24 --- /dev/null +++ b/app/logs.py @@ -0,0 +1,2 @@ +def log(msg): + print(msg) \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..4e2b177 --- /dev/null +++ b/app/main.py @@ -0,0 +1,85 @@ +import os +import shutil +import json +from time import sleep +import requests as r +import threading +import config +from logs import log +from game import download_match_submissions, unzip_match_submissions, run_game + + +def be_patient_and(func): + sleep_time = 10 + total_time_slept = 0 + func_res = func() + while not func_res: + sleep(sleep_time) + total_time_slept += sleep_time + sleep_time *= 1.5 + if total_time_slept >= config.wait_time_before_shutdown: + # Server is constantly failing us, time to die now. + return False, tuple() + func_res = func() + return func_res + + +def get_new_match(): + request_url = f'{config.server_url}/get-match' + response = r.post(request_url, json={'worker_id': config.worker_id}) + if response.status_code != 200: + log('Server returned non-200 status code. Don\'t know what to do...') + return False + data = response.json() + if not data['ok']: + log(f'Server error: {data["message"]}') + if 'shutdown' in data and data['shutdown']: + return False, tuple() + return False + + match_index = data['match_index'] + map_name = data['map_name'] + teams = data['teams'] + + return True, (match_index, map_name, teams) + + +def return_match_results(match_index, results): + log(f'Sending results of match #{match_index} back') + sent = False + retries = 5 + while not sent and retries > 0: + retries -= 1 + response = r.post(f'{config.server_url}/match-results', json={ + 'match_index': match_index, + 'results': results + }) + if response.status_code != 200 or not response.json()['ok']: + log(f'Results of match #{match_index} failed, retrying...') + else: + sent = True + log(f'Results of match #{match_index} sent back') + + +def thread_entrypoint(thread_id): + should_continue, data = be_patient_and(get_new_match) + while should_continue: + match_index, map_name, teams = data + download_match_submissions(match_index, teams) + unzip_match_submissions(match_index, teams) + successful_run, results = run_game(f'games/match_{match_index}', map_name, teams) + if not successful_run: + results = {team['name']: 0 for team in teams} + return_match_results(match_index, results) + should_continue, data = be_patient_and(get_new_match) + + +threads = [] +for i in range(config.number_of_threads): + threads.append(threading.Thread(target=thread_entrypoint, args=(i,))) + threads[-1].start() + +for thread in threads: + thread.join() + +exit() diff --git a/build_and_push.sh b/build_and_push.sh new file mode 100755 index 0000000..0c16796 --- /dev/null +++ b/build_and_push.sh @@ -0,0 +1,5 @@ +docker build -t arkhoshghalb/codequest:worker . && docker push arkhoshghalb/codequest:worker + +cd public_image +docker build -t arkhoshghalb/codequest:public_runner . && docker push arkhoshghalb/codequest:public_runner +cd .. \ No newline at end of file diff --git a/public_image/Dockerfile b/public_image/Dockerfile new file mode 100644 index 0000000..82ca931 --- /dev/null +++ b/public_image/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9 + +RUN apt update && apt install -y vim cmake pkg-config +RUN apt install -y mesa-utils libglu1-mesa-dev freeglut3-dev mesa-common-dev +RUN apt install -y libglew-dev libglfw3-dev libglm-dev +RUN apt install -y libao-dev libmpg123-dev + +COPY ./requirements.txt /codequest/requirements.txt +COPY ./run_game.sh /codequest/run_game.sh +COPY ./get_bot_names.py /codequest/get_bot_names.py + +WORKDIR /codequest +RUN chmod +x run_game.sh +RUN pip install -r requirements.txt + +CMD ./run_game.sh \ No newline at end of file diff --git a/public_image/README.md b/public_image/README.md new file mode 100644 index 0000000..387d85a --- /dev/null +++ b/public_image/README.md @@ -0,0 +1 @@ +The directory from which this image is run should have exactly 4 folders and each folder with a `main.py` at the root. \ No newline at end of file diff --git a/public_image/get_bot_names.py b/public_image/get_bot_names.py new file mode 100644 index 0000000..4b5a288 --- /dev/null +++ b/public_image/get_bot_names.py @@ -0,0 +1,13 @@ +from glob import glob + + +dirs = glob('bots/*/') +if len(dirs) != 4: + with open('command.sh', 'w') as f: + f.write("echo 'There should be exactly 4 folders in the directory'") + exit() + +dirs = [dir[:-1] if dir[-1] == '/' else dir for dir in dirs] +with open('command.sh', 'w') as f: + args = ' '.join(dirs) + f.write(f'codequest22 --no-visual {args}') \ No newline at end of file diff --git a/public_image/requirements.txt b/public_image/requirements.txt new file mode 100644 index 0000000..4ad5f17 --- /dev/null +++ b/public_image/requirements.txt @@ -0,0 +1 @@ +oaisudoiajsdoiausd==0.0.6 \ No newline at end of file diff --git a/public_image/run_game.sh b/public_image/run_game.sh new file mode 100644 index 0000000..ee7ab1b --- /dev/null +++ b/public_image/run_game.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +python get_bot_names.py +chmod +x command.sh +./command.sh +cp bots/replay.* ./ \ No newline at end of file diff --git a/public_image/run_image.sh b/public_image/run_image.sh new file mode 100644 index 0000000..4be3efb --- /dev/null +++ b/public_image/run_image.sh @@ -0,0 +1 @@ +docker run --rm -it --name codequest_runner -v "$(pwd)":/codequest/bots arkhoshghalb/codequest:public_runner \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4534083 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +oaisudoiajsdoiausd==0.0.6 \ No newline at end of file