From 884feab8d4be13d16ac6492e27a55603737e5ce4 Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 5 Jul 2023 18:12:47 +0200 Subject: [PATCH 01/26] Add doc strings and precommit --- .pre-commit-config.yaml | 42 ++++++++++++ minetester/__init__.py | 4 +- minetester/minetest_env.py | 80 +++++++++++++++++++--- minetester/proto/__init__.py | 1 + minetester/scripts/__init__.py | 1 + minetester/scripts/test_loop.py | 4 +- minetester/scripts/test_loop_parallel.py | 8 ++- minetester/utils.py | 86 ++++++++++++++++++++++-- setup.cfg | 5 ++ setup.py | 3 + 10 files changed, 214 insertions(+), 20 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 setup.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..0e1d694a90c17 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,42 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +# Linting +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-ast + files: ^minetester/.*$ + - id: trailing-whitespace + files: ^minetester/.*$ + - id: end-of-file-fixer + exclude_types: [jupyter] + files: ^minetester/.*$ + - id: check-added-large-files + files: ^minetester/.*$ +- repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black + files: ^minetester/.*$ + - id: black-jupyter + files: ^minetester/.*$ +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] + files: ^minetester/.*$ +# Python static analysis +- repo: https://github.com/pycqa/flake8 + rev: '5.0.4' + hooks: + - id: flake8 + additional_dependencies: + - darglint~=1.8.1 + - flake8-blind-except==0.2.1 + - flake8-builtins~=1.5.3 + - flake8-commas~=2.1.0 + - flake8-debugger~=4.1.2 + - flake8-docstrings~=1.6.0 + files: ^minetester/.*$ \ No newline at end of file diff --git a/minetester/__init__.py b/minetester/__init__.py index 912f496bd18df..896f847443517 100644 --- a/minetester/__init__.py +++ b/minetester/__init__.py @@ -1 +1,3 @@ -from minetester.minetest_env import Minetest +"""Minetester: expose Minetest as environment via the Gymnasium interface.""" + +from minetester.minetest_env import Minetest # noqa: F401 diff --git a/minetester/minetest_env.py b/minetester/minetest_env.py index 3ab4ab5d075ec..585511b7c87ac 100644 --- a/minetester/minetest_env.py +++ b/minetester/minetest_env.py @@ -1,3 +1,4 @@ +"""Minetest gym environment.""" import datetime import logging import os @@ -9,17 +10,20 @@ import matplotlib.pyplot as plt import numpy as np import zmq + from minetester.utils import ( KEY_MAP, pack_pb_action, start_minetest_client, start_minetest_server, - unpack_pb_obs, start_xserver, + unpack_pb_obs, ) class Minetest(gym.Env): + """Minetest gym environment.""" + metadata = {"render.modes": ["rgb_array", "human"]} default_display_size = (1024, 600) @@ -46,6 +50,31 @@ def __init__( start_xvfb: bool = False, x_display: Optional[int] = None, ): + """Initialize Minetest environment. + + Args: + env_port: Port between gym environment and a Minetest client + server_port: Port between Minetest client and server + minetest_executable: Path to Minetest executable + log_dir: Path to log directory + config_path: Path to minetest.conf + cursor_image_path: Path to cursor image for menu and inventory + world_dir: Path to Minetest world directory + display_size: Size in pixels of the Minetest window + fov: Field of view in degrees of the Minetest window + seed: Seed for the Minetest world + start_minetest: Whether to start Minetest server and client or + connect to existing processes + game_id: Name of the Minetest game + clientmods: List of client mod names + servermods: List of server mod names + config_dict: Dictionary of config options updating the loaded config file + sync_port: Port between Minetest client and server for synchronization + sync_dtime: In-game time between two steps + headless: Whether to run Minetest in headless mode + start_xvfb: Whether to start X server virtual framebuffer + x_display: Display number to use for the X server virtual framebuffer + """ self.unique_env_id = str(uuid.uuid4()) # Graphics settings @@ -324,10 +353,20 @@ def _write_config(self): for key, value in self.config_dict.items(): config_file.write(f"{key} = {value}\n") - def seed(self, seed: int): + def seed(self, seed: int) -> None: + """Seed the environment with a given seed. + + Args: + seed: The seed to use. + """ self.the_seed = seed - def reset(self): + def reset(self) -> np.ndarray: + """Reset the environment. + + Returns: + The initial observation. + """ if self.start_minetest: if self.reset_world: self._delete_world() @@ -343,7 +382,19 @@ def reset(self): logging.debug("Received first obs: {}".format(obs.shape)) return obs - def step(self, action: Dict[str, Any]): + def step( + self, + action: Dict[str, Any], + ) -> Tuple[np.ndarray, float, bool, Dict[str, Any]]: + """Perform an action in the environment. + + Args: + action: The action to perform. + + Returns: + The next observation, the reward, whether the episode is done, and + additional info. + """ # Send action if isinstance(action["MOUSE"], np.ndarray): action["MOUSE"] = action["MOUSE"].tolist() @@ -351,7 +402,8 @@ def step(self, action: Dict[str, Any]): pb_action = pack_pb_action(action) self.socket.send(pb_action.SerializeToString()) - # TODO more robust check for whether a server/client is alive while receiving observations + # TODO more robust check for whether a server/client + # is alive while receiving observations for process in [self.server_process, self.client_process]: if process is not None and process.poll() is not None: return self.last_obs, 0.0, True, {} @@ -368,7 +420,18 @@ def step(self, action: Dict[str, Any]): logging.debug(f"Received obs - {next_obs.shape}; reward - {rew}") return next_obs, rew, done, info - def render(self, render_mode: str = "human"): + def render(self, render_mode: str = "human") -> Optional[np.ndarray]: + """Render the environment. + + Args: + render_mode: The mode to render in. Can be "human" or "rgb_array". + + Returns: + If render_mode is "rgb_array", returns the rendered image. + + Raises: + NotImplementedError: If render_mode is not "human" or "rgb_array". + """ if render_mode == "human": if self.render_img is None: # Setup figure @@ -392,10 +455,11 @@ def render(self, render_mode: str = "human"): raise NotImplementedError( "You are calling 'render()' with an unsupported" f" render mode: '{render_mode}'. " - f"Supported modes: {self.metadata['render.modes']}" + f"Supported modes: {self.metadata['render.modes']}", ) - def close(self): + def close(self) -> None: + """Close the environment.""" if self.render_fig is not None: plt.close() if self.socket is not None: diff --git a/minetester/proto/__init__.py b/minetester/proto/__init__.py index e69de29bb2d1d..23d5556a5bc30 100644 --- a/minetester/proto/__init__.py +++ b/minetester/proto/__init__.py @@ -0,0 +1 @@ +"""Protobuf package for Minetester.""" diff --git a/minetester/scripts/__init__.py b/minetester/scripts/__init__.py index e69de29bb2d1d..4b7606ddc17b6 100644 --- a/minetester/scripts/__init__.py +++ b/minetester/scripts/__init__.py @@ -0,0 +1 @@ +"""Utility scripts for Minetester.""" diff --git a/minetester/scripts/test_loop.py b/minetester/scripts/test_loop.py index 8dfa8d1381238..29fd53b412a22 100755 --- a/minetester/scripts/test_loop.py +++ b/minetester/scripts/test_loop.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 +"""Test loop for Minetest environment.""" from minetester import Minetest -import os env = Minetest( seed=42, @@ -12,7 +12,7 @@ clientmods=["random_v0"], ) -render = True +render = True obs = env.reset() done = False while not done: diff --git a/minetester/scripts/test_loop_parallel.py b/minetester/scripts/test_loop_parallel.py index d14d1e41b1c02..552edc6b7de75 100644 --- a/minetester/scripts/test_loop_parallel.py +++ b/minetester/scripts/test_loop_parallel.py @@ -1,14 +1,16 @@ +"""Test loop with parallel Minetest environments.""" import random from typing import Any, Dict, Optional from gym.wrappers import TimeLimit +from stable_baselines3.common.vec_env import SubprocVecEnv + from minetester import Minetest from minetester.utils import start_xserver -from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv if __name__ == "__main__": - def make_env( + def _make_env( rank: int, seed: int = 0, max_steps: int = 1e9, @@ -49,7 +51,7 @@ def _init(): vec_env_cls = SubprocVecEnv # DummyVecEnv venv = vec_env_cls( [ - make_env(rank=i, seed=seed, max_steps=max_steps, env_kwargs=env_kwargs) + _make_env(rank=i, seed=seed, max_steps=max_steps, env_kwargs=env_kwargs) for i in range(num_envs) ], ) diff --git a/minetester/utils.py b/minetester/utils.py index 64a567545ee42..a5902c8ddf69b 100644 --- a/minetester/utils.py +++ b/minetester/utils.py @@ -1,8 +1,10 @@ +"""Utility functions for Minetester.""" import os import subprocess from typing import Any, Dict, Tuple import numpy as np + from minetester.proto import objects_pb2 as pb_objects from minetester.proto.objects_pb2 import KeyType @@ -37,7 +39,21 @@ NOOP_ACTION.update({"MOUSE": np.zeros(2, dtype=int)}) -def unpack_pb_obs(received_obs: str): +def unpack_pb_obs( + received_obs: str, +) -> Tuple[np.ndarray, float, bool, Dict[str, Any], Dict[str, int]]: + """Unpack a protobuf observation received from Minetest client. + + Note: here 'observation' encompasses all information received from the client + within one step and should not be confused with the observation + returned by a gym environment. + + Args: + received_obs: The received observation. + + Returns: + The displayed image, task reward, done flag, info dict and last action. + """ pb_obs = pb_objects.Observation() pb_obs.ParseFromString(received_obs) obs = np.frombuffer(pb_obs.image.data, dtype=np.uint8).reshape( @@ -53,7 +69,15 @@ def unpack_pb_obs(received_obs: str): return obs, rew, done, info, last_action -def unpack_pb_action(pb_action: pb_objects.Action): +def unpack_pb_action(pb_action: pb_objects.Action) -> Dict[str, int]: + """Unpack a protobuf action. + + Args: + pb_action: The protobuf action. + + Returns: + The unpacked action as dictionary. + """ action = dict(NOOP_ACTION) action["MOUSE"] = [pb_action.mouseDx, pb_action.mouseDy] for key_event in pb_action.keyEvents: @@ -63,7 +87,15 @@ def unpack_pb_action(pb_action: pb_objects.Action): return action -def pack_pb_action(action: Dict[str, Any]): +def pack_pb_action(action: Dict[str, Any]) -> pb_objects.Action: + """Pack a protobuf action. + + Args: + action: The action as dictionary. + + Returns: + The packed protobuf action. + """ pb_action = pb_objects.Action() pb_action.mouseDx, pb_action.mouseDy = action["MOUSE"] for key, v in action.items(): @@ -87,7 +119,22 @@ def start_minetest_server( sync_port: int = None, sync_dtime: float = 0.001, game_id: str = "minetest", -): +) -> subprocess.Popen: + """Start a Minetest server. + + Args: + minetest_path: Path to the Minetest executable. + config_path: Path to the minetest.conf file. + log_path: Path to the log files. + server_port: Port of the server. + world_dir: Path to the world directory. + sync_port: Port for the synchronization with the server. + sync_dtime: In-game time between two steps. + game_id: Game ID of the game to be used. + + Returns: + The server process. + """ cmd = [ minetest_path, "--server", @@ -122,7 +169,24 @@ def start_minetest_client( sync_port: int = None, headless: bool = False, display: int = None, -): +) -> subprocess.Popen: + """Start a Minetest client. + + Args: + minetest_path: Path to the Minetest executable. + config_path: Path to the minetest.conf file. + log_path: Path to the log files. + client_port: Port of the client. + server_port: Port of the server to connect to. + cursor_img: Path to the cursor image. + client_name: Name of the client. + sync_port: Port for the synchronization with the server. + headless: Whether to run the client in headless mode. + display: value of the DISPLAY variable. + + Returns: + The client process. + """ cmd = [ minetest_path, "--name", @@ -164,7 +228,17 @@ def start_xserver( display_idx: int = 1, display_size: Tuple[int, int] = (1024, 600), display_depth: int = 24, -): +) -> subprocess.Popen: + """Start an X server. + + Args: + display_idx: Value of the DISPLAY variable. + display_size: Size of the display. + display_depth: Depth of the display. + + Returns: + The X server process. + """ cmd = [ "Xvfb", f":{display_idx}", diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000..735473191b74b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + E203, \ No newline at end of file diff --git a/setup.py b/setup.py index 2b15d47b2b409..d5d564c9002e5 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ from setuptools import setup +DEV = ["pre-commit", "black", "isort", "flake8"] + setup( name='Minetester', version='0.0.1', @@ -15,4 +17,5 @@ 'protobuf==3.20.1', 'psutil', ], + extras_require={"dev": DEV} ) From 80479dcfbe1603e9625a1ba3c0bdd0cc4a60821f Mon Sep 17 00:00:00 2001 From: rk1a Date: Thu, 6 Jul 2023 10:07:18 +0200 Subject: [PATCH 02/26] WIP add sphinx documentation --- docs-minetester/build.sh | 1 + docs-minetester/make.bat | 35 +++++++++++++ docs-minetester/source/conf.py | 49 +++++++++++++++++++ docs-minetester/source/index.rst | 38 ++++++++++++++ docs-minetester/source/minetester.rst | 34 +++++++++++++ docs-minetester/source/minetester.scripts.rst | 26 ++++++++++ docs-minetester/source/modules.rst | 7 +++ .../source/quickstart/installation.rst | 3 ++ .../source/quickstart/what_is_minetester.rst | 2 + .../source/tutorials/create_task.rst | 2 + .../source/tutorials/play_task.rst | 2 + .../source/tutorials/train_RL_agent.rst | 2 + minetester/scripts/test_loop.py | 43 ++++++++-------- minetester/scripts/test_loop_parallel.py | 14 +++--- setup.py | 1 + 15 files changed, 231 insertions(+), 28 deletions(-) create mode 100755 docs-minetester/build.sh create mode 100644 docs-minetester/make.bat create mode 100644 docs-minetester/source/conf.py create mode 100644 docs-minetester/source/index.rst create mode 100644 docs-minetester/source/minetester.rst create mode 100644 docs-minetester/source/minetester.scripts.rst create mode 100644 docs-minetester/source/modules.rst create mode 100644 docs-minetester/source/quickstart/installation.rst create mode 100644 docs-minetester/source/quickstart/what_is_minetester.rst create mode 100644 docs-minetester/source/tutorials/create_task.rst create mode 100644 docs-minetester/source/tutorials/play_task.rst create mode 100644 docs-minetester/source/tutorials/train_RL_agent.rst diff --git a/docs-minetester/build.sh b/docs-minetester/build.sh new file mode 100755 index 0000000000000..a82913644699f --- /dev/null +++ b/docs-minetester/build.sh @@ -0,0 +1 @@ +sphinx-apidoc -o ./source ../minetester ../minetester/proto -M \ No newline at end of file diff --git a/docs-minetester/make.bat b/docs-minetester/make.bat new file mode 100644 index 0000000000000..747ffb7b30336 --- /dev/null +++ b/docs-minetester/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs-minetester/source/conf.py b/docs-minetester/source/conf.py new file mode 100644 index 0000000000000..099d95989c5cf --- /dev/null +++ b/docs-minetester/source/conf.py @@ -0,0 +1,49 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'Minetester' +copyright = '2023, EleutherAI' +author = 'EleutherAI' +release = '0.0.1' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +templates_path = ['_templates'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +import os +import sys +sys.path.insert(0, os.path.abspath('../minetester')) +# ... +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx_rtd_theme', + 'sphinx_autodoc_typehints', + 'sphinx.ext.viewcode', + 'sphinx_copybutton', + 'sphinx.ext.autosummary', +] +# ... +napoleon_google_docstring = True +napoleon_numpy_docstring = False +autosummary_generate = True +autodoc_default_options = { + "members": True, + "undoc-members": True, + "special-members": "__init__", + "show-inheritance": True, +} + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_title = "Minetester" +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs-minetester/source/index.rst b/docs-minetester/source/index.rst new file mode 100644 index 0000000000000..d5aaa19dfe02e --- /dev/null +++ b/docs-minetester/source/index.rst @@ -0,0 +1,38 @@ +.. Minetester documentation master file, created by + sphinx-quickstart on Wed Jul 5 21:35:27 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +====================================== +Welcome to Minetester's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Quickstart + + quickstart/installation + quickstart/what_is_minetester + +.. toctree:: + :maxdepth: 2 + :caption: Tutorials + + tutorials/create_task + tutorials/play_task + tutorials/train_RL_agent + + +.. toctree:: + :maxdepth: 2 + :caption: Package Reference + + modules + + +Index +===== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs-minetester/source/minetester.rst b/docs-minetester/source/minetester.rst new file mode 100644 index 0000000000000..ff33b51d983e7 --- /dev/null +++ b/docs-minetester/source/minetester.rst @@ -0,0 +1,34 @@ +minetester package +================== + +.. automodule:: minetester + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + minetester.scripts + +Submodules +---------- + +minetester.minetest\_env module +------------------------------- + +.. automodule:: minetester.minetest_env + :members: + :undoc-members: + :show-inheritance: + +minetester.utils module +----------------------- + +.. automodule:: minetester.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs-minetester/source/minetester.scripts.rst b/docs-minetester/source/minetester.scripts.rst new file mode 100644 index 0000000000000..5055343ad1a31 --- /dev/null +++ b/docs-minetester/source/minetester.scripts.rst @@ -0,0 +1,26 @@ +minetester.scripts package +========================== + +.. automodule:: minetester.scripts + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +minetester.scripts.test\_loop module +------------------------------------ + +.. automodule:: minetester.scripts.test_loop + :members: + :undoc-members: + :show-inheritance: + +minetester.scripts.test\_loop\_parallel module +---------------------------------------------- + +.. automodule:: minetester.scripts.test_loop_parallel + :members: + :undoc-members: + :show-inheritance: diff --git a/docs-minetester/source/modules.rst b/docs-minetester/source/modules.rst new file mode 100644 index 0000000000000..7827d4864c13c --- /dev/null +++ b/docs-minetester/source/modules.rst @@ -0,0 +1,7 @@ +minetester +========== + +.. toctree:: + :maxdepth: 4 + + minetester diff --git a/docs-minetester/source/quickstart/installation.rst b/docs-minetester/source/quickstart/installation.rst new file mode 100644 index 0000000000000..cd7a81df9f566 --- /dev/null +++ b/docs-minetester/source/quickstart/installation.rst @@ -0,0 +1,3 @@ +Installation +============ + diff --git a/docs-minetester/source/quickstart/what_is_minetester.rst b/docs-minetester/source/quickstart/what_is_minetester.rst new file mode 100644 index 0000000000000..cce4c1130991c --- /dev/null +++ b/docs-minetester/source/quickstart/what_is_minetester.rst @@ -0,0 +1,2 @@ +What is Minetester? +=================== \ No newline at end of file diff --git a/docs-minetester/source/tutorials/create_task.rst b/docs-minetester/source/tutorials/create_task.rst new file mode 100644 index 0000000000000..514104adb74de --- /dev/null +++ b/docs-minetester/source/tutorials/create_task.rst @@ -0,0 +1,2 @@ +How to create a Minetester task? +================================ \ No newline at end of file diff --git a/docs-minetester/source/tutorials/play_task.rst b/docs-minetester/source/tutorials/play_task.rst new file mode 100644 index 0000000000000..7446a07c2a249 --- /dev/null +++ b/docs-minetester/source/tutorials/play_task.rst @@ -0,0 +1,2 @@ +Playing a task as human +======================= \ No newline at end of file diff --git a/docs-minetester/source/tutorials/train_RL_agent.rst b/docs-minetester/source/tutorials/train_RL_agent.rst new file mode 100644 index 0000000000000..6c7bffb84b4e3 --- /dev/null +++ b/docs-minetester/source/tutorials/train_RL_agent.rst @@ -0,0 +1,2 @@ +Training an RL agent +==================== \ No newline at end of file diff --git a/minetester/scripts/test_loop.py b/minetester/scripts/test_loop.py index 29fd53b412a22..1f0982569b04e 100755 --- a/minetester/scripts/test_loop.py +++ b/minetester/scripts/test_loop.py @@ -2,25 +2,26 @@ """Test loop for Minetest environment.""" from minetester import Minetest -env = Minetest( - seed=42, - start_minetest=True, - sync_port=30010, - sync_dtime=0.05, - headless=True, - start_xvfb=True, - clientmods=["random_v0"], -) +if __name__ == "__main__": + env = Minetest( + seed=42, + start_minetest=True, + sync_port=30010, + sync_dtime=0.05, + headless=True, + start_xvfb=True, + clientmods=["random_v0"], + ) -render = True -obs = env.reset() -done = False -while not done: - try: - action = env.action_space.sample() - obs, rew, done, info = env.step(action) - if render: - env.render() - except KeyboardInterrupt: - break -env.close() + render = True + obs = env.reset() + done = False + while not done: + try: + action = env.action_space.sample() + obs, rew, done, info = env.step(action) + if render: + env.render() + except KeyboardInterrupt: + break + env.close() diff --git a/minetester/scripts/test_loop_parallel.py b/minetester/scripts/test_loop_parallel.py index 552edc6b7de75..e519c4a2af996 100644 --- a/minetester/scripts/test_loop_parallel.py +++ b/minetester/scripts/test_loop_parallel.py @@ -1,14 +1,14 @@ """Test loop with parallel Minetest environments.""" -import random -from typing import Any, Dict, Optional -from gym.wrappers import TimeLimit -from stable_baselines3.common.vec_env import SubprocVecEnv +if __name__ == "__main__": + import random + from typing import Any, Dict, Optional -from minetester import Minetest -from minetester.utils import start_xserver + from gym.wrappers import TimeLimit + from stable_baselines3.common.vec_env import SubprocVecEnv -if __name__ == "__main__": + from minetester import Minetest + from minetester.utils import start_xserver def _make_env( rank: int, diff --git a/setup.py b/setup.py index d5d564c9002e5..bcfc933fcb1fb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ from setuptools import setup DEV = ["pre-commit", "black", "isort", "flake8"] +DOCS = ["sphinx", "sphinx_rtd_theme", "sphinx-autobuild"] setup( name='Minetester', From 9d27cfd8f51f602b2d9679c0c7b8ed4ffa097109 Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 10 Jul 2023 23:14:46 +0200 Subject: [PATCH 03/26] WIP sphinx docs --- .../source => cursors}/modules.rst | 0 docs-minetester/build.sh | 2 +- docs-minetester/source/.gitignore | 1 + docs-minetester/source/_templates/module.rst | 30 ++++++ .../source/advanced/client_api.rst | 4 + docs-minetester/source/conf.py | 1 + docs-minetester/source/index.rst | 19 +++- docs-minetester/source/minetester.rst | 34 ------- docs-minetester/source/minetester.scripts.rst | 26 ------ .../source/quickstart/installation.rst | 14 +++ .../source/quickstart/what_is_minetester.rst | 12 ++- .../source/tutorials/create_task.rst | 93 ++++++++++++++++++- .../source/tutorials/play_task.rst | 4 +- .../source/tutorials/train_RL_agent.rst | 7 +- 14 files changed, 175 insertions(+), 72 deletions(-) rename {docs-minetester/source => cursors}/modules.rst (100%) create mode 100644 docs-minetester/source/.gitignore create mode 100644 docs-minetester/source/_templates/module.rst create mode 100644 docs-minetester/source/advanced/client_api.rst delete mode 100644 docs-minetester/source/minetester.rst delete mode 100644 docs-minetester/source/minetester.scripts.rst diff --git a/docs-minetester/source/modules.rst b/cursors/modules.rst similarity index 100% rename from docs-minetester/source/modules.rst rename to cursors/modules.rst diff --git a/docs-minetester/build.sh b/docs-minetester/build.sh index a82913644699f..19dad8b2131b3 100755 --- a/docs-minetester/build.sh +++ b/docs-minetester/build.sh @@ -1 +1 @@ -sphinx-apidoc -o ./source ../minetester ../minetester/proto -M \ No newline at end of file +sphinx-apidoc -o ./source ../minetester ../minetester/proto \ No newline at end of file diff --git a/docs-minetester/source/.gitignore b/docs-minetester/source/.gitignore new file mode 100644 index 0000000000000..e64636897f362 --- /dev/null +++ b/docs-minetester/source/.gitignore @@ -0,0 +1 @@ +_api \ No newline at end of file diff --git a/docs-minetester/source/_templates/module.rst b/docs-minetester/source/_templates/module.rst new file mode 100644 index 0000000000000..7db58b1ed84be --- /dev/null +++ b/docs-minetester/source/_templates/module.rst @@ -0,0 +1,30 @@ +{{ fullname | escape | underline }} + +Description +----------- + +.. automodule:: {{ fullname | escape }} + +{% if classes %} +Classes +------- +.. autosummary: + :toctree: _autosummary + + {% for class in classes %} + {{ class }} + {% endfor %} + +{% endif %} + +{% if functions %} +Functions +--------- +.. autosummary: + :toctree: _autosummary + + {% for function in functions %} + {{ function }} + {% endfor %} + +{% endif %} \ No newline at end of file diff --git a/docs-minetester/source/advanced/client_api.rst b/docs-minetester/source/advanced/client_api.rst new file mode 100644 index 0000000000000..aff5d494736c1 --- /dev/null +++ b/docs-minetester/source/advanced/client_api.rst @@ -0,0 +1,4 @@ +Client API +========== + +The client API uses zmq for communication. \ No newline at end of file diff --git a/docs-minetester/source/conf.py b/docs-minetester/source/conf.py index 099d95989c5cf..868f41567a0a9 100644 --- a/docs-minetester/source/conf.py +++ b/docs-minetester/source/conf.py @@ -23,6 +23,7 @@ # ... extensions = [ 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', 'sphinx.ext.napoleon', 'sphinx_rtd_theme', 'sphinx_autodoc_typehints', diff --git a/docs-minetester/source/index.rst b/docs-minetester/source/index.rst index d5aaa19dfe02e..eb1a4e8b4c933 100644 --- a/docs-minetester/source/index.rst +++ b/docs-minetester/source/index.rst @@ -15,19 +15,29 @@ Welcome to Minetester's documentation! quickstart/what_is_minetester .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Tutorials tutorials/create_task tutorials/play_task tutorials/train_RL_agent - .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + :caption: Advanced Usage + + advanced/client_api + + +Package Reference +~~~~~~~~~~~~~~~~~ +.. autosummary:: + :toctree: _api :caption: Package Reference + :recursive: + :template: autosummary/module.rst - modules + minetester Index @@ -35,4 +45,3 @@ Index * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/docs-minetester/source/minetester.rst b/docs-minetester/source/minetester.rst deleted file mode 100644 index ff33b51d983e7..0000000000000 --- a/docs-minetester/source/minetester.rst +++ /dev/null @@ -1,34 +0,0 @@ -minetester package -================== - -.. automodule:: minetester - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - minetester.scripts - -Submodules ----------- - -minetester.minetest\_env module -------------------------------- - -.. automodule:: minetester.minetest_env - :members: - :undoc-members: - :show-inheritance: - -minetester.utils module ------------------------ - -.. automodule:: minetester.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs-minetester/source/minetester.scripts.rst b/docs-minetester/source/minetester.scripts.rst deleted file mode 100644 index 5055343ad1a31..0000000000000 --- a/docs-minetester/source/minetester.scripts.rst +++ /dev/null @@ -1,26 +0,0 @@ -minetester.scripts package -========================== - -.. automodule:: minetester.scripts - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -minetester.scripts.test\_loop module ------------------------------------- - -.. automodule:: minetester.scripts.test_loop - :members: - :undoc-members: - :show-inheritance: - -minetester.scripts.test\_loop\_parallel module ----------------------------------------------- - -.. automodule:: minetester.scripts.test_loop_parallel - :members: - :undoc-members: - :show-inheritance: diff --git a/docs-minetester/source/quickstart/installation.rst b/docs-minetester/source/quickstart/installation.rst index cd7a81df9f566..9766b94171d5a 100644 --- a/docs-minetester/source/quickstart/installation.rst +++ b/docs-minetester/source/quickstart/installation.rst @@ -1,3 +1,17 @@ Installation ============ +For normal usage it suffices to install via pip: + +.. code-block:: bash + + pip install minetester + +Developers should install the repo in edit mode: + +.. code-block:: bash + + pip install -e .[dev] + + +.. note:: This is a note \ No newline at end of file diff --git a/docs-minetester/source/quickstart/what_is_minetester.rst b/docs-minetester/source/quickstart/what_is_minetester.rst index cce4c1130991c..05d673f1006b2 100644 --- a/docs-minetester/source/quickstart/what_is_minetester.rst +++ b/docs-minetester/source/quickstart/what_is_minetester.rst @@ -1,2 +1,12 @@ What is Minetester? -=================== \ No newline at end of file +=================== + +Minetester extends the open-source voxel engine `Minetest `_ to support training and evaluation of AI / RL agents. +The Minetest core was modified to add the following main features: + +- a dummy client that adds an `API for controlling a player and receiving game information <../tutorials/client_api.html>`_ +- support for `custom tasks using the Minetest modding API <../tutorials/create_task.html>`_ +- headless client operation +- client-server synchronization + +In addition, the *minetester* Python package provides a gymnasium environment and utilities for communication with Minetest clients. \ No newline at end of file diff --git a/docs-minetester/source/tutorials/create_task.rst b/docs-minetester/source/tutorials/create_task.rst index 514104adb74de..fa0f759a0a941 100644 --- a/docs-minetester/source/tutorials/create_task.rst +++ b/docs-minetester/source/tutorials/create_task.rst @@ -1,2 +1,91 @@ -How to create a Minetester task? -================================ \ No newline at end of file +How to create a new task? +================================ + +Tasks are easy to create using Minetest's builtin Lua modding API. +The task-relevant information (rewards, task completion status, meta information, ...) are passed through the Minetest processes and made available to the `client API `_. + + +Task definition +--------------- + +A basic Minetest mod is defined by a directory with the following structure + +.. code-block:: + + my_task\ + mod.conf + init.lua + settingtypes.txt + +where `mod.conf` contains meta data, `init.lua` the necessary Lua code, and `settingtypes.txt` possible mod settings. + +A task has to define two global variables in the Lua script environment: + +1. `REWARD`: a scalar floting point number indicating the reward received at the current step. +2. `TERMINAL`: a boolean indicating whether the agent has reached a terminal state of the task. + +The variables can be changed at every step, or based on in-game events. + +Example: A simple treechop task +------------------------------- + +`mod.conf` + +.. code-block:: + + name=treechop + +`settingtypes.txt` + +.. code-block:: + + # Number of tree chops required for completing task + # Default is 10. At least one chop is required. + treechop_goal (Tree chop goal) int 10 1 65535 + + +`ìnit.lua` + +.. code-block:: lua + + -- task settings + TREECHOP_GOAL = minetest.settings:get("treechop_goal") or 10 + + minetest.register_on_dignode(function(pos, node, digger) + if string.find(node["name"], "tree") then + minetest.debug("Dug a tree!") + REWARD[digger:get_player_name()] = 1.0 + end + + -- count the number of tree items of digging player + local num_tree_items = 0 + local inv = digger:get_inventory() + local size = inv:get_size("main") + for i = 1, size do + local stack = inv:get_stack("main", i) + if string.find(stack:get_name(), "tree") then + num_tree_items = num_tree_items + stack:get_count() + end + end + if num_tree_items >= TREECHOP_GOAL then + minetest.debug(digger:get_player_name() .. " reached the goal!") + TERMINAL[digger:get_player_name()] = true + end + end) + + +Asynchronous mode and client mods +--------------------------------- + +Currently, server mods can only be used to modify the Minetester global variables if client-server synchronization is active. +Otherwise, a client mod has to be used instead. +Client mods have certain limitations to what information they can access. +For example, there is no acces to a player's inventory. +To circumvent these limitations, one can use a pair of mods (one client and one server mod) that establish a mod channel between them to share the missing information. +However, it is recommended to use client-server synchronization with a single server mod where possible. + +Further resources +----------------- + +- `Minetest Modding API Reference `_ +- `Minetest Modding Forum `_ \ No newline at end of file diff --git a/docs-minetester/source/tutorials/play_task.rst b/docs-minetester/source/tutorials/play_task.rst index 7446a07c2a249..b8d2ec19b048e 100644 --- a/docs-minetester/source/tutorials/play_task.rst +++ b/docs-minetester/source/tutorials/play_task.rst @@ -1,2 +1,4 @@ Playing a task as human -======================= \ No newline at end of file +======================= + +use utility scripts \ No newline at end of file diff --git a/docs-minetester/source/tutorials/train_RL_agent.rst b/docs-minetester/source/tutorials/train_RL_agent.rst index 6c7bffb84b4e3..17bbbcd55b1bb 100644 --- a/docs-minetester/source/tutorials/train_RL_agent.rst +++ b/docs-minetester/source/tutorials/train_RL_agent.rst @@ -1,2 +1,5 @@ -Training an RL agent -==================== \ No newline at end of file +Training an RL agent using the Minetest gymnasium environment +============================================================= + +In this tutorial we will be using +`Minetest Baselines `_ \ No newline at end of file From 32bd9429a4595c038f375f5f008004915c2847e2 Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 10 Jul 2023 23:14:46 +0200 Subject: [PATCH 04/26] WIP sphinx docs --- docs-minetester/build.sh | 2 +- docs-minetester/source/.gitignore | 1 + docs-minetester/source/_templates/module.rst | 30 ++++++ .../source/advanced/building_source.rst | 5 + .../source/advanced/client_api.rst | 5 + .../source/community/contributing.rst | 8 ++ docs-minetester/source/community/support.rst | 2 + docs-minetester/source/conf.py | 1 + docs-minetester/source/index.rst | 30 ++++-- docs-minetester/source/minetester.rst | 34 ------- docs-minetester/source/minetester.scripts.rst | 26 ----- docs-minetester/source/modules.rst | 7 -- .../source/quickstart/hello_minetest.rst | 96 +++++++++++++++++++ .../source/quickstart/installation.rst | 13 +++ .../source/quickstart/minetest_conf_dummy.rst | 5 + .../source/quickstart/what_is_minetester.rst | 14 ++- .../source/tutorials/create_task.rst | 92 +++++++++++++++++- .../source/tutorials/headless_mode.rst | 43 +++++++++ .../source/tutorials/play_task.rst | 2 - .../source/tutorials/synchronization.rst | 6 ++ .../source/tutorials/train_RL_agent.rst | 7 +- minetester/utils.py | 2 +- scripts/sync_server.sh | 2 +- 23 files changed, 350 insertions(+), 83 deletions(-) create mode 100644 docs-minetester/source/.gitignore create mode 100644 docs-minetester/source/_templates/module.rst create mode 100644 docs-minetester/source/advanced/building_source.rst create mode 100644 docs-minetester/source/advanced/client_api.rst create mode 100644 docs-minetester/source/community/contributing.rst create mode 100644 docs-minetester/source/community/support.rst delete mode 100644 docs-minetester/source/minetester.rst delete mode 100644 docs-minetester/source/minetester.scripts.rst delete mode 100644 docs-minetester/source/modules.rst create mode 100644 docs-minetester/source/quickstart/hello_minetest.rst create mode 100644 docs-minetester/source/quickstart/minetest_conf_dummy.rst create mode 100644 docs-minetester/source/tutorials/headless_mode.rst delete mode 100644 docs-minetester/source/tutorials/play_task.rst create mode 100644 docs-minetester/source/tutorials/synchronization.rst diff --git a/docs-minetester/build.sh b/docs-minetester/build.sh index a82913644699f..19dad8b2131b3 100755 --- a/docs-minetester/build.sh +++ b/docs-minetester/build.sh @@ -1 +1 @@ -sphinx-apidoc -o ./source ../minetester ../minetester/proto -M \ No newline at end of file +sphinx-apidoc -o ./source ../minetester ../minetester/proto \ No newline at end of file diff --git a/docs-minetester/source/.gitignore b/docs-minetester/source/.gitignore new file mode 100644 index 0000000000000..e64636897f362 --- /dev/null +++ b/docs-minetester/source/.gitignore @@ -0,0 +1 @@ +_api \ No newline at end of file diff --git a/docs-minetester/source/_templates/module.rst b/docs-minetester/source/_templates/module.rst new file mode 100644 index 0000000000000..7db58b1ed84be --- /dev/null +++ b/docs-minetester/source/_templates/module.rst @@ -0,0 +1,30 @@ +{{ fullname | escape | underline }} + +Description +----------- + +.. automodule:: {{ fullname | escape }} + +{% if classes %} +Classes +------- +.. autosummary: + :toctree: _autosummary + + {% for class in classes %} + {{ class }} + {% endfor %} + +{% endif %} + +{% if functions %} +Functions +--------- +.. autosummary: + :toctree: _autosummary + + {% for function in functions %} + {{ function }} + {% endfor %} + +{% endif %} \ No newline at end of file diff --git a/docs-minetester/source/advanced/building_source.rst b/docs-minetester/source/advanced/building_source.rst new file mode 100644 index 0000000000000..10c5a51f5746a --- /dev/null +++ b/docs-minetester/source/advanced/building_source.rst @@ -0,0 +1,5 @@ +Building Minetester from source +=============================== + +.. literalinclude:: ../../../build_instructions.txt + :language: text \ No newline at end of file diff --git a/docs-minetester/source/advanced/client_api.rst b/docs-minetester/source/advanced/client_api.rst new file mode 100644 index 0000000000000..a4f142a198712 --- /dev/null +++ b/docs-minetester/source/advanced/client_api.rst @@ -0,0 +1,5 @@ +Client API +========== + +The client API uses zmq for communication. +Protobuf objects are passed between client and gym environment. \ No newline at end of file diff --git a/docs-minetester/source/community/contributing.rst b/docs-minetester/source/community/contributing.rst new file mode 100644 index 0000000000000..f2a1cb2b2490f --- /dev/null +++ b/docs-minetester/source/community/contributing.rst @@ -0,0 +1,8 @@ +Contributions guide +=================== + +Developers should install the repo in edit mode: + +.. code-block:: bash + + pip install -e .[dev] \ No newline at end of file diff --git a/docs-minetester/source/community/support.rst b/docs-minetester/source/community/support.rst new file mode 100644 index 0000000000000..4522fc0e59a66 --- /dev/null +++ b/docs-minetester/source/community/support.rst @@ -0,0 +1,2 @@ +Support +======= \ No newline at end of file diff --git a/docs-minetester/source/conf.py b/docs-minetester/source/conf.py index 099d95989c5cf..868f41567a0a9 100644 --- a/docs-minetester/source/conf.py +++ b/docs-minetester/source/conf.py @@ -23,6 +23,7 @@ # ... extensions = [ 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', 'sphinx.ext.napoleon', 'sphinx_rtd_theme', 'sphinx_autodoc_typehints', diff --git a/docs-minetester/source/index.rst b/docs-minetester/source/index.rst index d5aaa19dfe02e..f2cc6f9f9e83b 100644 --- a/docs-minetester/source/index.rst +++ b/docs-minetester/source/index.rst @@ -13,26 +13,44 @@ Welcome to Minetester's documentation! quickstart/installation quickstart/what_is_minetester + quickstart/hello_minetest +.. _tutorials: .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Tutorials + tutorials/headless_mode + tutorials/synchronization tutorials/create_task - tutorials/play_task tutorials/train_RL_agent +.. toctree:: + :maxdepth: 1 + :caption: Advanced Usage + + advanced/client_api + advanced/building_source .. toctree:: - :maxdepth: 2 - :caption: Package Reference + :maxdepth: 1 + :caption: Community + + community/support + community/contributing - modules +Package Reference +~~~~~~~~~~~~~~~~~ +.. autosummary:: + :toctree: _api + :caption: Package Reference + :recursive: + :template: autosummary/module.rst + minetester Index ===== * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/docs-minetester/source/minetester.rst b/docs-minetester/source/minetester.rst deleted file mode 100644 index ff33b51d983e7..0000000000000 --- a/docs-minetester/source/minetester.rst +++ /dev/null @@ -1,34 +0,0 @@ -minetester package -================== - -.. automodule:: minetester - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - minetester.scripts - -Submodules ----------- - -minetester.minetest\_env module -------------------------------- - -.. automodule:: minetester.minetest_env - :members: - :undoc-members: - :show-inheritance: - -minetester.utils module ------------------------ - -.. automodule:: minetester.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs-minetester/source/minetester.scripts.rst b/docs-minetester/source/minetester.scripts.rst deleted file mode 100644 index 5055343ad1a31..0000000000000 --- a/docs-minetester/source/minetester.scripts.rst +++ /dev/null @@ -1,26 +0,0 @@ -minetester.scripts package -========================== - -.. automodule:: minetester.scripts - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -minetester.scripts.test\_loop module ------------------------------------- - -.. automodule:: minetester.scripts.test_loop - :members: - :undoc-members: - :show-inheritance: - -minetester.scripts.test\_loop\_parallel module ----------------------------------------------- - -.. automodule:: minetester.scripts.test_loop_parallel - :members: - :undoc-members: - :show-inheritance: diff --git a/docs-minetester/source/modules.rst b/docs-minetester/source/modules.rst deleted file mode 100644 index 7827d4864c13c..0000000000000 --- a/docs-minetester/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -minetester -========== - -.. toctree:: - :maxdepth: 4 - - minetester diff --git a/docs-minetester/source/quickstart/hello_minetest.rst b/docs-minetester/source/quickstart/hello_minetest.rst new file mode 100644 index 0000000000000..355d134135ef2 --- /dev/null +++ b/docs-minetester/source/quickstart/hello_minetest.rst @@ -0,0 +1,96 @@ +Hello, Minetest(er)! +==================== + +To get started with Minetester you should first know your way around the directory structure. +The most important directories are these: + +.. code-block:: yaml + + minetest/ + ├── bin # contains minetest executable + ├── clientmods # contains client-side mods + ├── doc # contains documentation + ├── docs-minetester # contains Minetester documentation + ├── games # contains games (vanilla Minetest by default) + ├── log # contains log files + ├── mods # contains server-side mods + ├── minetester # contains Minetester Python module + ├── scripts # contains utility scripts + └── src # contains C++ source code + + +Executing Minetest +------------------ + +You can jump directly into the game (skipping the main menu) by running + +.. code-block:: bash + + ./bin/minetest --go + +Internally, this command starts a server and a client on your computer and connects them to each other. + +There are many more command line options available. You can find them by running + +.. code-block:: bash + + ./bin/minetest --help + +For example, you can start a server and a client separately by running these commands in different terminals: + +.. code-block:: bash + + ./bin/minetest --server + ./bin/minetest --address 0.0.0.0 --port 30000 --go --name Bob + +A powerful way to modify Minetest is to pass a configuration file (see `minetest.conf.example `_ for available options): + +.. code-block:: bash + + ./bin/minetest --config minetest.conf + +Minetester command line options +------------------------------- + +.. list-table:: + :widths: 1 2 + :header-rows: 1 + + * - Minetester CLI Option + - Description + * - ``--dumb`` + - Start a dumb client that needs to connect to an external controller client. + * - ``--record`` + - Start a recording client that needs to connect to an external data gathering client. + * - ``--client-address`` + - Address to the controller / data gathering client. + * - ``--headless`` + - Start client in headless mode. + * - ``--sync-port`` + - Internal port used for syncing server and clients. + * - ``--sync-dtime`` + - Ingame time difference between steps when using server-client synchronization. + * - ``--noresizing`` + - Disallow screen resizing. + * - ``--cursor-image`` + - Path to cursor image file that is rendered at the mouse position in the dumb client mode. + +To get started the remainder of this guide focusses on connecting a **server**, a **dumb client** and the `builtin Python controller client wrapped as gymnasium environment. <../_api/minetester.minetest_env.html#minetester.minetest_env.Minetest>`_ + +To learn more about the other CLI options please refer to the :ref:`tutorials `. + +Sending random actions +---------------------- + +TODO + +There are a few useful scripts to start servers and clients with a default configuration file. + +- how to start a client and a server +- how to start a dumb client and connect the gym environment + + +Further Resources +----------------- + +- `minetest.net `_ \ No newline at end of file diff --git a/docs-minetester/source/quickstart/installation.rst b/docs-minetester/source/quickstart/installation.rst index cd7a81df9f566..5c6f993a9be61 100644 --- a/docs-minetester/source/quickstart/installation.rst +++ b/docs-minetester/source/quickstart/installation.rst @@ -1,3 +1,16 @@ Installation ============ +For normal usage it suffices to install pre-built binaries via pip: + +.. code-block:: bash + + pip install minetester + +Verify your installation by running + +.. code-block:: bash + + python -m minetester.scripts.test_loop + +You should see a window pop up with a player performing random actions in Minetest. \ No newline at end of file diff --git a/docs-minetester/source/quickstart/minetest_conf_dummy.rst b/docs-minetester/source/quickstart/minetest_conf_dummy.rst new file mode 100644 index 0000000000000..21699253ef15b --- /dev/null +++ b/docs-minetester/source/quickstart/minetest_conf_dummy.rst @@ -0,0 +1,5 @@ +Minetest configuration file +=========================== + +.. literalinclude:: ../../../minetest.conf.example + :language: text diff --git a/docs-minetester/source/quickstart/what_is_minetester.rst b/docs-minetester/source/quickstart/what_is_minetester.rst index cce4c1130991c..620de9791a84f 100644 --- a/docs-minetester/source/quickstart/what_is_minetester.rst +++ b/docs-minetester/source/quickstart/what_is_minetester.rst @@ -1,2 +1,14 @@ What is Minetester? -=================== \ No newline at end of file +=================== + +Minetester extends the open-source voxel engine `Minetest `_ to support training and evaluation of AI / RL agents. +The Minetest core was modified to add the following main features: + +- a dummy client that adds an `API for controlling a player and receiving game information <../advanced/client_api.html>`_ +- support for `custom tasks using the Minetest modding API <../tutorials/create_task.html>`_ +- `headless client operation <../tutorials/headless_mode.html>`_ +- `client-server synchronization <../tutorials/synchronization.html>`_ + +In addition, the :py:mod:`minetester` Python package provides a gymnasium environment and utilities for communication with Minetest clients. + +For a motivation of the project, see the introductory `blog post `_. diff --git a/docs-minetester/source/tutorials/create_task.rst b/docs-minetester/source/tutorials/create_task.rst index 514104adb74de..dd0488547025f 100644 --- a/docs-minetester/source/tutorials/create_task.rst +++ b/docs-minetester/source/tutorials/create_task.rst @@ -1,2 +1,90 @@ -How to create a Minetester task? -================================ \ No newline at end of file +How to create a new task? +================================ + +Tasks are easy to create using Minetest's builtin Lua modding API. +The task-relevant information (rewards, task completion status, meta information, ...) are passed through the Minetest processes and made available to the `client API `_. + + +Task definition +--------------- + +A basic Minetest mod is defined by a directory with the following structure + +.. code-block:: + + my_task\ + mod.conf + init.lua + settingtypes.txt + +where `mod.conf` contains meta data, `init.lua` the necessary Lua code, and `settingtypes.txt` possible mod settings. + +A task has to define two global variables in the Lua script environment: + +1. `REWARD`: a scalar floting point number indicating the reward received at the current step. +2. `TERMINAL`: a boolean indicating whether the agent has reached a terminal state of the task. + +The variables can be changed at every step, or based on in-game events. + +Example: A simple treechop task +------------------------------- + +`mod.conf` + +.. literalinclude :: ../../../mods/treechop_v2/mod.conf + :language: text + +`settingtypes.txt` + +.. literalinclude :: ../../../mods/treechop_v2/settingtypes.txt + :language: text + +`ìnit.lua` + +.. literalinclude :: ../../../mods/treechop_v2/init.lua + :language: lua + + +.. code-block:: lua + + -- task settings + TREECHOP_GOAL = minetest.settings:get("treechop_goal") or 10 + + minetest.register_on_dignode(function(pos, node, digger) + if string.find(node["name"], "tree") then + minetest.debug("Dug a tree!") + REWARD[digger:get_player_name()] = 1.0 + end + + -- count the number of tree items of digging player + local num_tree_items = 0 + local inv = digger:get_inventory() + local size = inv:get_size("main") + for i = 1, size do + local stack = inv:get_stack("main", i) + if string.find(stack:get_name(), "tree") then + num_tree_items = num_tree_items + stack:get_count() + end + end + if num_tree_items >= TREECHOP_GOAL then + minetest.debug(digger:get_player_name() .. " reached the goal!") + TERMINAL[digger:get_player_name()] = true + end + end) + + +Asynchronous mode and client mods +--------------------------------- + +Currently, server mods can only be used to modify the Minetester global variables if client-server synchronization is active. +Otherwise, a client mod has to be used instead. +Client mods have certain limitations to what information they can access. +For example, there is no acces to a player's inventory. +To circumvent these limitations, one can use a pair of mods (one client and one server mod) that establish a mod channel between them to share the missing information. +However, it is recommended to use client-server synchronization with a single server mod where possible. + +Further resources +----------------- + +- `Minetest Modding API Reference `_ +- `Minetest Modding Forum `_ \ No newline at end of file diff --git a/docs-minetester/source/tutorials/headless_mode.rst b/docs-minetester/source/tutorials/headless_mode.rst new file mode 100644 index 0000000000000..7cb5af38f09f5 --- /dev/null +++ b/docs-minetester/source/tutorials/headless_mode.rst @@ -0,0 +1,43 @@ +Headless modes +============== + +There are two different modes, one using virtual framebuffer X server (`Xvfb `_) and one using SDL2's offscreen mode. + +Xvfb +~~~~ +In order to use the Xvfb headless mode, before starting the Minetest client, a new virtual framebuffer X server has to be started at an unused display number. +You can either use the Xvfb CLI: + +.. code-block:: bash + + export DISPLAY=:4 + export DISPLAY_WIDTH=1024 + export DISPLAY_HEIGHT=600 + export DISPLAY_DEPTH=24 + Xvfb $DISPLAY -screen 0 ${DISPLAY_WIDTH}x${DISPLAY_HEIGHT}x${DISPLAY_DEPTH} + +or this Python utiltity: :py:func:`minetester.utils.start_xserver` + + +In addition, the ``--headless`` runtime flag has to be passed to the Minetest client, e.g. + +.. code-block:: Python + + from minetester import Minetest + env = Minetest(headless=True) + +For convenience you can also tell the Minetest gym environment to start a Xvfb server: + +.. code-block:: Python + + from minetester import Minetest + env = Minetest(start_xvfb=True, headless=True) + +SDL2 offscreen mode +~~~~~~~~~~~~~~~~~~~ + +Using the SDL2-based headless mode requires compilation with the following build flags + +``-DBUILD_HEADLESS=1 -DSDL2_DIR=/SDL/build/lib/cmake/SDL2/`` + +It also requires the ``--headless`` runtime flag to be passed to the Minetest client. \ No newline at end of file diff --git a/docs-minetester/source/tutorials/play_task.rst b/docs-minetester/source/tutorials/play_task.rst deleted file mode 100644 index 7446a07c2a249..0000000000000 --- a/docs-minetester/source/tutorials/play_task.rst +++ /dev/null @@ -1,2 +0,0 @@ -Playing a task as human -======================= \ No newline at end of file diff --git a/docs-minetester/source/tutorials/synchronization.rst b/docs-minetester/source/tutorials/synchronization.rst new file mode 100644 index 0000000000000..7f96981491ab3 --- /dev/null +++ b/docs-minetester/source/tutorials/synchronization.rst @@ -0,0 +1,6 @@ +Client-server synchronization +============================= + +stuff +~~~~~ +lel \ No newline at end of file diff --git a/docs-minetester/source/tutorials/train_RL_agent.rst b/docs-minetester/source/tutorials/train_RL_agent.rst index 6c7bffb84b4e3..17bbbcd55b1bb 100644 --- a/docs-minetester/source/tutorials/train_RL_agent.rst +++ b/docs-minetester/source/tutorials/train_RL_agent.rst @@ -1,2 +1,5 @@ -Training an RL agent -==================== \ No newline at end of file +Training an RL agent using the Minetest gymnasium environment +============================================================= + +In this tutorial we will be using +`Minetest Baselines `_ \ No newline at end of file diff --git a/minetester/utils.py b/minetester/utils.py index a5902c8ddf69b..794931d7957eb 100644 --- a/minetester/utils.py +++ b/minetester/utils.py @@ -229,7 +229,7 @@ def start_xserver( display_size: Tuple[int, int] = (1024, 600), display_depth: int = 24, ) -> subprocess.Popen: - """Start an X server. + """Start a virtual framebuffer X server. Args: display_idx: Value of the DISPLAY variable. diff --git a/scripts/sync_server.sh b/scripts/sync_server.sh index 87812a1107b39..25a8d303956ab 100755 --- a/scripts/sync_server.sh +++ b/scripts/sync_server.sh @@ -1,2 +1,2 @@ #!/bin/bash -exec bin/minetest --server --world newworld --gameid minetest --sync-port 30010 --sync-dtime 0.1 --config hacking_testing/minetest.conf +exec bin/minetest --server --world newworld --gameid minetest --sync-port 30010 --sync-dtime 0.1 --config scripts/minetest.conf From 7915e133f3c40ef11d41541a11fabd752158d3a0 Mon Sep 17 00:00:00 2001 From: rk1a Date: Fri, 14 Jul 2023 20:15:02 +0200 Subject: [PATCH 05/26] WIP sphinx docs --- .../source/community/CODE_OF_CONDUCT.md | 128 ++++++++++++++++++ .../source/community/contributing.rst | 49 ++++++- docs-minetester/source/community/support.rst | 5 +- docs-minetester/source/conf.py | 6 + docs-minetester/source/index.rst | 1 - .../source/quickstart/hello_minetest.rst | 29 +++- .../source/quickstart/what_is_minetester.rst | 2 +- .../source/tutorials/train_RL_agent.rst | 5 - 8 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 docs-minetester/source/community/CODE_OF_CONDUCT.md delete mode 100644 docs-minetester/source/tutorials/train_RL_agent.rst diff --git a/docs-minetester/source/community/CODE_OF_CONDUCT.md b/docs-minetester/source/community/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000..efdecf5e89f5f --- /dev/null +++ b/docs-minetester/source/community/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +contact@eleuther.ai. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/docs-minetester/source/community/contributing.rst b/docs-minetester/source/community/contributing.rst index f2a1cb2b2490f..072382fb5c44e 100644 --- a/docs-minetester/source/community/contributing.rst +++ b/docs-minetester/source/community/contributing.rst @@ -1,8 +1,53 @@ Contributions guide =================== -Developers should install the repo in edit mode: +Thank you for your interest in contributing to Minetester! This guide is meant to provide potential contributors with a clear overview +of how they can participate in the project, adhere to the community standards, and enhance the codebase. +All contributions, no matter how minor, are greatly appreciated. +All contributors are expected to adhere to the project's `Code of Conduct `_. + +Getting Started +--------------- + +- `Fork the repository on GitHub `_. + +- Clone the repository: + +.. code-block:: bash + + git clone https://github.com/your_username/minetest.git + +- Follow the `build instructions <../advanced/building_source.html>`_ to build from your fork's source code. +- In addition, you should install development tools and pre-commit: .. code-block:: bash - pip install -e .[dev] \ No newline at end of file + pip install -e .[dev] + pre-commit install + +Making a Contribution +--------------------- + +- Look through the `GitHub Issues `_ to find one you are interested in addressing. You're also free to create your own issue if you identify a bug or enhancement that hasn't been raised yet. + +- Create a new branch with a somewhat informative name of what you are doing. + +.. code-block:: bash + + git checkout -b your_branch_name + +- Develop your bug fix or feature and write tests if possible. + +- Commit your changes: + +.. code-block:: bash + + git commit -m "Brief description of changes" + +- Push your changes to your fork. + +.. code-block:: bash + + git push origin your_branch_name + +- Open a pull request in the original repository. Please fill in the pull request template and reference the issue you're addressing. Feel free to request a review from one of the maintainers. diff --git a/docs-minetester/source/community/support.rst b/docs-minetester/source/community/support.rst index 4522fc0e59a66..5e7d9ebc5246c 100644 --- a/docs-minetester/source/community/support.rst +++ b/docs-minetester/source/community/support.rst @@ -1,2 +1,5 @@ Support -======= \ No newline at end of file +======= + +- `Discord Channel `_ +- `GitHub Issues `_ \ No newline at end of file diff --git a/docs-minetester/source/conf.py b/docs-minetester/source/conf.py index 868f41567a0a9..333d501371cce 100644 --- a/docs-minetester/source/conf.py +++ b/docs-minetester/source/conf.py @@ -30,6 +30,7 @@ 'sphinx.ext.viewcode', 'sphinx_copybutton', 'sphinx.ext.autosummary', + 'myst_parser', ] # ... napoleon_google_docstring = True @@ -41,6 +42,11 @@ "special-members": "__init__", "show-inheritance": True, } +source_suffix = { + ".rst": "restructuredtext", + ".txt": "markdown", + ".md": "markdown", +} # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs-minetester/source/index.rst b/docs-minetester/source/index.rst index f2cc6f9f9e83b..6a3aba5c9ee36 100644 --- a/docs-minetester/source/index.rst +++ b/docs-minetester/source/index.rst @@ -23,7 +23,6 @@ Welcome to Minetester's documentation! tutorials/headless_mode tutorials/synchronization tutorials/create_task - tutorials/train_RL_agent .. toctree:: :maxdepth: 1 diff --git a/docs-minetester/source/quickstart/hello_minetest.rst b/docs-minetester/source/quickstart/hello_minetest.rst index 355d134135ef2..49a95dcdabdd5 100644 --- a/docs-minetester/source/quickstart/hello_minetest.rst +++ b/docs-minetester/source/quickstart/hello_minetest.rst @@ -59,9 +59,9 @@ Minetester command line options * - Minetester CLI Option - Description * - ``--dumb`` - - Start a dumb client that needs to connect to an external controller client. + - Start a dumb client that can receive input from an external controller client. * - ``--record`` - - Start a recording client that needs to connect to an external data gathering client. + - Start a recording client that outputs observations to an external data gathering client. * - ``--client-address`` - Address to the controller / data gathering client. * - ``--headless`` @@ -82,13 +82,30 @@ To learn more about the other CLI options please refer to the :ref:`tutorials `_ to support training and evaluation of AI / RL agents. The Minetest core was modified to add the following main features: -- a dummy client that adds an `API for controlling a player and receiving game information <../advanced/client_api.html>`_ +- a dumb client that adds an `API for controlling a player and receiving game information <../advanced/client_api.html>`_ - support for `custom tasks using the Minetest modding API <../tutorials/create_task.html>`_ - `headless client operation <../tutorials/headless_mode.html>`_ - `client-server synchronization <../tutorials/synchronization.html>`_ diff --git a/docs-minetester/source/tutorials/train_RL_agent.rst b/docs-minetester/source/tutorials/train_RL_agent.rst deleted file mode 100644 index 17bbbcd55b1bb..0000000000000 --- a/docs-minetester/source/tutorials/train_RL_agent.rst +++ /dev/null @@ -1,5 +0,0 @@ -Training an RL agent using the Minetest gymnasium environment -============================================================= - -In this tutorial we will be using -`Minetest Baselines `_ \ No newline at end of file From b58e6d8be85af9db21b8b3a3ae3bd07efda42c89 Mon Sep 17 00:00:00 2001 From: rk1a Date: Fri, 14 Jul 2023 21:32:12 +0200 Subject: [PATCH 06/26] Fix no curser image segfaulting and make dumb client recording by default --- src/client/clientlauncher.cpp | 2 ++ src/client/game.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/clientlauncher.cpp b/src/client/clientlauncher.cpp index 9de2136f97d4b..82b13d30517bd 100644 --- a/src/client/clientlauncher.cpp +++ b/src/client/clientlauncher.cpp @@ -350,6 +350,8 @@ void ClientLauncher::init_args(GameStartData &start_data, const Settings &cmd_ar if (cmd_args.exists("cursor-image")) start_data.cursor_image_path = cmd_args.get("cursor-image"); + else + start_data.cursor_image_path = ""; if (dumb && cmd_args.exists("sync-port")) start_data.sync_port = cmd_args.get("sync-port"); diff --git a/src/client/game.cpp b/src/client/game.cpp index 689fa92747688..13cfe64529a38 100644 --- a/src/client/game.cpp +++ b/src/client/game.cpp @@ -905,7 +905,7 @@ class Game { zmqpp::socket* sync_socket = nullptr; // cursor image used in Gui recording - irr::video::IImage* cursorImage; + irr::video::IImage* cursorImage = nullptr; IWritableTextureSource *texture_src = nullptr; IWritableShaderSource *shader_src = nullptr; @@ -1224,7 +1224,7 @@ bool Game::startup(bool *kill, if (start_data.isDumbClient()) { dynamic_cast(input)->socket = data_socket; } - if (start_data.isRecording()) { + if (start_data.isRecording() || start_data.isDumbClient()) { createRecorder(start_data); recorder->sender = data_socket; } From f03ceaaee9e171d69bc81e0cf25480aaa2d6744d Mon Sep 17 00:00:00 2001 From: rk1a Date: Fri, 14 Jul 2023 21:32:52 +0200 Subject: [PATCH 07/26] WIP documentation --- .../source/quickstart/hello_minetest.rst | 37 +++++++++---------- .../quickstart/random_controller_loop.py | 9 +++++ 2 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 docs-minetester/source/quickstart/random_controller_loop.py diff --git a/docs-minetester/source/quickstart/hello_minetest.rst b/docs-minetester/source/quickstart/hello_minetest.rst index 49a95dcdabdd5..5e0c2204b05f9 100644 --- a/docs-minetester/source/quickstart/hello_minetest.rst +++ b/docs-minetester/source/quickstart/hello_minetest.rst @@ -22,7 +22,7 @@ The most important directories are these: Executing Minetest ------------------ -You can jump directly into the game (skipping the main menu) by running +You can jump directly into a single player game (skipping the main menu) by running .. code-block:: bash @@ -59,9 +59,9 @@ Minetester command line options * - Minetester CLI Option - Description * - ``--dumb`` - - Start a dumb client that can receive input from an external controller client. + - Start a dumb client that can receive actions from and return observations to an external controller client. * - ``--record`` - - Start a recording client that outputs observations to an external data gathering client. + - Start a recording client that returns observations to an external data gathering client. * - ``--client-address`` - Address to the controller / data gathering client. * - ``--headless`` @@ -75,7 +75,7 @@ Minetester command line options * - ``--cursor-image`` - Path to cursor image file that is rendered at the mouse position in the dumb client mode. -To get started the remainder of this guide focusses on connecting a **server**, a **dumb client** and the `builtin Python controller client wrapped as gymnasium environment. <../_api/minetester.minetest_env.html#minetester.minetest_env.Minetest>`_ +Below we focus on connecting a **server**, a **dumb client** and the `builtin Python controller client wrapped as gymnasium environment. <../_api/minetester.minetest_env.html#minetester.minetest_env.Minetest>`_ To learn more about the other CLI options please refer to the :ref:`tutorials `. @@ -86,28 +86,25 @@ After manually starting a server and a dumb client via .. code-block:: bash - ./bin/minetest --server - ./bin/minetest --name Bob --password whyisthisnecessary --address 0.0.0.0 --port 30000 --go --dumb --record --client-address "tcp://localhost:5555" + # remember how a server is started internally in singleplayer? + ./bin/minetest --go --dumb --client-address "tcp://localhost:5555" you can use :py:class:`minetester.minetest_env.Minetest` as a Python controller client, e.g. by running the following script: -.. code-block:: python - - from minetester import Minetest - - mt = Minetest(seed=0, start_minetest=False) - mt.reset() +.. literalinclude:: random_controller_loop.py + :language: python + :linenos: - while True: - action = mt.action_space.sample() - mt.step(action) - mt.render() +You should see two windows pop up: one from the Minetest client and one from the Python controller rendering the received observations. +.. note:: -By default ``start_minetest=True`` such that server and dumb client are started automatically. + By default ``start_minetest=True`` such that server and dumb client are started automatically as subprocesses. -Further Resources ------------------ +Further resources about Minetest +-------------------------------- -- `minetest.net `_ \ No newline at end of file +- `minetest.net `_ +- `Minetest Wiki `_ +- `Minetest Forum `_ \ No newline at end of file diff --git a/docs-minetester/source/quickstart/random_controller_loop.py b/docs-minetester/source/quickstart/random_controller_loop.py new file mode 100644 index 0000000000000..d85854db6959d --- /dev/null +++ b/docs-minetester/source/quickstart/random_controller_loop.py @@ -0,0 +1,9 @@ +from minetester import Minetest + +mt = Minetest(seed=0, start_minetest=False) +mt.reset() + +while True: + action = mt.action_space.sample() + mt.step(action) + mt.render() \ No newline at end of file From 46a98fc90060de8029dbb669fe0c448635fa8970 Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 17 Jul 2023 01:44:01 +0200 Subject: [PATCH 08/26] Improves docs and adds page on sync mode --- .../source/advanced/building_source.rst | 2 +- .../source/quickstart/hello_minetest.rst | 7 +- .../quickstart/random_controller_loop.py | 6 +- .../source/tutorials/create_task.rst | 82 +++++++++---------- .../source/tutorials/synchronization.rst | 30 ++++++- lib/irrlichtmt | 2 +- lib/zmqpp | 2 +- 7 files changed, 77 insertions(+), 54 deletions(-) mode change 160000 => 120000 lib/irrlichtmt mode change 160000 => 120000 lib/zmqpp diff --git a/docs-minetester/source/advanced/building_source.rst b/docs-minetester/source/advanced/building_source.rst index 10c5a51f5746a..f9b33914b3e8d 100644 --- a/docs-minetester/source/advanced/building_source.rst +++ b/docs-minetester/source/advanced/building_source.rst @@ -1,4 +1,4 @@ -Building Minetester from source +Manual build from source =============================== .. literalinclude:: ../../../build_instructions.txt diff --git a/docs-minetester/source/quickstart/hello_minetest.rst b/docs-minetester/source/quickstart/hello_minetest.rst index 5e0c2204b05f9..a53e3910d45e7 100644 --- a/docs-minetester/source/quickstart/hello_minetest.rst +++ b/docs-minetester/source/quickstart/hello_minetest.rst @@ -82,7 +82,7 @@ To learn more about the other CLI options please refer to the :ref:`tutorials `_. - Task definition --------------- @@ -18,70 +17,65 @@ A basic Minetest mod is defined by a directory with the following structure settingtypes.txt where `mod.conf` contains meta data, `init.lua` the necessary Lua code, and `settingtypes.txt` possible mod settings. +It is either located in the **clientmods** or the **mods** directory (see :ref:`below `). -A task has to define two global variables in the Lua script environment: +A task has to define or modify the following two global variables in the Lua script environment: -1. `REWARD`: a scalar floting point number indicating the reward received at the current step. -2. `TERMINAL`: a boolean indicating whether the agent has reached a terminal state of the task. +1. ``REWARD``: a scalar floting point number indicating the reward received at the current step. +2. ``TERMINAL``: a boolean indicating whether the agent has reached a terminal state of the task. The variables can be changed at every step, or based on in-game events. +.. note:: + + In order to avoid multiple definitions of ``REWARD`` and ``TERMINAL`` when using multiple task mods together, + there is a default mod called **rewards** (see `client-side `_, + `server-side `_) that takes care of the definition. If a mod with this name is found, + it will be automatically loaded. + Example: A simple treechop task ------------------------------- +The following files define a simple task, ``treechop-v0``, that rewards chopping trees and terminates after +a certain number of tree nodes have been chopped. + `mod.conf` -.. literalinclude :: ../../../mods/treechop_v2/mod.conf +.. literalinclude :: ../../../clientmods/treechop_v0/mod.conf :language: text + :linenos: `settingtypes.txt` -.. literalinclude :: ../../../mods/treechop_v2/settingtypes.txt +.. literalinclude :: ../../../clientmods/treechop_v0/settingtypes.txt :language: text + :linenos: `ìnit.lua` -.. literalinclude :: ../../../mods/treechop_v2/init.lua +.. literalinclude :: ../../../clientmods/treechop_v0/init.lua :language: lua + :linenos: + +.. _client_server_mods: + +Client and server mods +---------------------- + +Minetest provides two mod types, client-side mods located in the **clientmods** directory and server-side mods located in **mods**. +In the default, asynchronous client-server operation tasks are specified as client-side mods, meaning each client +tracks its own reward and task termination variables. +One downside of client-side mods is that they don't have access to all information that is available on the server-side, +e.g. the inventory of the player. +In order to still obtain this information one can have an additional server-side mod and make use of so-called mod channels +to communicate the required data (see ``treechop-v1``: `client-side `_, +`server-side `_). -.. code-block:: lua - - -- task settings - TREECHOP_GOAL = minetest.settings:get("treechop_goal") or 10 - - minetest.register_on_dignode(function(pos, node, digger) - if string.find(node["name"], "tree") then - minetest.debug("Dug a tree!") - REWARD[digger:get_player_name()] = 1.0 - end - - -- count the number of tree items of digging player - local num_tree_items = 0 - local inv = digger:get_inventory() - local size = inv:get_size("main") - for i = 1, size do - local stack = inv:get_stack("main", i) - if string.find(stack:get_name(), "tree") then - num_tree_items = num_tree_items + stack:get_count() - end - end - if num_tree_items >= TREECHOP_GOAL then - minetest.debug(digger:get_player_name() .. " reached the goal!") - TERMINAL[digger:get_player_name()] = true - end - end) - - -Asynchronous mode and client mods ---------------------------------- - -Currently, server mods can only be used to modify the Minetester global variables if client-server synchronization is active. -Otherwise, a client mod has to be used instead. -Client mods have certain limitations to what information they can access. -For example, there is no acces to a player's inventory. -To circumvent these limitations, one can use a pair of mods (one client and one server mod) that establish a mod channel between them to share the missing information. -However, it is recommended to use client-server synchronization with a single server mod where possible. +On the other hand, when using `client-server synchronization `_, task data is distributed to the client +via the synchronization channel, such that **it is required** to exclusively use server mods for the task definition. +In this case, ``REWARD`` and ``TERMINAL`` are each tables of floats and booleans, respectively, containing values for each player name +(see ``treechop-v2``: `server-side only `_) Further resources ----------------- diff --git a/docs-minetester/source/tutorials/synchronization.rst b/docs-minetester/source/tutorials/synchronization.rst index 7f96981491ab3..265377989f9f9 100644 --- a/docs-minetester/source/tutorials/synchronization.rst +++ b/docs-minetester/source/tutorials/synchronization.rst @@ -1,6 +1,30 @@ Client-server synchronization ============================= -stuff -~~~~~ -lel \ No newline at end of file + +Motivation +---------- +Solid support of client-server synchronization is useful because of the following reasons: + +1. reproducibility of experiments + a. elimination of latency related differences in game state + b. direct loading of missing chunks (additional thread of the server) +2. eliminating communication overhead in the single-agent setting by running client and server in the same process +3. global shared game state in multi-agent setting + + +Current implementation +---------------------- + +The current implementation of client-server synchronization only covers case 1. a. and can be considered an experimental feature. +Instead of running the server process asynchronously it adds a message queue between client and server to operate them in lock step. +It makes the following assumption: + +- single agent (client) +- both processes run on the same machine +- task is specified in a server mod (see `task tutorial `_) + +There are two CLI arguments that need to be configured to make use of synchronization: + +1. ``--sync-port``: Internal port used for syncing server and client. Make sure to pass the same port to both processes. +2. ``--sync-dtime``: Ingame time difference between steps when using server-client synchronization. For example, sync-dtime = 0.05 seconds => 20 steps per second. The default walking speed in Minetest is 4 nodes / second, i.e. 0.2 nodes / step for this setting of sync-dtime. \ No newline at end of file diff --git a/lib/irrlichtmt b/lib/irrlichtmt deleted file mode 160000 index 3b7fa4a8f6f48..0000000000000 --- a/lib/irrlichtmt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3b7fa4a8f6f48c7e94bd01c939a475ac6f8bae68 diff --git a/lib/irrlichtmt b/lib/irrlichtmt new file mode 120000 index 0000000000000..f0c2d62bfbc2a --- /dev/null +++ b/lib/irrlichtmt @@ -0,0 +1 @@ +../../irrlicht/ \ No newline at end of file diff --git a/lib/zmqpp b/lib/zmqpp deleted file mode 160000 index ba4230d5d03d2..0000000000000 --- a/lib/zmqpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ba4230d5d03d29ced9ca788e3bd1095477db08ae diff --git a/lib/zmqpp b/lib/zmqpp new file mode 120000 index 0000000000000..3e455fb3b1892 --- /dev/null +++ b/lib/zmqpp @@ -0,0 +1 @@ +../../zmqpp \ No newline at end of file From 64bb27431058dfdc0813405b0c71246c1eef56d9 Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 17 Jul 2023 17:47:07 +0200 Subject: [PATCH 09/26] Adds client API doc and fixes --- .../source/advanced/building_source.rst | 5 --- .../source/advanced/client_api.rst | 34 +++++++++++++++++-- docs-minetester/source/index.rst | 1 - .../source/quickstart/hello_minetest.rst | 5 +-- .../source/quickstart/installation.rst | 13 ++----- .../source/tutorials/create_task.rst | 9 ++--- 6 files changed, 43 insertions(+), 24 deletions(-) delete mode 100644 docs-minetester/source/advanced/building_source.rst diff --git a/docs-minetester/source/advanced/building_source.rst b/docs-minetester/source/advanced/building_source.rst deleted file mode 100644 index f9b33914b3e8d..0000000000000 --- a/docs-minetester/source/advanced/building_source.rst +++ /dev/null @@ -1,5 +0,0 @@ -Manual build from source -=============================== - -.. literalinclude:: ../../../build_instructions.txt - :language: text \ No newline at end of file diff --git a/docs-minetester/source/advanced/client_api.rst b/docs-minetester/source/advanced/client_api.rst index 7ab4ba0452308..b79028071e503 100644 --- a/docs-minetester/source/advanced/client_api.rst +++ b/docs-minetester/source/advanced/client_api.rst @@ -1,5 +1,35 @@ Client API ========== -The client API uses zmq for communication. -Protobuf objects are passed between client and gym environment. +The API provided for controlling and reading out the Minetest client is based on two libraries: + +- `Protocol Buffers `_ +- `ZeroMQ `_ + +Protbuf objects are used for serializing and deserializing the messages, that are being sent using ZeroMQ. + +Protobuf objects +---------------- + +The objects are defined in `/proto/objects.proto `_. +The Minetest client receives ``Action`` messages and sends out ``Observation`` messages. + +By default the Protobuf objects are compiled for C++ and Python using the +`/scripts/compile_proto.sh `_ script. +This can be easily adjusted to compile for other languages. + +ZeroMQ message patterns +----------------------- + +There are two differnt ZeroMQ messaging patterns in use depending on the type of the client: + +- dumb Minetest client <-> controller client: ``REQ/REP`` + + In each step the dumb client sends a blocking request containing an observation to the controller + which awaits the request and replies with an action. +- recording Minetest client <-> data gathering client : ``PUB/SUB`` + + In each step the recording client publishes an observation and the data gathering client + subscribes to the topic in order to read out the observations. + See `this script `_ + for an example of how to implement a data gathering client. diff --git a/docs-minetester/source/index.rst b/docs-minetester/source/index.rst index 6a3aba5c9ee36..95009ecffacc0 100644 --- a/docs-minetester/source/index.rst +++ b/docs-minetester/source/index.rst @@ -29,7 +29,6 @@ Welcome to Minetester's documentation! :caption: Advanced Usage advanced/client_api - advanced/building_source .. toctree:: :maxdepth: 1 diff --git a/docs-minetester/source/quickstart/hello_minetest.rst b/docs-minetester/source/quickstart/hello_minetest.rst index a53e3910d45e7..29bf01ae9ce34 100644 --- a/docs-minetester/source/quickstart/hello_minetest.rst +++ b/docs-minetester/source/quickstart/hello_minetest.rst @@ -75,7 +75,7 @@ Minetester command line options * - ``--cursor-image`` - Path to cursor image file that is rendered at the mouse position in the dumb client mode. -Below we focus on connecting a **server**, a **dumb client** and the `builtin Python controller client wrapped as gymnasium environment. <../_api/minetester.minetest_env.html#minetester.minetest_env.Minetest>`_ +Below we focus on connecting a **server**, a **dumb client** and `Minetester's gymnasium environment. <../_api/minetester.minetest_env.html#minetester.minetest_env.Minetest>`_ To learn more about the other CLI options please refer to the :ref:`tutorials `. @@ -90,7 +90,8 @@ Let's manually start a server and a dumb client via ./bin/minetest --go --dumb --client-address "tcp://localhost:5555" -We can set up a little Python script that acts as controller client using :py:class:`minetester.minetest_env.Minetest`. +We can set up a little Python script that acts as controller client using :py:class:`minetester.minetest_env.Minetest`, +which implements the gymnasium environment interface. .. literalinclude:: random_controller_loop.py :language: python diff --git a/docs-minetester/source/quickstart/installation.rst b/docs-minetester/source/quickstart/installation.rst index fa68c9cb88748..dcf8c7c4afc6f 100644 --- a/docs-minetester/source/quickstart/installation.rst +++ b/docs-minetester/source/quickstart/installation.rst @@ -1,16 +1,9 @@ Installation ============ -For normal usage it suffices to install pre-built binaries via pip: +Follow the build instructions: -.. code-block:: bash - - pip install minetester - -Verify your installation by running - -.. code-block:: bash - - python -m minetester.scripts.test_loop +.. literalinclude:: ../../../build_instructions.txt + :language: text You should see a window pop up with a player performing random actions in Minetest. diff --git a/docs-minetester/source/tutorials/create_task.rst b/docs-minetester/source/tutorials/create_task.rst index 1e6ee89f5a415..8f3436dbae4f9 100644 --- a/docs-minetester/source/tutorials/create_task.rst +++ b/docs-minetester/source/tutorials/create_task.rst @@ -2,7 +2,8 @@ How to create a new task? ================================ Tasks are easy to create using Minetest's builtin Lua modding API. -The task-relevant information (rewards, task completion status, meta information, ...) are passed through the Minetest processes and made available to the `client API `_. +The task-relevant information (rewards, task completion status, meta information, ...) are passed through +the Minetest processes and made available to the `client API <../advanced/client_api.html>`_. Task definition --------------- @@ -21,7 +22,7 @@ It is either located in the **clientmods** or the **mods** directory (see :ref: A task has to define or modify the following two global variables in the Lua script environment: -1. ``REWARD``: a scalar floting point number indicating the reward received at the current step. +1. ``REWARD``: a scalar floating point number indicating the reward received at the current step. 2. ``TERMINAL``: a boolean indicating whether the agent has reached a terminal state of the task. The variables can be changed at every step, or based on in-game events. @@ -29,8 +30,8 @@ The variables can be changed at every step, or based on in-game events. .. note:: In order to avoid multiple definitions of ``REWARD`` and ``TERMINAL`` when using multiple task mods together, - there is a default mod called **rewards** (see `client-side `_, - `server-side `_) that takes care of the definition. If a mod with this name is found, + there is a default mod called **rewards** (see `client-side rewards mod `_, + `server-side rewards mod `_) that takes care of the definition. If a mod with this name is found, it will be automatically loaded. Example: A simple treechop task From e81b7a96644d6a1b71f7ff6e5c0179d100fec8ba Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 17 Jul 2023 21:01:00 +0200 Subject: [PATCH 10/26] Update sphinx versions and README --- README.md | 9 +++++++-- docs-minetester/source/advanced/client_api.rst | 2 ++ docs-minetester/source/conf.py | 1 - docs-minetester/source/tutorials/create_task.rst | 2 +- setup.py | 11 +++++++++-- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bebc0019067fc..00a3408754825 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,18 @@ Minetester ========== Minetester is the Python package that exposes Minetest environments via the `gym(nasium)` interface. After [building the minetest executable](https://github.com/EleutherAI/minetest/blob/develop/build_instructions.txt) you can install it with: -``` +``` bash pip install -e . ``` To verify the installation run -``` +``` bash python -m minetester.scripts.test_loop ``` +To build the documentation in `docs-minetester` please run +``` bash +pip install -e .[docs] +cd docs-minetester && make livehtml +``` Minetest ======== diff --git a/docs-minetester/source/advanced/client_api.rst b/docs-minetester/source/advanced/client_api.rst index b79028071e503..527589647e6dd 100644 --- a/docs-minetester/source/advanced/client_api.rst +++ b/docs-minetester/source/advanced/client_api.rst @@ -33,3 +33,5 @@ There are two differnt ZeroMQ messaging patterns in use depending on the type of subscribes to the topic in order to read out the observations. See `this script `_ for an example of how to implement a data gathering client. + +In both cases the ZMQ socket address is passed to the Minetest client via the ``--client-address`` command line argument. diff --git a/docs-minetester/source/conf.py b/docs-minetester/source/conf.py index 333d501371cce..117f6ae884ddc 100644 --- a/docs-minetester/source/conf.py +++ b/docs-minetester/source/conf.py @@ -26,7 +26,6 @@ 'sphinx_autodoc_typehints', 'sphinx.ext.napoleon', 'sphinx_rtd_theme', - 'sphinx_autodoc_typehints', 'sphinx.ext.viewcode', 'sphinx_copybutton', 'sphinx.ext.autosummary', diff --git a/docs-minetester/source/tutorials/create_task.rst b/docs-minetester/source/tutorials/create_task.rst index 8f3436dbae4f9..cab9f5a9247d7 100644 --- a/docs-minetester/source/tutorials/create_task.rst +++ b/docs-minetester/source/tutorials/create_task.rst @@ -69,7 +69,7 @@ In the default, asynchronous client-server operation tasks are specified as clie tracks its own reward and task termination variables. One downside of client-side mods is that they don't have access to all information that is available on the server-side, e.g. the inventory of the player. -In order to still obtain this information one can have an additional server-side mod and make use of so-called mod channels +In order to still obtain this information one can have an additional server-side mod and make use of so-called **mod channels** to communicate the required data (see ``treechop-v1``: `client-side `_, `server-side `_). diff --git a/setup.py b/setup.py index bcfc933fcb1fb..76963c034244c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,14 @@ from setuptools import setup DEV = ["pre-commit", "black", "isort", "flake8"] -DOCS = ["sphinx", "sphinx_rtd_theme", "sphinx-autobuild"] +DOCS = [ + "sphinx==6.2.1", + "sphinx_rtd_theme==1.2.2", + "sphinx-autobuild", + "sphinx_autodoc_typehints", + "sphinx_copybutton", + "myst_parser", +] setup( name='Minetester', @@ -18,5 +25,5 @@ 'protobuf==3.20.1', 'psutil', ], - extras_require={"dev": DEV} + extras_require={"dev": DEV, "docs": DOCS} ) From 46db1ebd2a463aaee6780e40fb7fc07e16b8a74f Mon Sep 17 00:00:00 2001 From: rk1a Date: Sun, 3 Dec 2023 16:52:23 +0100 Subject: [PATCH 11/26] Fix docstrings --- minetester/minetest_env.py | 81 +++++++++++++++++------ minetester/scripts/gymnasium_api_check.py | 3 +- minetester/scripts/test_loop_parallel.py | 3 +- minetester/utils.py | 21 +++++- 4 files changed, 84 insertions(+), 24 deletions(-) diff --git a/minetester/minetest_env.py b/minetester/minetest_env.py index a15d60dfff814..e26509ee0ead0 100644 --- a/minetester/minetest_env.py +++ b/minetester/minetest_env.py @@ -1,4 +1,4 @@ -"""Minetest gym environment.""" +"""Minetest Gymnasium Environment.""" import datetime import logging import os @@ -25,6 +25,8 @@ class Minetest(gym.Env): + """Minetest Gymnasium Environment.""" + metadata = {"render_modes": ["rgb_array", "human"], "render_fps": 20} default_display_size = (1024, 600) @@ -59,25 +61,29 @@ def __init__( Args: env_port: Port between gym environment and a Minetest client server_port: Port between Minetest client and server - minetest_executable: Path to Minetest executable - log_dir: Path to log directory + minetest_root: Path to Minetest root + artefact_dir: Artefact directory, e.g. for logs, media cache, etc. config_path: Path to minetest.conf - cursor_image_path: Path to cursor image for menu and inventory world_dir: Path to Minetest world directory display_size: Size in pixels of the Minetest window fov: Field of view in degrees of the Minetest window - seed: Seed for the Minetest world + base_seed: Seed for the Minetest environment. + world_seed: Fixed seed for world generation. If not set, world seeds + are randomly generated from `base_seed` at each `reset` start_minetest: Whether to start Minetest server and client or connect to existing processes game_id: Name of the Minetest game + client_name: Name of the client/player. clientmods: List of client mod names servermods: List of server mod names config_dict: Dictionary of config options updating the loaded config file sync_port: Port between Minetest client and server for synchronization sync_dtime: In-game time between two steps + dtime: Client-side in-game time between time steps headless: Whether to run Minetest in headless mode start_xvfb: Whether to start X server virtual framebuffer x_display: Display number to use for the X server virtual framebuffer + render_mode: Gymnasium render mode. Supports 'human' and 'rgb_array'. """ self.unique_env_id = str(uuid.uuid4()) @@ -90,10 +96,12 @@ def __init__( # Define Minetest paths self.start_xvfb = start_xvfb and self.headless self._set_artefact_dirs( - artefact_dir, world_dir, config_path + artefact_dir, + world_dir, + config_path, ) # Stores minetest artefacts and outputs self._set_minetest_dirs( - minetest_root + minetest_root, ) # Stores actual minetest dirs and executable # Whether to start minetest server and client @@ -194,7 +202,11 @@ def _configure_spaces(self): ) def _set_graphics( - self, headless: bool, display_size: Tuple[int, int], fov: int, render_mode: str + self, + headless: bool, + display_size: Tuple[int, int], + fov: int, + render_mode: str, ): # gymnasium render mode self.render_mode = render_mode @@ -210,7 +222,9 @@ def _set_minetest_dirs(self, minetest_root): # check for local install candidate_minetest_root = os.path.dirname(os.path.dirname(__file__)) candidate_minetest_executable = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "bin", "minetest" + os.path.dirname(os.path.dirname(__file__)), + "bin", + "minetest", ) if os.path.isfile(candidate_minetest_executable): self.minetest_root = candidate_minetest_root @@ -219,13 +233,14 @@ def _set_minetest_dirs(self, minetest_root): # check for package install try: candidate_minetest_executable = pkg_resources.resource_filename( - __name__, os.path.join("minetest", "bin", "minetest") + __name__, + os.path.join("minetest", "bin", "minetest"), ) if os.path.isfile(candidate_minetest_executable): self.minetest_root = os.path.dirname( - os.path.dirname(candidate_minetest_executable) + os.path.dirname(candidate_minetest_executable), ) - except Exception as e: + except Exception as e: # noqa: B902 logging.warning(f"Error loading resource file 'bin.minetest': {e}") if self.minetest_root is None: @@ -233,11 +248,15 @@ def _set_minetest_dirs(self, minetest_root): if not self.start_xvfb and self.headless: self.minetest_executable = os.path.join( - self.minetest_root, "bin", "minetest_headless" + self.minetest_root, + "bin", + "minetest_headless", ) else: self.minetest_executable = os.path.join( - self.minetest_root, "bin", "minetest" + self.minetest_root, + "bin", + "minetest", ) self.cursor_image_path = os.path.join( @@ -255,7 +274,8 @@ def _set_artefact_dirs(self, artefact_dir, world_dir, config_path): if config_path is None: self.clean_config = True self.config_path = os.path.join( - self.artefact_dir, f"{self.unique_env_id}.conf" + self.artefact_dir, + f"{self.unique_env_id}.conf", ) else: self.clean_config = True @@ -440,8 +460,21 @@ def _sample_world_seed(self): self.world_seed = self._np_random.integers(np.iinfo(np.int64).max) def reset( - self, seed: Optional[int] = None, options: Optional[Dict[str, Any]] = None + self, + seed: Optional[int] = None, + options: Optional[Dict[str, Any]] = None, ): + """Reset the environment. + + Args: + seed: Seed for the environment. + e.g. used for deriving seeds for world generation. + options: Currently unused. + + Returns: + Tuple of inital observation and empty info dictionary. + """ + del options self._seed(seed=seed) if self.start_minetest: if self.reset_world: @@ -471,8 +504,8 @@ def step( action: The action to perform. Returns: - The next observation, the reward, whether the episode is done, and - additional info. + The next observation, the reward, whether the episode is truncated, + or done, and additional info. """ # Send action if isinstance(action["MOUSE"], np.ndarray): @@ -502,7 +535,15 @@ def step( logging.debug(f"Received obs - {next_obs.shape}; reward - {rew}; info - {info}") return next_obs, rew, done, False, {"minetest_info": info} - def render(self): + def render(self) -> Optional[np.ndarray]: + """Render the environment. + + Returns: + Render image if `env.render_mode='rgb_array'`. + + Raises: + NotImplementedError: if `env.render_mode` not in ['human', 'rgb_array']. + """ if self.render_mode == "human": # TODO replace with pygame if self.render_img is None: @@ -527,7 +568,7 @@ def render(self): raise NotImplementedError( "You are calling 'render()' with an unsupported" f" render mode: '{self.render_mode}'. " - f"Supported modes: {self.metadata['render_modes']}" + f"Supported modes: {self.metadata['render_modes']}", ) def close(self) -> None: diff --git a/minetester/scripts/gymnasium_api_check.py b/minetester/scripts/gymnasium_api_check.py index 14bc1c34f165b..7e405f63513b0 100644 --- a/minetester/scripts/gymnasium_api_check.py +++ b/minetester/scripts/gymnasium_api_check.py @@ -1,7 +1,8 @@ +"""Gymnasium API checker for Minetest environment.""" import gymnasium as gym from gymnasium.utils.env_checker import check_env -import minetester +import minetester # noqa: F401 env = gym.make("Minetest-v0") diff --git a/minetester/scripts/test_loop_parallel.py b/minetester/scripts/test_loop_parallel.py index 637bc036184e2..c327bc69b4751 100644 --- a/minetester/scripts/test_loop_parallel.py +++ b/minetester/scripts/test_loop_parallel.py @@ -30,7 +30,8 @@ def _init(): ) # Assign random timelimit to check that resets work properly env = TimeLimit( - env, max_episode_steps=random.randint(max_steps // 2, max_steps) + env, + max_episode_steps=random.randint(max_steps // 2, max_steps), ) return env diff --git a/minetester/utils.py b/minetester/utils.py index b5dea483fc9a5..313e15acb638b 100644 --- a/minetester/utils.py +++ b/minetester/utils.py @@ -183,7 +183,9 @@ def start_minetest_client( server_port: Port of the server to connect to. cursor_img: Path to the cursor image. client_name: Name of the client. + media_cache_dir: Directory of minetest's media cache. sync_port: Port for the synchronization with the server. + dtime: In-game time step in seconds. headless: Whether to run the client in headless mode. display: value of the DISPLAY variable. set_gpu_vars: whether to enable Nvidia GPU usage @@ -267,7 +269,15 @@ def start_xserver( return xserver_process -def read_config_file(file_path): +def read_config_file(file_path: os.PathLike): + """Read and parse minetest config files. + + Args: + file_path: Path to minetest config. + + Returns: + dictionary containing the parsed config. + """ config = {} with open(file_path, "r") as f: for line in f: @@ -288,7 +298,14 @@ def read_config_file(file_path): return config -def write_config_file(file_path, config): +def write_config_file(file_path: os.PathLike, config: Dict[str, Any]): + """Write a minetest config file. + + Args: + file_path: Write path for minetest config. + config: Dictionary representing minetest config. + + """ with open(file_path, "w") as f: for key, value in config.items(): f.write(f"{key} = {value}\n") From ef97cbe740df4dfb8c1e03b03ce88496fcccec79 Mon Sep 17 00:00:00 2001 From: rk1a Date: Sun, 3 Dec 2023 17:07:26 +0100 Subject: [PATCH 12/26] Fix build --- lib/{irrlichtmt~HEAD => irrlichtmt} | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/{irrlichtmt~HEAD => irrlichtmt} (100%) diff --git a/lib/irrlichtmt~HEAD b/lib/irrlichtmt similarity index 100% rename from lib/irrlichtmt~HEAD rename to lib/irrlichtmt diff --git a/setup.py b/setup.py index fa5e6325144c4..35eba934592aa 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ 'psutil', 'patchelf', ], - extras_require={"dev": DEV, "docs": DOCS} + extras_require={"dev": DEV, "docs": DOCS}, package_data={ 'minetester': [ 'minetest/bin/minetest', From fa846c5be1f5079545be9be58f96a395bb64a7f1 Mon Sep 17 00:00:00 2001 From: rk1a Date: Sun, 3 Dec 2023 18:06:38 +0100 Subject: [PATCH 13/26] Update documentation --- README.md | 2 +- build_instructions.txt | 66 ++++++------------- clientmods/mods.conf | 4 +- .../source/advanced/client_api.rst | 2 +- .../source/quickstart/hello_minetest.rst | 4 +- .../source/quickstart/installation.rst | 23 ++++++- minetester/minetest_env.py | 2 +- 7 files changed, 49 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index dc7b2ce1108d8..b547f3df56082 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ make clean #clean up build artifacts Additionally the makefile supports a utility to clean only the minetester install -``` +```bash make clean_minetester #remove existing minetester install make minetester #build minetester python library make install #install python library into local environment along with nessesary dependencies diff --git a/build_instructions.txt b/build_instructions.txt index ec95d41369ddb..b3db4471c39a0 100644 --- a/build_instructions.txt +++ b/build_instructions.txt @@ -1,46 +1,20 @@ -1. Install prereqs - 1. sudo apt-get install xvfb g++ make libzmq3-dev libtool pkg-config build-essential autoconf automake libc6-dev cmake libpng-dev libjpeg-dev libxi-dev libgl1-mesa-dev libsqlite3-dev libogg-dev libvorbis-dev libopenal-dev libcurl4-gnutls-dev libfreetype6-dev zlib1g-dev libgmp-dev libjsoncpp-dev libzstd-dev libluajit-5.1-dev protobuf-compiler - -2. Build SDL2 - 1. clone the SDL2 repo https://github.com/libsdl-org/SDL - 2. checkout release-2.26.2 https://github.com/libsdl-org/SDL/tree/release-2.26.2 - 3. create a build directory inside the SDL repo - 4. cd into it and run ../configure --prefix=/path/to/SDL/dir/build && make && make install - -3. Build zmqpp - 1. clone https://github.com/zeromq/zmqpp - 2. checkout the develop branch - 3. run make - -4. Clone EAI alignment minetest repos - 1. clone https://github.com/EleutherAI/minetest - 2. checkout the develop branch - 3. clone https://github.com/EleutherAI/irrlicht - 4. checkout headless-renderer - -5. Clone minetest game spec - 1. clone https://github.com/minetest/minetest_game - 2. checkout master branch - -5. Establish symlinks - 1. cd into minetest/lib - 2. rm -r zmqpp irrlichtmt - 3. ln -s ../../zmqpp/ zmqpp - 4. ln -s ../../irrlicht/ irrlichtmt - 5. cd into minetest/games - 6. ln -s ../../minetest_game/ minetest_game - -6. Build minetest - 1. cd into minetest - 2. either run - cmake . -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=1 -DSDL2_DIR=/SDL/build/lib/cmake/SDL2/ - or - cmake . -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=0 -DSDL2_DIR= - 3. run make -j$(nproc) - -7. setup python - 1. create and activate a new python conda env or venv (tested with python3.9) - 2. pip install gym matplotlib protobuf==3.20.1 psutil zmq -e ./minetest - 3. cd into the scripts directory and run compile_proto.sh - 4. run python -m minetester.scripts.test_loop - +# 1. Install debian dependencies, equivalent commands are nessesary for other distros +make deb_deps +# 2. Install build dependencies into the local python environment (we reccomend using a venv) +make python_build_deps +# 3. Init submodules +make repos +# 4. Build sdl2 +make sdl2 +# 5. Build zmqpp +make zmqpp +# 6. Create c++ and python protobuf files +make proto +# 7. Build minetest binary +make minetest +# 8. Build minetester python library +make minetester +# 9. Install minetester into local environment along with necessary dependencies +make install +# 10. Run the demo script +make demo \ No newline at end of file diff --git a/clientmods/mods.conf b/clientmods/mods.conf index 0b96651b71bd5..f517fccc8dfab 100644 --- a/clientmods/mods.conf +++ b/clientmods/mods.conf @@ -1,7 +1,7 @@ -load_mod_random_v0 = true load_mod_rewards = true load_mod_treechop_v1 = false load_mod_treechop_shaped_v0 = false load_mod_random_v1 = false -load_mod_preview = false load_mod_treechop_v0 = false +load_mod_random_v0 = false +load_mod_preview = false diff --git a/docs-minetester/source/advanced/client_api.rst b/docs-minetester/source/advanced/client_api.rst index 527589647e6dd..b51fb9cc9b996 100644 --- a/docs-minetester/source/advanced/client_api.rst +++ b/docs-minetester/source/advanced/client_api.rst @@ -6,7 +6,7 @@ The API provided for controlling and reading out the Minetest client is based on - `Protocol Buffers `_ - `ZeroMQ `_ -Protbuf objects are used for serializing and deserializing the messages, that are being sent using ZeroMQ. +Protobuf objects are used for serializing and deserializing the messages, that are being sent using ZeroMQ. Protobuf objects ---------------- diff --git a/docs-minetester/source/quickstart/hello_minetest.rst b/docs-minetester/source/quickstart/hello_minetest.rst index 29bf01ae9ce34..299009f8032a6 100644 --- a/docs-minetester/source/quickstart/hello_minetest.rst +++ b/docs-minetester/source/quickstart/hello_minetest.rst @@ -1,5 +1,5 @@ -Hello, Minetest(er)! -==================== +Hello, Minetester! +================== To get started with Minetester you should first know your way around the directory structure. The most important directories are these: diff --git a/docs-minetester/source/quickstart/installation.rst b/docs-minetester/source/quickstart/installation.rst index dcf8c7c4afc6f..23332d7037bb8 100644 --- a/docs-minetester/source/quickstart/installation.rst +++ b/docs-minetester/source/quickstart/installation.rst @@ -1,9 +1,30 @@ Installation ============ +Building submodules, minetest and minetester +-------------------------------------------- + Follow the build instructions: .. literalinclude:: ../../../build_instructions.txt - :language: text + :language: bash You should see a window pop up with a player performing random actions in Minetest. + +To clean build artifacts run: + +.. code-block:: bash + + make clean + +Rebuilding minetester +--------------------- + +Use the following instructions to only rebuild minetester (not minetest): + +.. code-block:: bash + + make clean_minetester + make minetester + make install + make demo \ No newline at end of file diff --git a/minetester/minetest_env.py b/minetester/minetest_env.py index e26509ee0ead0..4db8c553362ce 100644 --- a/minetester/minetest_env.py +++ b/minetester/minetest_env.py @@ -463,7 +463,7 @@ def reset( self, seed: Optional[int] = None, options: Optional[Dict[str, Any]] = None, - ): + ) -> Tuple[np.ndarray, Dict]: """Reset the environment. Args: From d54ac96fddeb497dc434b8dee011bb51a515db7a Mon Sep 17 00:00:00 2001 From: Robert Klassert <96744960+rk1a@users.noreply.github.com> Date: Sun, 24 Dec 2023 17:40:59 +0100 Subject: [PATCH 14/26] Create minetester.yml --- .github/workflows/minetester.yml | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/minetester.yml diff --git a/.github/workflows/minetester.yml b/.github/workflows/minetester.yml new file mode 100644 index 0000000000000..11dd5a85624a9 --- /dev/null +++ b/.github/workflows/minetester.yml @@ -0,0 +1,45 @@ +name: Build and test Minetester + +on: + push: + branches: [ "develop" ] + pull_request: + branches: [ "develop" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install Debian dependencies + run: make deb_deps + - name: Install build dependencies + run: make python_build_deps + - name: Init submodules + run: make repos + - name: Build SDL2 + run: make sdl2 + - name: Build zmqpp + run: make zmqpp + - name: Create Protobuf files + run: make proto + - name: Build minetest binary + run: make minetest + - name: Build minetester wheel + run: make minetester + - name: Install minetester along with dependencies + run: | + python -m pip install --upgrade pip + make install + - name: Run pre-commit hooks + run: pre-commit run --all-files From b82da26090be5249792918ac4797bcfd92f9eeb6 Mon Sep 17 00:00:00 2001 From: rk1a Date: Sun, 24 Dec 2023 18:29:49 +0100 Subject: [PATCH 15/26] Fix make target name --- .github/workflows/minetester.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/minetester.yml b/.github/workflows/minetester.yml index 11dd5a85624a9..749709fdd85ab 100644 --- a/.github/workflows/minetester.yml +++ b/.github/workflows/minetester.yml @@ -21,8 +21,8 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install Debian dependencies - run: make deb_deps + - name: Install Linux dependencies + run: make linux_deps - name: Install build dependencies run: make python_build_deps - name: Init submodules From 48df16f720d06108322d106d3deeddb95bfb7ec5 Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 27 Dec 2023 12:00:00 +0100 Subject: [PATCH 16/26] Use SDL2 branch --- .gitmodules | 2 +- lib/SDL | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 60fe8cd6917e6..11463af8a7ab9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,6 @@ path = lib/irrlichtmt url = https://github.com/EleutherAI/irrlicht.git branch = headless-rendering - [submodule "lib/zmqpp"] path = lib/zmqpp url = https://github.com/zeromq/zmqpp.git @@ -12,3 +11,4 @@ [submodule "lib/SDL"] path = lib/SDL url = https://github.com/libsdl-org/SDL.git + branch = SDL2 diff --git a/lib/SDL b/lib/SDL index 825d344756bc5..8030e7546a03f 160000 --- a/lib/SDL +++ b/lib/SDL @@ -1 +1 @@ -Subproject commit 825d344756bc5d67346a2e1156694b36e8d5ce2d +Subproject commit 8030e7546a03f08cdd3115d35e833030ebd45524 From 9db394f31eadc39c7f5b28f302c9990a1f06288b Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 27 Dec 2023 12:16:04 +0100 Subject: [PATCH 17/26] Cd back to root dir --- util/minetester/build_sdl2.sh | 1 + util/minetester/build_zmqpp.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/util/minetester/build_sdl2.sh b/util/minetester/build_sdl2.sh index b220e7e6c46d3..a509172afd616 100755 --- a/util/minetester/build_sdl2.sh +++ b/util/minetester/build_sdl2.sh @@ -2,3 +2,4 @@ cd lib/SDL mkdir build cd build ../configure --prefix=`pwd` && make && make install +cd ../../.. \ No newline at end of file diff --git a/util/minetester/build_zmqpp.sh b/util/minetester/build_zmqpp.sh index c244a2a829c36..9dcc2dfb58ea9 100755 --- a/util/minetester/build_zmqpp.sh +++ b/util/minetester/build_zmqpp.sh @@ -2,3 +2,4 @@ cd lib/zmqpp mkdir build cmake .. -G "Unix Makefiles" make +cd ../.. From 6da2f82ae68655137a064d867d83558d96d9a552 Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 27 Dec 2023 12:32:20 +0100 Subject: [PATCH 18/26] Add IrrlichtMt build script --- .github/workflows/minetester.yml | 2 ++ Makefile | 8 +++++++- util/minetester/build_irrlicht.sh | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 util/minetester/build_irrlicht.sh diff --git a/.github/workflows/minetester.yml b/.github/workflows/minetester.yml index 749709fdd85ab..a546921a04961 100644 --- a/.github/workflows/minetester.yml +++ b/.github/workflows/minetester.yml @@ -33,6 +33,8 @@ jobs: run: make zmqpp - name: Create Protobuf files run: make proto + - name: Build irrlicht + run: bash util/minetester/build_irrlicht.sh - name: Build minetest binary run: make minetest - name: Build minetester wheel diff --git a/Makefile b/Makefile index fba60893a26a2..6d88678923327 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ -.PHONY: all deps repos sdl2 package zmqpp minetester minetest install demo proto clean +.PHONY: all deps repos sdl2 package zmqpp irrlicht minetester minetest install demo proto clean MINETESTER_VERSION := 0.0.1 SDL2_CMAKE_FILE := lib/SDL/build/lib/cmake/SDL2/sdl2-config.cmake ZMQPP_LIB_FILE := lib/zmqpp/build/max-g++/libzmqpp.a +IRRLICHTMT_LIB_FILE := lib/irrlichtmt/lib/Linux/libIrrlichtMt.a MINETEST_BINARY := bin/minetest DEBUG_BINARY := bin/debug MINETESTER_WHEEL := build/package/wheel/minetester-$(MINETESTER_VERSION)-py3-none-manylinux_2_35_x86_64.whl @@ -38,6 +39,11 @@ $(ZMQPP_LIB_FILE): zmqpp: $(ZMQPP_LIB_FILE) +$(IRRLICHTMT_LIB_FILE): + # build IrrlichtMt + util/minetester/build_irrlicht.sh + +irrlicht: $(IRRLICHTMT_LIB_FILE) $(MINETEST_BINARY): #build minetest binary diff --git a/util/minetester/build_irrlicht.sh b/util/minetester/build_irrlicht.sh new file mode 100644 index 0000000000000..b254e93a1d813 --- /dev/null +++ b/util/minetester/build_irrlicht.sh @@ -0,0 +1,4 @@ +cd lib/irrlichtmt +cmake . -DBUILD_SHARED_LIBS=OFF +make -j$(nproc) +cd ../.. \ No newline at end of file From 7171bf76b079aa4b6a4da2bbe315cbf95335c538 Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 27 Dec 2023 12:32:20 +0100 Subject: [PATCH 19/26] Add IrrlichtMt build script --- .github/workflows/minetester.yml | 2 ++ Makefile | 7 ++++++- util/minetester/build_irrlicht.sh | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 util/minetester/build_irrlicht.sh diff --git a/.github/workflows/minetester.yml b/.github/workflows/minetester.yml index 749709fdd85ab..a546921a04961 100644 --- a/.github/workflows/minetester.yml +++ b/.github/workflows/minetester.yml @@ -33,6 +33,8 @@ jobs: run: make zmqpp - name: Create Protobuf files run: make proto + - name: Build irrlicht + run: bash util/minetester/build_irrlicht.sh - name: Build minetest binary run: make minetest - name: Build minetester wheel diff --git a/Makefile b/Makefile index fba60893a26a2..11cdb142c0ccf 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ -.PHONY: all deps repos sdl2 package zmqpp minetester minetest install demo proto clean +.PHONY: all deps repos sdl2 package zmqpp irrlicht minetester minetest install demo proto clean MINETESTER_VERSION := 0.0.1 SDL2_CMAKE_FILE := lib/SDL/build/lib/cmake/SDL2/sdl2-config.cmake ZMQPP_LIB_FILE := lib/zmqpp/build/max-g++/libzmqpp.a +IRRLICHTMT_LIB_FILE := lib/irrlichtmt/lib/Linux/libIrrlichtMt.a MINETEST_BINARY := bin/minetest DEBUG_BINARY := bin/debug MINETESTER_WHEEL := build/package/wheel/minetester-$(MINETESTER_VERSION)-py3-none-manylinux_2_35_x86_64.whl @@ -38,6 +39,10 @@ $(ZMQPP_LIB_FILE): zmqpp: $(ZMQPP_LIB_FILE) +$(IRRLICHTMT_LIB_FILE): + util/minetester/build_irrlicht.sh + +irrlicht: $(IRRLICHTMT_LIB_FILE) $(MINETEST_BINARY): #build minetest binary diff --git a/util/minetester/build_irrlicht.sh b/util/minetester/build_irrlicht.sh new file mode 100644 index 0000000000000..b254e93a1d813 --- /dev/null +++ b/util/minetester/build_irrlicht.sh @@ -0,0 +1,4 @@ +cd lib/irrlichtmt +cmake . -DBUILD_SHARED_LIBS=OFF +make -j$(nproc) +cd ../.. \ No newline at end of file From 223e0436af476a94ff6fdfc1c8b360a26ad905d9 Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 27 Dec 2023 12:54:42 +0100 Subject: [PATCH 20/26] Explicitly set irrlicht dir --- util/minetester/build_minetest.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/util/minetester/build_minetest.sh b/util/minetester/build_minetest.sh index f075a94f1455c..490c627e41cef 100755 --- a/util/minetester/build_minetest.sh +++ b/util/minetester/build_minetest.sh @@ -7,10 +7,11 @@ cd build/headless SDL2_DIR=${ROOT}/lib/SDL/build/lib/cmake/SDL2/ +IRRLICHTMT_DIR=${ROOT}/lib/irrlichtmt echo ${SDL2_DIR} -cmake ../.. -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=1 -DSDL2_DIR=${SDL2_DIR} +cmake ../.. -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=1 -DSDL2_DIR=${SDL2_DIR} -DIRRLICHTMT_BUILD_DIR=${IRRLICHTMT_DIR} make -j$(( $(nproc) > 1 ? $(nproc) - 1 : 1 )) #use max(nproc - 1,1) threads cd ../.. @@ -18,7 +19,9 @@ cd ../.. mv bin/minetest bin/minetest_headless cd build/normal -cmake ../.. -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=0 -DSDL2_DIR= +cmake ../.. -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=0 -DSDL2_DIR= -DIRRLICHTMT_BUILD_DIR=${IRRLICHTMT_DIR} make -j$(( $(nproc) > 1 ? $(nproc) - 1 : 1 )) #use max(nproc - 1,1) threads +cd ../.. + From a1b3a9613cd75f0735ee56dec4a1a66c8a908d13 Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 8 Jan 2024 22:14:10 +0100 Subject: [PATCH 21/26] Fix submodules --- .gitmodules | 14 +++++++------- lib/irrlichmt | 1 + lib/irrlichtmt | 1 - lib/zmqpp | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) create mode 160000 lib/irrlichmt delete mode 120000 lib/irrlichtmt mode change 120000 => 160000 lib/zmqpp diff --git a/.gitmodules b/.gitmodules index 11463af8a7ab9..bf7b03151babc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,3 @@ -[submodule "lib/irrlichtmt"] - path = lib/irrlichtmt - url = https://github.com/EleutherAI/irrlicht.git - branch = headless-rendering -[submodule "lib/zmqpp"] - path = lib/zmqpp - url = https://github.com/zeromq/zmqpp.git [submodule "games/minetest_game"] path = games/minetest_game url = https://github.com/minetest/minetest_game.git @@ -12,3 +5,10 @@ path = lib/SDL url = https://github.com/libsdl-org/SDL.git branch = SDL2 +[submodule "lib/zmqpp"] + path = lib/zmqpp + url = https://github.com/zeromq/zmqpp.git +[submodule "lib/irrlicht"] + path = lib/irrlichmt + url = https://github.com/EleutherAI/irrlicht.git + branch = headless-rendering diff --git a/lib/irrlichmt b/lib/irrlichmt new file mode 160000 index 0000000000000..8f13ae81e5898 --- /dev/null +++ b/lib/irrlichmt @@ -0,0 +1 @@ +Subproject commit 8f13ae81e58984a4096229fb644d0d4e88136536 diff --git a/lib/irrlichtmt b/lib/irrlichtmt deleted file mode 120000 index f0c2d62bfbc2a..0000000000000 --- a/lib/irrlichtmt +++ /dev/null @@ -1 +0,0 @@ -../../irrlicht/ \ No newline at end of file diff --git a/lib/zmqpp b/lib/zmqpp deleted file mode 120000 index 3e455fb3b1892..0000000000000 --- a/lib/zmqpp +++ /dev/null @@ -1 +0,0 @@ -../../zmqpp \ No newline at end of file diff --git a/lib/zmqpp b/lib/zmqpp new file mode 160000 index 0000000000000..ba4230d5d03d2 --- /dev/null +++ b/lib/zmqpp @@ -0,0 +1 @@ +Subproject commit ba4230d5d03d29ced9ca788e3bd1095477db08ae From 6a809017acc1e2c2a43dce07b859f38716de70ac Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 8 Jan 2024 22:16:47 +0100 Subject: [PATCH 22/26] Fix custom IrrlichtMt branch --- lib/irrlichmt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irrlichmt b/lib/irrlichmt index 8f13ae81e5898..f178f25377a65 160000 --- a/lib/irrlichmt +++ b/lib/irrlichmt @@ -1 +1 @@ -Subproject commit 8f13ae81e58984a4096229fb644d0d4e88136536 +Subproject commit f178f25377a654bc2133b26a89fd054b1ee8b387 From 3fca9acfe12d2bdddb8e7a14dbea55b98ab7ea5f Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 8 Jan 2024 22:22:19 +0100 Subject: [PATCH 23/26] Fix IrrlichtMt path --- .gitmodules | 2 +- lib/{irrlichmt => irrlichtmt} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/{irrlichmt => irrlichtmt} (100%) diff --git a/.gitmodules b/.gitmodules index bf7b03151babc..ee38106856a2f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,6 +9,6 @@ path = lib/zmqpp url = https://github.com/zeromq/zmqpp.git [submodule "lib/irrlicht"] - path = lib/irrlichmt + path = lib/irrlichtmt url = https://github.com/EleutherAI/irrlicht.git branch = headless-rendering diff --git a/lib/irrlichmt b/lib/irrlichtmt similarity index 100% rename from lib/irrlichmt rename to lib/irrlichtmt From df5a140b59e654fe3d9ec1b9101bd19d25c0f412 Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 8 Jan 2024 22:32:47 +0100 Subject: [PATCH 24/26] Fix zmqpp build script --- util/minetester/build_zmqpp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/minetester/build_zmqpp.sh b/util/minetester/build_zmqpp.sh index 9dcc2dfb58ea9..a48b304973be8 100755 --- a/util/minetester/build_zmqpp.sh +++ b/util/minetester/build_zmqpp.sh @@ -1,5 +1,5 @@ cd lib/zmqpp -mkdir build +mkdir build && cd build cmake .. -G "Unix Makefiles" make -cd ../.. +cd ../../.. From b0da980084e966526bc215b61a52de0910067e9c Mon Sep 17 00:00:00 2001 From: rk1a Date: Mon, 8 Jan 2024 22:58:04 +0100 Subject: [PATCH 25/26] Fix pre-commit step --- .github/workflows/minetester.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/minetester.yml b/.github/workflows/minetester.yml index a546921a04961..65f0655c46f02 100644 --- a/.github/workflows/minetester.yml +++ b/.github/workflows/minetester.yml @@ -44,4 +44,7 @@ jobs: python -m pip install --upgrade pip make install - name: Run pre-commit hooks - run: pre-commit run --all-files + run: | + pip install pre-commit + pre-commit install + pre-commit run --all-files From e982a28a4c12a863537ea97228a64b17747ace56 Mon Sep 17 00:00:00 2001 From: rk1a Date: Wed, 17 Jan 2024 21:36:51 +0100 Subject: [PATCH 26/26] Allow manually triggering the pipeline --- .github/workflows/minetester.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/minetester.yml b/.github/workflows/minetester.yml index 65f0655c46f02..514bf8b01c380 100644 --- a/.github/workflows/minetester.yml +++ b/.github/workflows/minetester.yml @@ -5,6 +5,7 @@ on: branches: [ "develop" ] pull_request: branches: [ "develop" ] + workflow_dispatch: jobs: build: