Headless renderer

.gitignore

@@ -134,7 +134,8 @@ compile_commands.json
 .vs/
 
 # Optional user provided library folder
-lib/irrlichtmt
+# TODO: don't add irrlichtmt, apply patch from CMakeLists.txt
+# lib/irrlichtmt
 
 # Generated mod storage database
 client/mod_storage.sqlite

.gitmodules

@@ -1,3 +1,8 @@
+[submodule "lib/irrlichtmt"]
+	path = lib/irrlichtmt
+	url = [email protected]:EleutherAI/irrlicht.git 
+        branch = headless-rendering
+
 [submodule "lib/zmqpp"]
 	path = lib/zmqpp
 	url = [email protected]:zeromq/zmqpp.git

CMakeLists.txt

@@ -50,6 +50,7 @@ set(RUN_IN_PLACE ${DEFAULT_RUN_IN_PLACE} CACHE BOOL
 
 
 set(BUILD_CLIENT TRUE CACHE BOOL "Build client")
+set(BUILD_HEADLESS TRUE CACHE BOOL "Build in headless mode")
 set(BUILD_SERVER FALSE CACHE BOOL "Build server")
 set(BUILD_UNITTESTS TRUE CACHE BOOL "Build unittests")
 set(BUILD_BENCHMARKS FALSE CACHE BOOL "Build benchmarks")
@@ -67,7 +68,6 @@ set(ENABLE_UPDATE_CHECKER (NOT ${DEVELOPMENT_BUILD}) CACHE BOOL
 # Included stuff
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 
-
 set(IRRLICHTMT_BUILD_DIR "" CACHE PATH "Path to IrrlichtMt build directory.")
 if(NOT "${IRRLICHTMT_BUILD_DIR}" STREQUAL "")
 	find_package(IrrlichtMt QUIET

build_instructions.txt

@@ -0,0 +1,49 @@
+1. Install prereqs 
+  1. sudo apt-get install xvfb g++ make libzmq3-dev libtool pkg-config build-essential autoconf automake libc6-dev cmake libpng-dev libjpeg-dev libxi-dev libgl1-mesa-dev libsqlite3-dev libogg-dev libvorbis-dev libopenal-dev libcurl4-gnutls-dev libfreetype6-dev zlib1g-dev libgmp-dev libjsoncpp-dev libzstd-dev libluajit-5.1-dev protobuf-compiler
+
+2. Build SDL2
+  1. clone the SDL2 repo https://github.com/libsdl-org/SDL
+  2. checkout release-2.26.2 https://github.com/libsdl-org/SDL/tree/release-2.26.2
+  3. create a build directory inside the SDL repo
+  4. cd into it and run ../configure --prefix=/path/to/SDL/dir/build && make && make install
+
+3. Build zmqpp
+  1. clone https://github.com/zeromq/zmqpp
+  2. checkout the develop branch
+  3. run make
+
+4. Clone EAI alignment minetest repos
+  1. clone https://github.com/EleutherAI/minetest
+  2. checkout the develop branch
+  3. clone https://github.com/EleutherAI/irrlicht
+  4. checkout headless-renderer
+
+5. Clone minetest game spec
+  1. clone https://github.com/minetest/minetest_game
+  2. checkout master branch
+
+5. Establish symlinks
+  1. cd into minetest/lib
+  2. rm -r zmqpp irrlichtmt
+  3. ln -s ../../zmqpp/ zmqpp
+  4. ln -s ../../irrlicht/ irrlichtmt
+  5. cd into minetest/games
+  6. ln -s ../../minetest_game/ minetest_game
+
+6. Build minetest
+  1. cd into minetest
+  2. either run
+     cmake . -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=1 -DSDL2_DIR=<path to SDL repo>/SDL/build/lib/cmake/SDL2/ 
+     or 
+     cmake . -DRUN_IN_PLACE=TRUE -DBUILD_HEADLESS=0 -DSDL2_DIR=
+  3. run make -j$(nproc)
+
+7. setup python 
+  1. create a new python conda env or venv (tested with python3.9)
+  2. pip install gym matplotlib protobuf==3.20.1 psutil zmq
+  3. cd into the scripts directory and run compile_proto.sh
+  4. run python -m minetester.scripts.test_loop
+
+
+
+

minetester/minetest_env.py

@@ -248,6 +248,8 @@ def _write_config(self):
             config_file.write("mute_sound = true\n")
             config_file.write("show_debug = false\n")
             config_file.write("enable_client_modding = true\n")
+            #config_file.write("video_driver = null\n")
+            #config_file.write("enable_shaders = false\n")
             config_file.write("csm_restriction_flags = 0\n")
             config_file.write("enable_mod_channels = true\n")
 
@@ -308,7 +310,7 @@ def step(self, action: Dict[str, Any]):
 
         self.last_obs = next_obs
         logging.debug(f"Received obs - {next_obs.shape}; reward - {rew}")
-        return next_obs, rew, done, info
+        return next_obs, rew, info
 
     def render(self, render_mode: str = "human"):
         if render_mode == "human":

minetester/scripts/test_loop.py

@@ -9,13 +9,13 @@
     clientmods=["random_v0"],
 )
 
-render = True
+render = True 
 obs = env.reset()
 done = False
 while not done:
     try:
         action = env.action_space.sample()
-        obs, rew, done, info = env.step(action)
+        obs, rew, info = env.step(action)
         if render:
             env.render()
     except KeyboardInterrupt:

src/CMakeLists.txt

@@ -36,6 +36,17 @@ set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE STRING
 # Set some random things default to not being visible in the GUI
 mark_as_advanced(EXECUTABLE_OUTPUT_PATH LIBRARY_OUTPUT_PATH)
 
+if(BUILD_CLIENT AND BUILD_HEADLESS)
+	find_package(SDL2 REQUIRED NO_DEFAULT_PATH)
+	# SDL2 exports targets since 2.0.6, but some distributions override config.
+	if(NOT TARGET SDL2::SDL2)
+		add_library(SDL2::SDL2 INTERFACE IMPORTED)
+		set_target_properties(SDL2::SDL2 PROPERTIES
+			INTERFACE_INCLUDE_DIRECTORIES ${SDL2_INCLUDE_DIRS}
+			INTERFACE_LINK_LIBRARIES ${SDL2_LIBRARIES}
+		)
+	endif()
+endif()
 
 if(NOT (BUILD_CLIENT OR BUILD_SERVER))
 	message(WARNING "Neither BUILD_CLIENT nor BUILD_SERVER is set! Setting BUILD_SERVER=true")
@@ -64,7 +75,6 @@ if(NOT USE_CURL)
 	endif()
 endif()
 
-
 option(ENABLE_GETTEXT "Use GetText for internationalization" ${BUILD_CLIENT})
 set(USE_GETTEXT FALSE)
 
@@ -528,10 +538,12 @@ include_directories(SYSTEM
 	${ZMQ_INCLUDE_DIR}
 	${ZMQPP_INCLUDE_DIR}
 	${SQLITE3_INCLUDE_DIR}
+	${SDL2_INCLUDE_DIR}
 	${LUA_INCLUDE_DIR}
 	${GMP_INCLUDE_DIR}
 	${JSON_INCLUDE_DIR}
 	${LUA_BIT_INCLUDE_DIR}
+	${SDL2_INCLUDE_DIR}
 )
 
 if(USE_GETTEXT)
@@ -560,16 +572,22 @@ endif()
 if(BUILD_CLIENT)
 	add_executable(${PROJECT_NAME} ${client_SRCS} ${extra_windows_SRCS})
 	add_dependencies(${PROJECT_NAME} GenerateVersion)
+
+	if(BUILD_HEADLESS)
+		set(SDL2_TARGET SDL2::SDL2)
+	endif()
 	target_link_libraries(
 		${PROJECT_NAME}
 		${ZLIB_LIBRARIES}
 		IrrlichtMt::IrrlichtMt
+		${SDL2_TARGET}
 		${ZSTD_LIBRARY}
 		${ZMQ_LIBRARY}
 		${ZMQPP_LIBRARY}
 		${X11_LIBRARIES}
 		${SOUND_LIBRARIES}
 		${SQLITE3_LIBRARY}
+		${SDL2_LIBRARY}
 		${LUA_LIBRARY}
 		${GMP_LIBRARY}
 		${JSON_LIBRARY}

src/client/client.cpp

@@ -17,6 +17,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
+#include "cmake_config.h"
 #include <iostream>
 #include <algorithm>
 #include <sstream>
@@ -1825,6 +1826,7 @@ void Client::makeScreenshot()
 {
 	irr::video::IVideoDriver *driver = m_rendering_engine->get_video_driver();
 	irr::video::IImage* raw_image;
+
 	if(m_rendering_engine->headless) {
 		raw_image = m_rendering_engine->get_screenshot();
 	} else {
@@ -1946,6 +1948,7 @@ bool Client::getTerminal() {
 
 pb_objects::Image Client::getSendableData(core::position2di cursorPosition, bool isMenuActive, irr::video::IImage* cursorImage) {
 	irr::video::IVideoDriver *driver = m_rendering_engine->get_video_driver();
+
 	irr::video::IImage* raw_image;
 	if(m_rendering_engine->headless) {
 		raw_image = m_rendering_engine->get_screenshot();
@@ -1959,7 +1962,6 @@ pb_objects::Image Client::getSendableData(core::position2di cursorPosition, bool
 	irr::video::IImage* const image =
 			driver->createImage(video::ECF_R8G8B8, raw_image->getDimension());
 	raw_image->copyTo(image);
-	raw_image->drop();
 
 	// if provided draw the cursor image at the current mouse position when GUI is open
 	if (isMenuActive && cursorImage) {
@@ -1975,6 +1977,7 @@ pb_objects::Image Client::getSendableData(core::position2di cursorPosition, bool
 	pb_img.set_width(dim.Width);
 	pb_img.set_height(dim.Height);
 	image->drop();
+	raw_image->drop();
 	return pb_img;
 }
 

src/client/game.cpp

@@ -1186,7 +1186,7 @@ bool Game::startup(bool *kill,
 		if (start_data.isDumbClient()) {
 			dynamic_cast<DumbClientInputHandler*>(input)->socket = zmqclient;
 		}
-		if (start_data.record)  {
+		if (start_data.isRecording())  {
 			createRecorder(start_data);
 			recorder->sender = zmqclient;
 		}
@@ -1233,7 +1233,6 @@ void Game::run()
 	while (m_rendering_engine->run() &&
 			!(*kill || g_gamecallback->shutdown_requested ||
 					(server && server->isShutdownRequested()))) {
-
 		// send data out
 		std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
 		if(recorder && !firstIter) {

src/client/render/core.cpp

@@ -18,6 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
+#include "cmake_config.h"
 #include "core.h"
 #include "plain.h"
 #include "client/shadows/dynamicshadowsrender.h"
@@ -64,8 +65,6 @@ void RenderingCore::draw(video::SColor _skycolor, bool _show_hud, bool _show_min
 		buffer->setTexture(0, v2f(1.0f, 1.0f), "idk_lol", video::ECF_R8G8B8);
 		auto tex = new TextureBufferOutput(buffer, 0);
 		pipeline->setRenderTarget(tex);
-
-
 		pipeline->reset(context);
 		pipeline->run(context);
 
@@ -105,4 +104,4 @@ video::IImage *RenderingCore::get_screenshot() {
 v2u32 RenderingCore::getVirtualSize() const
 {
 	return virtual_size;
-}
+}

src/client/render/pipeline.cpp

@@ -17,6 +17,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
+#include "cmake_config.h"
 #include "pipeline.h"
 #include "client/client.h"
 #include "client/hud.h"

src/client/render/plain.cpp

@@ -18,6 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
+#include "cmake_config.h"
 #include "plain.h"
 #include "secondstage.h"
 #include "client/camera.h"

src/client/renderingengine.cpp

@@ -18,6 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
 
+#include "cmake_config.h"
 #include <IrrlichtDevice.h>
 #include "fontengine.h"
 #include "client.h"
@@ -35,6 +36,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "inputhandler.h"
 #include "gettext.h"
 #include "../gui/guiSkin.h"
+#if BUILD_HEADLESS
+	#include <SDL_video.h>
+#endif
 
 #if !defined(_WIN32) && !defined(__APPLE__) && !defined(__ANDROID__) && \
 		!defined(SERVER) && !defined(__HAIKU__)
@@ -141,6 +145,10 @@ RenderingEngine::RenderingEngine(IEventReceiver *receiver)
 	params.OGLES2ShaderPath = (porting::path_share + DIR_DELIM + rel_path + DIR_DELIM).c_str();
 #endif
 
+#if BUILD_HEADLESS
+	SDL_VideoInit("offscreen");
+#endif
+
 	m_device = createDeviceEx(params);
 	driver = m_device->getVideoDriver();
 
@@ -655,3 +663,4 @@ float RenderingEngine::getDisplayDensity()
 }
 
 #endif // __ANDROID__
+

src/cmake_config.h.in

@@ -14,6 +14,7 @@
 #define STATIC_LOCALEDIR "@LOCALEDIR@"
 #define BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 #define ICON_DIR "@ICONDIR@"
+#cmakedefine01 BUILD_HEADLESS
 #cmakedefine01 RUN_IN_PLACE
 #cmakedefine01 DEVELOPMENT_BUILD
 #cmakedefine01 ENABLE_UPDATE_CHECKER

src/gameparams.h

@@ -46,6 +46,7 @@ struct GameStartData : GameParams
 
 	bool isSinglePlayer() const { return address.empty() && !local_server; }
 	bool isDumbClient() const { return dumb && !client_address.empty(); }
+	bool isRecording() const { return record && !client_address.empty(); }
 	bool isResizable() const { return resizable; }
 	bool isHeadless() const { return headless; }