initial rendering config

minetest-mapserver · Jul 2, 2018 · 6c72f80 · 6c72f80
1 parent 65cd5a4
commit 6c72f80
Show file tree

Hide file tree

Showing 8 changed files with 146 additions and 3 deletions.
diff --git a/cleanup.sql b/cleanup.sql
@@ -3,4 +3,5 @@ drop trigger if exists blocks_insert on blocks;
 drop table if exists tileserver_block_changes;
 drop table if exists tileserver_block_depth;
 drop table if exists tileserver_tiles;
-drop table if exists flyway_schema_history;
+drop table if exists flyway_schema_history;
+alter table blocks drop column mtime;
diff --git a/src/main/java/io/rudin/minetest/tileserver/TileRenderer.java b/src/main/java/io/rudin/minetest/tileserver/TileRenderer.java
@@ -71,7 +71,7 @@ public byte[] render(int tileX, int tileY, int zoom) throws IllegalArgumentExcep
 
 		MapBlockCoordinateInfo mapblockInfo = CoordinateResolver.fromTile(tileX, tileY, zoom);
 
-		if (mapblockInfo.x > 32000 || mapblockInfo.x < -32000 || mapblockInfo.z > 32000 || mapblockInfo.z < -32000)
+		if (mapblockInfo.x > 2048 || mapblockInfo.x < -2048 || mapblockInfo.z > 2048 || mapblockInfo.z < -2048)
 			//Out of range...
 			return WhiteTile.getPNG();
 

diff --git a/src/main/java/io/rudin/minetest/tileserver/TileServer.java b/src/main/java/io/rudin/minetest/tileserver/TileServer.java
@@ -8,6 +8,7 @@
 import com.google.inject.Injector;
 
 import io.rudin.minetest.tileserver.config.TileServerConfig;
+import io.rudin.minetest.tileserver.job.InitialTileRendererJob;
 import io.rudin.minetest.tileserver.job.UpdateChangedTilesJob;
 import io.rudin.minetest.tileserver.job.UpdatePlayerJob;
 import io.rudin.minetest.tileserver.module.ConfigModule;
@@ -66,6 +67,11 @@ public static void main(String[] args) throws Exception {
 		executor.scheduleAtFixedRate(injector.getInstance(UpdateChangedTilesJob.class), 0, cfg.tilerendererUpdateInterval(), TimeUnit.SECONDS);
 		executor.scheduleAtFixedRate(injector.getInstance(UpdatePlayerJob.class), 0, cfg.playerUpdateInterval(), TimeUnit.SECONDS);
 
+		if (cfg.tilerendererEnableInitialRendering()){
+			//Start initial rendering
+			executor.submit(injector.getInstance(InitialTileRendererJob.class));
+		}
+
 		AtomicBoolean running = new AtomicBoolean(true);
 
 		Runtime.getRuntime().addShutdownHook(new Thread(() -> {

diff --git a/src/main/java/io/rudin/minetest/tileserver/config/TileServerConfig.java b/src/main/java/io/rudin/minetest/tileserver/config/TileServerConfig.java
@@ -21,6 +21,15 @@ public interface TileServerConfig extends Config {
 	@DefaultValue("-1")
 	int tilesMinY();
 
+	/*
+	Tile renderer stuff
+	 */
+
+	//This should only be enabled once after a fresh install
+	@Key("tilerenderer.initialrendering.enable")
+	@DefaultValue("false")
+	boolean tilerendererEnableInitialRendering();
+
 	@Key("tilerenderer.processes")
 	@DefaultValue("24")
 	int tilerendererProcesses();

diff --git a/src/main/java/io/rudin/minetest/tileserver/job/InitialTileRendererJob.java b/src/main/java/io/rudin/minetest/tileserver/job/InitialTileRendererJob.java
@@ -0,0 +1,124 @@
+package io.rudin.minetest.tileserver.job;
+
+import io.rudin.minetest.tileserver.TileRenderer;
+import io.rudin.minetest.tileserver.config.TileServerConfig;
+import io.rudin.minetest.tileserver.util.CoordinateResolver;
+import org.jooq.Condition;
+import org.jooq.DSLContext;
+import org.jooq.Record2;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import static io.rudin.minetest.tileserver.blockdb.tables.Blocks.BLOCKS;
+
+@Singleton
+public class InitialTileRendererJob implements Runnable {
+
+    private static final Logger logger = LoggerFactory.getLogger(InitialTileRendererJob.class);
+
+    @Inject
+    public InitialTileRendererJob(DSLContext ctx, TileServerConfig cfg, TileRenderer renderer){
+        this.ctx = ctx;
+        this.renderer = renderer;
+        this.yCondition = BLOCKS.POSY.between(cfg.tilesMinY(), cfg.tilesMaxY());
+    }
+
+    private final DSLContext ctx;
+
+    private final TileRenderer renderer;
+
+    private final Condition yCondition;
+
+    @Override
+    public void run() {
+        logger.info("Starting initial tilerendering");
+
+        Record2<Integer, Integer> minMaxRecords = ctx
+                .select(DSL.max(BLOCKS.POSX), DSL.min(BLOCKS.POSX))
+                .from(BLOCKS)
+                .where(yCondition)
+                .fetchOne();
+
+        Integer maxX = minMaxRecords.get(DSL.max(BLOCKS.POSX));
+        Integer minX = minMaxRecords.get(DSL.min(BLOCKS.POSX));
+
+        long start = System.currentTimeMillis();
+        int xStripes = maxX - minX;
+        long byteCount = 0;
+        int errorCount = 0;
+        long lastProgressUpdate = 0;
+        int tileCount = 0;
+
+        logger.info("X-Stripe count: {} (from {} to {})", xStripes, minX, maxX);
+
+        for (int posx = minX; posx <= maxX; posx++){
+
+            double xProgress = (posx - minX) / (double)xStripes;
+
+            minMaxRecords = ctx
+                    .select(DSL.max(BLOCKS.POSZ), DSL.min(BLOCKS.POSZ))
+                    .from(BLOCKS)
+                    .where(yCondition)
+                    .and(BLOCKS.POSX.eq(posx))
+                    .fetchOne();
+
+            Integer maxZ = minMaxRecords.get(DSL.max(BLOCKS.POSZ));
+            Integer minZ = minMaxRecords.get(DSL.min(BLOCKS.POSZ));
+
+            int zCount = maxZ - minZ;
+
+            for (int posz = minZ; posz <= maxZ; posz++){
+
+                double zProgress = (posz - minZ) / (double)zCount;
+
+                CoordinateResolver.TileInfo tileInfo = CoordinateResolver.fromCoordinates(posx, posz);
+
+                try {
+                    byte[] data = renderer.render(tileInfo.x, tileInfo.x, CoordinateResolver.ONE_TO_ONE_ZOOM);
+                    byteCount += data.length;
+                    tileCount++;
+
+                } catch (Exception e){
+                    errorCount++;
+                    logger.error("run", e);
+
+                    if (errorCount > 100){
+                        logger.error("Error-count: {}, bailing out!", errorCount);
+                        return;
+                    }
+                }
+
+                long now = System.currentTimeMillis();
+                long diff = now - lastProgressUpdate;
+
+                if (diff > 5000){
+                    //Report every 5 seconds
+                    lastProgressUpdate = now;
+
+                    logger.info("Initial rendering status: x-progress({}%) z-progress({}%) tiles({}) data({} MB) posx({}) posz({})",
+                            Math.floor(xProgress*100), Math.floor(zProgress*100), tileCount, Math.floor(byteCount/1000)/1000,
+                            posx, posz
+                    );
+
+                    //Explicit gc()
+                    Runtime.getRuntime().gc();
+                }
+
+            }
+
+        }
+
+        long renderTime = System.currentTimeMillis() - start;
+
+        logger.info("Initial rendering completed in {} seconds ({} MB, {} tiles)",
+                renderTime, Math.floor(byteCount/1000)/1000, tileCount
+                );
+
+
+
+    }
+}
diff --git a/src/main/java/io/rudin/minetest/tileserver/provider/ExecutorProvider.java b/src/main/java/io/rudin/minetest/tileserver/provider/ExecutorProvider.java
@@ -11,7 +11,7 @@ public class ExecutorProvider implements Provider<ScheduledExecutorService> {
 
 	public ExecutorProvider() {
 
-		executor = Executors.newScheduledThreadPool(10);
+		executor = Executors.newScheduledThreadPool(64);
 
 		//TODO: war-package-case
 		Runtime.getRuntime().addShutdownHook(new Thread(this::stop));

diff --git a/src/main/java/io/rudin/minetest/tileserver/service/impl/DatabaseTileCache.java b/src/main/java/io/rudin/minetest/tileserver/service/impl/DatabaseTileCache.java
@@ -37,6 +37,8 @@ public DatabaseTileCache(@TileDB DSLContext ctx, ExecutorService executor){
         this.cache = CacheBuilder
                 .newBuilder()
                 .expireAfterWrite(10, TimeUnit.SECONDS)
+                .weakValues()
+                .maximumSize(500)
                 .build();
     }
 

diff --git a/src/main/java/io/rudin/minetest/tileserver/util/MapBlockAccessor.java b/src/main/java/io/rudin/minetest/tileserver/util/MapBlockAccessor.java
@@ -40,6 +40,7 @@ public MapBlockAccessor(DSLContext ctx, TileServerConfig cfg, EventBus eventBus)
 
         this.cache = CacheBuilder.newBuilder()
                 .expireAfterAccess(20, TimeUnit.SECONDS)
+                .weakValues()
                 .maximumSize(500)
                 .build(this);
     }