From 6c72f8052fddb8afe2808f86b809f05dda21afcc Mon Sep 17 00:00:00 2001 From: Thomas Rudin Date: Mon, 2 Jul 2018 11:39:58 +0200 Subject: [PATCH] initial rendering config --- cleanup.sql | 3 +- .../minetest/tileserver/TileRenderer.java | 2 +- .../rudin/minetest/tileserver/TileServer.java | 6 + .../tileserver/config/TileServerConfig.java | 9 ++ .../job/InitialTileRendererJob.java | 124 ++++++++++++++++++ .../tileserver/provider/ExecutorProvider.java | 2 +- .../service/impl/DatabaseTileCache.java | 2 + .../tileserver/util/MapBlockAccessor.java | 1 + 8 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 src/main/java/io/rudin/minetest/tileserver/job/InitialTileRendererJob.java diff --git a/cleanup.sql b/cleanup.sql index b6702f8..c01864b 100644 --- a/cleanup.sql +++ b/cleanup.sql @@ -3,4 +3,5 @@ drop trigger if exists blocks_insert on blocks; drop table if exists tileserver_block_changes; drop table if exists tileserver_block_depth; drop table if exists tileserver_tiles; -drop table if exists flyway_schema_history; \ No newline at end of file +drop table if exists flyway_schema_history; +alter table blocks drop column mtime; \ No newline at end of file diff --git a/src/main/java/io/rudin/minetest/tileserver/TileRenderer.java b/src/main/java/io/rudin/minetest/tileserver/TileRenderer.java index cc61b1d..1892542 100644 --- a/src/main/java/io/rudin/minetest/tileserver/TileRenderer.java +++ b/src/main/java/io/rudin/minetest/tileserver/TileRenderer.java @@ -71,7 +71,7 @@ public byte[] render(int tileX, int tileY, int zoom) throws IllegalArgumentExcep MapBlockCoordinateInfo mapblockInfo = CoordinateResolver.fromTile(tileX, tileY, zoom); - if (mapblockInfo.x > 32000 || mapblockInfo.x < -32000 || mapblockInfo.z > 32000 || mapblockInfo.z < -32000) + if (mapblockInfo.x > 2048 || mapblockInfo.x < -2048 || mapblockInfo.z > 2048 || mapblockInfo.z < -2048) //Out of range... return WhiteTile.getPNG(); diff --git a/src/main/java/io/rudin/minetest/tileserver/TileServer.java b/src/main/java/io/rudin/minetest/tileserver/TileServer.java index 8c6dd35..2e65a29 100644 --- a/src/main/java/io/rudin/minetest/tileserver/TileServer.java +++ b/src/main/java/io/rudin/minetest/tileserver/TileServer.java @@ -8,6 +8,7 @@ import com.google.inject.Injector; import io.rudin.minetest.tileserver.config.TileServerConfig; +import io.rudin.minetest.tileserver.job.InitialTileRendererJob; import io.rudin.minetest.tileserver.job.UpdateChangedTilesJob; import io.rudin.minetest.tileserver.job.UpdatePlayerJob; import io.rudin.minetest.tileserver.module.ConfigModule; @@ -66,6 +67,11 @@ public static void main(String[] args) throws Exception { executor.scheduleAtFixedRate(injector.getInstance(UpdateChangedTilesJob.class), 0, cfg.tilerendererUpdateInterval(), TimeUnit.SECONDS); executor.scheduleAtFixedRate(injector.getInstance(UpdatePlayerJob.class), 0, cfg.playerUpdateInterval(), TimeUnit.SECONDS); + if (cfg.tilerendererEnableInitialRendering()){ + //Start initial rendering + executor.submit(injector.getInstance(InitialTileRendererJob.class)); + } + AtomicBoolean running = new AtomicBoolean(true); Runtime.getRuntime().addShutdownHook(new Thread(() -> { diff --git a/src/main/java/io/rudin/minetest/tileserver/config/TileServerConfig.java b/src/main/java/io/rudin/minetest/tileserver/config/TileServerConfig.java index 03d1750..1b60252 100644 --- a/src/main/java/io/rudin/minetest/tileserver/config/TileServerConfig.java +++ b/src/main/java/io/rudin/minetest/tileserver/config/TileServerConfig.java @@ -21,6 +21,15 @@ public interface TileServerConfig extends Config { @DefaultValue("-1") int tilesMinY(); + /* + Tile renderer stuff + */ + + //This should only be enabled once after a fresh install + @Key("tilerenderer.initialrendering.enable") + @DefaultValue("false") + boolean tilerendererEnableInitialRendering(); + @Key("tilerenderer.processes") @DefaultValue("24") int tilerendererProcesses(); diff --git a/src/main/java/io/rudin/minetest/tileserver/job/InitialTileRendererJob.java b/src/main/java/io/rudin/minetest/tileserver/job/InitialTileRendererJob.java new file mode 100644 index 0000000..0374f0f --- /dev/null +++ b/src/main/java/io/rudin/minetest/tileserver/job/InitialTileRendererJob.java @@ -0,0 +1,124 @@ +package io.rudin.minetest.tileserver.job; + +import io.rudin.minetest.tileserver.TileRenderer; +import io.rudin.minetest.tileserver.config.TileServerConfig; +import io.rudin.minetest.tileserver.util.CoordinateResolver; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record2; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import static io.rudin.minetest.tileserver.blockdb.tables.Blocks.BLOCKS; + +@Singleton +public class InitialTileRendererJob implements Runnable { + + private static final Logger logger = LoggerFactory.getLogger(InitialTileRendererJob.class); + + @Inject + public InitialTileRendererJob(DSLContext ctx, TileServerConfig cfg, TileRenderer renderer){ + this.ctx = ctx; + this.renderer = renderer; + this.yCondition = BLOCKS.POSY.between(cfg.tilesMinY(), cfg.tilesMaxY()); + } + + private final DSLContext ctx; + + private final TileRenderer renderer; + + private final Condition yCondition; + + @Override + public void run() { + logger.info("Starting initial tilerendering"); + + Record2 minMaxRecords = ctx + .select(DSL.max(BLOCKS.POSX), DSL.min(BLOCKS.POSX)) + .from(BLOCKS) + .where(yCondition) + .fetchOne(); + + Integer maxX = minMaxRecords.get(DSL.max(BLOCKS.POSX)); + Integer minX = minMaxRecords.get(DSL.min(BLOCKS.POSX)); + + long start = System.currentTimeMillis(); + int xStripes = maxX - minX; + long byteCount = 0; + int errorCount = 0; + long lastProgressUpdate = 0; + int tileCount = 0; + + logger.info("X-Stripe count: {} (from {} to {})", xStripes, minX, maxX); + + for (int posx = minX; posx <= maxX; posx++){ + + double xProgress = (posx - minX) / (double)xStripes; + + minMaxRecords = ctx + .select(DSL.max(BLOCKS.POSZ), DSL.min(BLOCKS.POSZ)) + .from(BLOCKS) + .where(yCondition) + .and(BLOCKS.POSX.eq(posx)) + .fetchOne(); + + Integer maxZ = minMaxRecords.get(DSL.max(BLOCKS.POSZ)); + Integer minZ = minMaxRecords.get(DSL.min(BLOCKS.POSZ)); + + int zCount = maxZ - minZ; + + for (int posz = minZ; posz <= maxZ; posz++){ + + double zProgress = (posz - minZ) / (double)zCount; + + CoordinateResolver.TileInfo tileInfo = CoordinateResolver.fromCoordinates(posx, posz); + + try { + byte[] data = renderer.render(tileInfo.x, tileInfo.x, CoordinateResolver.ONE_TO_ONE_ZOOM); + byteCount += data.length; + tileCount++; + + } catch (Exception e){ + errorCount++; + logger.error("run", e); + + if (errorCount > 100){ + logger.error("Error-count: {}, bailing out!", errorCount); + return; + } + } + + long now = System.currentTimeMillis(); + long diff = now - lastProgressUpdate; + + if (diff > 5000){ + //Report every 5 seconds + lastProgressUpdate = now; + + logger.info("Initial rendering status: x-progress({}%) z-progress({}%) tiles({}) data({} MB) posx({}) posz({})", + Math.floor(xProgress*100), Math.floor(zProgress*100), tileCount, Math.floor(byteCount/1000)/1000, + posx, posz + ); + + //Explicit gc() + Runtime.getRuntime().gc(); + } + + } + + } + + long renderTime = System.currentTimeMillis() - start; + + logger.info("Initial rendering completed in {} seconds ({} MB, {} tiles)", + renderTime, Math.floor(byteCount/1000)/1000, tileCount + ); + + + + } +} diff --git a/src/main/java/io/rudin/minetest/tileserver/provider/ExecutorProvider.java b/src/main/java/io/rudin/minetest/tileserver/provider/ExecutorProvider.java index ccba527..0cec31a 100644 --- a/src/main/java/io/rudin/minetest/tileserver/provider/ExecutorProvider.java +++ b/src/main/java/io/rudin/minetest/tileserver/provider/ExecutorProvider.java @@ -11,7 +11,7 @@ public class ExecutorProvider implements Provider { public ExecutorProvider() { - executor = Executors.newScheduledThreadPool(10); + executor = Executors.newScheduledThreadPool(64); //TODO: war-package-case Runtime.getRuntime().addShutdownHook(new Thread(this::stop)); diff --git a/src/main/java/io/rudin/minetest/tileserver/service/impl/DatabaseTileCache.java b/src/main/java/io/rudin/minetest/tileserver/service/impl/DatabaseTileCache.java index 7eab1a0..1a4a72d 100644 --- a/src/main/java/io/rudin/minetest/tileserver/service/impl/DatabaseTileCache.java +++ b/src/main/java/io/rudin/minetest/tileserver/service/impl/DatabaseTileCache.java @@ -37,6 +37,8 @@ public DatabaseTileCache(@TileDB DSLContext ctx, ExecutorService executor){ this.cache = CacheBuilder .newBuilder() .expireAfterWrite(10, TimeUnit.SECONDS) + .weakValues() + .maximumSize(500) .build(); } diff --git a/src/main/java/io/rudin/minetest/tileserver/util/MapBlockAccessor.java b/src/main/java/io/rudin/minetest/tileserver/util/MapBlockAccessor.java index 9dddf70..7556231 100644 --- a/src/main/java/io/rudin/minetest/tileserver/util/MapBlockAccessor.java +++ b/src/main/java/io/rudin/minetest/tileserver/util/MapBlockAccessor.java @@ -40,6 +40,7 @@ public MapBlockAccessor(DSLContext ctx, TileServerConfig cfg, EventBus eventBus) this.cache = CacheBuilder.newBuilder() .expireAfterAccess(20, TimeUnit.SECONDS) + .weakValues() .maximumSize(500) .build(this); }