significantly improve saving performance

build.gradle.kts

@@ -18,6 +18,9 @@ repositories {
 dependencies {
     compileOnly(libs.minestom)
     implementation(libs.zstd)
+    // Fastutil is only included because minestom already uses it, otherwise it is a crazy dependency
+    // for how it is used in this project.
+    implementation(libs.fastutil)
 
     testImplementation("ch.qos.logback:logback-core:1.4.7")
     testImplementation("ch.qos.logback:logback-classic:1.4.7")

gradle/libs.versions.toml

@@ -1,14 +1,16 @@
 metadata.format.version = "1.1"
 
 [versions]
-minestom = "f13a7b49fa"
+minestom = "9f3ee89506"
 zstd = "1.5.5-3"
+fastutil = "8.5.12"
 
 nexuspublish = "1.3.0"
 
 [libraries]
 minestom = { group = "dev.hollowcube", name = "minestom-ce", version.ref = "minestom" }
 zstd = { group = "com.github.luben", name = "zstd-jni", version.ref = "zstd" }
+fastutil = { group = "it.unimi.dsi", name = "fastutil", version.ref = "fastutil" }
 
 [plugins]
 nexuspublish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "nexuspublish" }

src/main/java/net/hollowcube/polar/PolarLoader.java

@@ -1,5 +1,9 @@
 package net.hollowcube.polar;
 
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.shorts.Short2ObjectArrayMap;
+import it.unimi.dsi.fastutil.shorts.Short2ObjectMap;
+import it.unimi.dsi.fastutil.shorts.Short2ObjectOpenHashMap;
 import net.hollowcube.polar.compat.ChunkSupplierShim;
 import net.minestom.server.MinecraftServer;
 import net.minestom.server.command.builder.arguments.minecraft.ArgumentBlockState;
@@ -21,10 +25,9 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ForkJoinPool;
 
 @SuppressWarnings("UnstableApiUsage")
@@ -37,6 +40,8 @@ public class PolarLoader implements IChunkLoader {
     // Account for changes between main Minestom and minestom-ce.
     private static final ChunkSupplierShim CHUNK_SUPPLIER = ChunkSupplierShim.select();
 
+    private static final Map<String, Biome> biomeCache = new ConcurrentHashMap<>();
+
     private final Path savePath;
     private final PolarWorld worldData;
 
@@ -128,20 +133,30 @@ private void loadSection(@NotNull PolarSection sectionData, @NotNull Section sec
         var rawBiomePalette = sectionData.biomePalette();
         var biomePalette = new Biome[rawBiomePalette.length];
         for (int i = 0; i < rawBiomePalette.length; i++) {
-            var biome = BIOME_MANAGER.getByName(NamespaceID.from(rawBiomePalette[i]));
-            if (biome == null) {
-                logger.error("Failed to find biome: {}", rawBiomePalette[i]);
-                biome = Biome.PLAINS;
-            }
-            biomePalette[i] = biome;
+            biomePalette[i] = biomeCache.computeIfAbsent(rawBiomePalette[i], id -> {
+                var biome = BIOME_MANAGER.getByName(NamespaceID.from(id));
+                if (biome == null) {
+                    logger.error("Failed to find biome: {}", id);
+                    biome = Biome.PLAINS;
+                }
+                return biome;
+            });
         }
         if (biomePalette.length == 1) {
             section.biomePalette().fill(biomePalette[0].id());
         } else {
             final var paletteData = sectionData.biomeData();
             section.biomePalette().setAll((x, y, z) -> {
                 int index = x / 4 + (z / 4) * 4 + (y / 4) * 16;
-                return biomePalette[paletteData[index]].id();
+
+                var paletteIndex = paletteData[index];
+                if (paletteIndex >= biomePalette.length) {
+                    logger.error("Invalid biome palette index. This is probably a corrupted world, " +
+                            "but it has been loaded with plains instead. No data has been written.");
+                    return Biome.PLAINS.id();
+                }
+
+                return biomePalette[paletteIndex].id();
             });
         }
 
@@ -171,13 +186,15 @@ private void loadBlockEntity(@NotNull PolarChunk.BlockEntity blockEntity, @NotNu
 
     @Override
     public void unloadChunk(Chunk chunk) {
-        updateChunkData(chunk);
+        updateChunkData(new Short2ObjectOpenHashMap<>(), chunk);
     }
 
     @Override
     public @NotNull CompletableFuture<Void> saveChunks(@NotNull Collection<Chunk> chunks) {
+        var blockCache = new Short2ObjectOpenHashMap<String>();
+
         // Update state of each chunk locally
-        chunks.forEach(this::updateChunkData);
+        chunks.forEach(c -> updateChunkData(blockCache, c));
 
         // Write the file to disk
         if (savePath != null) {
@@ -193,7 +210,7 @@ public void unloadChunk(Chunk chunk) {
         return CompletableFuture.completedFuture(null);
     }
 
-    private void updateChunkData(@NotNull Chunk chunk) {
+    private void updateChunkData(@NotNull Short2ObjectMap<String> blockCache, @NotNull Chunk chunk) {
         var dimension = chunk.getInstance().getDimensionType();
 
         var blockEntities = new ArrayList<PolarChunk.BlockEntity>();
@@ -206,32 +223,43 @@ private void updateChunkData(@NotNull Chunk chunk) {
             //todo check if section is empty and skip
 
             var blockPalette = new ArrayList<String>();
-            var blockData = new int[PolarSection.BLOCK_PALETTE_SIZE];
-
-            for (int sectionLocalY = 0; sectionLocalY < Chunk.CHUNK_SECTION_SIZE; sectionLocalY++) {
-                for (int z = 0; z < Chunk.CHUNK_SIZE_Z; z++) {
-                    for (int x = 0; x < Chunk.CHUNK_SIZE_X; x++) {
-                        int y = sectionLocalY + sectionY * Chunk.CHUNK_SECTION_SIZE;
-                        var block = chunk.getBlock(x, y, z);
-
-                        final int blockIndex = x + sectionLocalY * 16 * 16 + z * 16;
-
-                        // Section palette
-                        var namespace = blockToString(block);
-                        int paletteId = blockPalette.indexOf(namespace);
-                        if (paletteId == -1) {
-                            paletteId = blockPalette.size();
-                            blockPalette.add(namespace);
-                        }
-                        blockData[blockIndex] = paletteId;
-
-                        // Block entities
-                        var handlerId = block.handler() == null ? null : block.handler().getNamespaceId().asString();
-                        if (handlerId != null || block.hasNbt()) {
-                            blockEntities.add(new PolarChunk.BlockEntity(
-                                    x, y, z, handlerId,
-                                    block.hasNbt() ? Objects.requireNonNull(block.nbt()) : new NBTCompound()
-                            ));
+            int[] blockData = null;
+            if (section.blockPalette().count() == 0) {
+                // Short circuit empty palette
+                blockPalette.add("air");
+            } else {
+                var localBlockData = new int[PolarSection.BLOCK_PALETTE_SIZE];
+
+                section.blockPalette().getAll((x, sectionLocalY, z, blockStateId) -> {
+                    final int blockIndex = x + sectionLocalY * 16 * 16 + z * 16;
+
+                    // Section palette
+                    var namespace = blockCache.computeIfAbsent((short) blockStateId, unused -> blockToString(Block.fromStateId((short) blockStateId)));
+                    int paletteId = blockPalette.indexOf(namespace);
+                    if (paletteId == -1) {
+                        paletteId = blockPalette.size();
+                        blockPalette.add(namespace);
+                    }
+                    localBlockData[blockIndex] = paletteId;
+                });
+
+                blockData = localBlockData;
+
+                // Block entities
+                for (int sectionLocalY = 0; sectionLocalY < Chunk.CHUNK_SECTION_SIZE; sectionLocalY++) {
+                    for (int z = 0; z < Chunk.CHUNK_SIZE_Z; z++) {
+                        for (int x = 0; x < Chunk.CHUNK_SIZE_X; x++) {
+                            int y = sectionLocalY + sectionY * Chunk.CHUNK_SECTION_SIZE;
+                            var block = chunk.getBlock(x, y, z, Block.Getter.Condition.CACHED);
+                            if (block == null) return;
+
+                            var handlerId = block.handler() == null ? null : block.handler().getNamespaceId().asString();
+                            if (handlerId != null || block.hasNbt()) {
+                                blockEntities.add(new PolarChunk.BlockEntity(
+                                        x, y, z, handlerId,
+                                        block.hasNbt() ? Objects.requireNonNull(block.nbt()) : new NBTCompound()
+                                ));
+                            }
                         }
                     }
                 }

src/test/java/net/hollowcube/polar/demo/DemoServer.java

@@ -13,11 +13,13 @@
 
 public class DemoServer {
     public static void main(String[] args) throws Exception {
+        System.setProperty("minestom.chunk-view-distance", "32");
+
         var server = MinecraftServer.init();
 
         var instance = MinecraftServer.getInstanceManager().createInstanceContainer();
-        instance.setChunkLoader(new PolarLoader(Path.of("./src/test/resources/bench/bench.polar")));
-        instance.setChunkSupplier(LightingChunk::new);
+        instance.setChunkLoader(new PolarLoader(Path.of("./src/test/resources/bench.polar")));
+//        instance.setChunkSupplier(LightingChunk::new);
 
         MinecraftServer.getGlobalEventHandler()
                 .addListener(PlayerLoginEvent.class, event -> {
@@ -29,8 +31,12 @@ public static void main(String[] args) throws Exception {
                 })
                 .addListener(PlayerChatEvent.class, event -> {
                     if (!event.getMessage().equals("save")) return;
+
+                    var start = System.currentTimeMillis();
                     instance.saveChunksToStorage().join();
-                    event.getPlayer().sendMessage("Done!");
+
+                    var time = System.currentTimeMillis() - start;
+                    event.getPlayer().sendMessage("Done in " + (time) + "ms!");
                 });
 
         server.start("0.0.0.0", 25565);