diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..2ce6d79 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,30 @@ +name: Gradle Publish to Maven Central + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + +jobs: + build: + runs-on: ubuntu-latest + env: + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSWORD }} + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 17 + uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'temurin' + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + - name: Publish to Sonatype + env: + TAG_VERSION: ${{ github.ref_name }} + run: | + ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -Peula=true + echo "Version: ${TAG_VERSION}" >> $GITHUB_STEP_SUMMARY diff --git a/BENCHMARK.md b/BENCHMARK.md new file mode 100644 index 0000000..0a52b4a --- /dev/null +++ b/BENCHMARK.md @@ -0,0 +1,34 @@ +# Benchmark + +> Reminder: The benchmark is very simple, and should only be valued as a rough estimate. + +The tests were run against [`minestom-ce`](https://github.com/hollow-cube/minestom-ce) on 1.19.4 (`f13a7b49fa`), +on a Macbook Pro (M1 Max). The source code of the test can be seen below. + +```java +public class ScuffedBenchmark { + public static void main(String[] args) throws Exception { + MinecraftServer.init(); + var instance = MinecraftServer.getInstanceManager().createInstanceContainer(); + + long start = System.nanoTime(); + + for (int iter = 0; iter < 10; iter++) { + System.out.println("Starting iteration " + iter); + // TNTLoader loader = new TNTLoader(new FileTNTSource(Path.of("src/test/resources/bench/bench.tnt"))); + // AnvilLoader loader = new AnvilLoader(Path.of("src/test/resources/bench")); + PolarLoader loader = new PolarLoader(PolarReader.read(Files.readAllBytes(Path.of("src/test/resources/bench.polar")))); + for (int x = 0; x < 32; x++) { + for (int z = 0; z < 32; z++) { + loader.loadChunk(instance, 0, 0).join(); + } + } + + } + + long end = System.nanoTime(); + System.out.println("Took " + (end - start) / 1_000_000_000.0 / 10.0 + " seconds/iter"); + MinecraftServer.stopCleanly(); + } +} +``` diff --git a/spec.md b/FORMAT.md similarity index 89% rename from spec.md rename to FORMAT.md index 8167220..5b0a52c 100644 --- a/spec.md +++ b/FORMAT.md @@ -1,13 +1,15 @@ # Polar v1.0 + The polar format resembles the anvil format in many ways, though it is binary, not NBT. ### Header + ``` int - magic number byte - major version byte - minor version byte - compression type (0 = none, 1 = zstd) -varint - length of the rest of the data +varint - length of the rest of the data (uncompressed) byte - min section byte - max section @@ -17,6 +19,7 @@ varint - number of chunks ``` ### Chunk + ``` varint - chunk x varint - chunk z @@ -36,7 +39,10 @@ int - heightmap bitmask todo entities ``` +todo need to support block entities without ids (minestom does) + ### Sections + ``` bool - is empty (if set, nothing follows) diff --git a/README.md b/README.md index 0c09162..3c17465 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,25 @@ # Polar -A world format blah blah docs -## Features +[![license](https://img.shields.io/github/license/Minestom/MinestomDataGenerator.svg)](LICENSE) + +A world format for Minestom designed for simpler and smaller handling of small worlds, particularly for user generated +content where size matters. + +Polar generally should not be used for large worlds, since it stores worlds in a single file and does not +allow random access of chunks (the entire world must be loaded to read chunks). As a general rule of thumb, +Polar should only be used for worlds small enough that they are OK being completely kept in memory. + +The Polar format is described in [FORMAT.md](FORMAT.md). -* todo -* write -* these +## Features -The format is described in `/spec` +* [Fast to load](#benchmark) +* [Small file size](#benchmark) +* Simple to use +* [Anvil conversion](#anvil-interop) ## Install + Polar is (to be) available on [maven central](https://search.maven.org/search?q=g:dev.hollowcube%20AND%20a:polar). ```groovy @@ -23,7 +33,66 @@ dependencies { ``` ## Usage -todo + +Polar provides a `ChunkLoader` implementation for use with Minestom `Instance`s. + +``` +// Loading +Instance instance=...; +instance.setChunkLoader(new PolarLoader(Path.of("/path/to/file.polar"))); + +// Saving +instance.saveChunksToStorage(); +``` + +### Anvil interop + +Anvil conversion utilities are also included, and can be used something like the following. + +``` +var polarWorld = AnvilPolar.anvilToPolar(Path.of("/path/to/anvil/world/dir")); +var polarWorldBytes=PolarWriter.write(polarWorld); +``` + +### ChunkSelector + +Most Polar functions take a `ChunkSelector` as an optional parameter to select which chunks to include in that +operation. +For example, to convert an anvil world while only selecting a 5 chunk radius around 0,0, you could do the following: + +``` +AnvilPolar.anvilToPolar(Path.of("/path/to/anvil/world/dir"), ChunkSelector.radius(5)); +``` ## Comparison to others -todo comparison to anvil, tnt, anything else? + +### "Benchmark" + +Using a very basic benchmark, we can make some rough guesses about performance between Polar, Anvil, and TNT. +The benchmark loads a single region 10 times, averaging the runtime of each iteration. +More information about the test can be found in [BENCHMARK.md](BENCHMARK.md) + +| Scenario | Iterations | Polar (v1, zstd) | Polar (v1, uncompressed) | TNT (v1) | Anvil | +|-----------------|------------|------------------|--------------------------|----------------|----------------| +| 1.19.4 Region | 10 | 0.61400 s/iter | 0.56449 s/iter | 3.56732 s/iter | 9.65274 s/iter | +| EmortalMC Lobby | 10 | 0.06565 s/iter | 0.04759 s/iter | 0.05501 s/iter | 0.56378 s/iter | +| EmortalMC Lobby | 500 | 0.06777 s/iter | 0.06650 s/iter | 0.07553 s/iter | - | + +| Scenario | Polar (v1, zstd) | Polar (v1, uncompressed) | TNT (v1) | Anvil | +|-----------------|------------------|--------------------------|----------|-------| +| 1.19.4 Region | 5.9mb | 26.1mb | 115.9mb | 9.7mb | +| EmortalMC Lobby | 105kb | 800kb | 1.3mb | 13mb* | + +* This is not a fair comparison. Polar and TNT are only covering the 10x10 relevant chunks, anvil has 4 regions. + +1.19.4 Region is a single 32x32 chunk region created in 1.19.4, see `src/test/resources/bench` for the world. + +EmortalMC Lobby is 10x10 chunk world, see `mc.emortal.dev` for more info. + +## Contributing + +Contributions via PRs and issues are always welcome. + +## License + +This project is licensed under the [MIT License](LICENSE). diff --git a/build.gradle.kts b/build.gradle.kts index db4ed08..2ee4e36 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,9 +1,14 @@ plugins { - id("java") + `java-library` + + `maven-publish` + signing + alias(libs.plugins.nexuspublish) } group = "dev.hollowcube" -version = "1.0.0" +version = System.getenv("TAG_VERSION") ?: "dev" +description = "Fast and small world format for Minestom" repositories { mavenCentral() @@ -12,14 +17,92 @@ repositories { dependencies { compileOnly(libs.minestom) - testImplementation(libs.minestom) - implementation(libs.zstd) testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation(libs.minestom) +} + +java { + withSourcesJar() + withJavadocJar() + + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 } tasks.test { + maxHeapSize = "2g" useJUnitPlatform() } + +nexusPublishing { + this.packageGroup.set("dev.hollowcube") + + repositories.sonatype { + nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/")) + snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/")) + + if (System.getenv("SONATYPE_USERNAME") != null) { + username.set(System.getenv("SONATYPE_USERNAME")) + password.set(System.getenv("SONATYPE_PASSWORD")) + } + } +} + +publishing.publications.create("maven") { + groupId = "dev.hollowcube" + artifactId = "polar" + version = project.version.toString() + + from(project.components["java"]) + + pom { + name.set(artifactId) + description.set(project.description) + url.set("https://github.com/hollow-cube/polar") + + licenses { + license { + name.set("MIT") + url.set("https://github.com/hollow-cube/polar/blob/main/LICENSE") + } + } + + developers { + developer { + id.set("mworzala") + name.set("Matt Worzala") + email.set("matt@hollowcube.dev") + } + } + + issueManagement { + system.set("GitHub") + url.set("https://github.com/hollow-cube/polar/issues") + } + + scm { + connection.set("scm:git:git://github.com/hollow-cube/polar.git") + developerConnection.set("scm:git:git@github.com:hollow-cube/polar.git") + url.set("https://github.com/hollow-cube/polar") + tag.set(System.getenv("TAG_VERSION") ?: "HEAD") + } + + ciManagement { + system.set("Github Actions") + url.set("https://github.com/hollow-cube/polar/actions") + } + } +} + +signing { + isRequired = System.getenv("CI") != null + + val privateKey = System.getenv("GPG_PRIVATE_KEY") + val keyPassphrase = System.getenv()["GPG_PASSPHRASE"] + useInMemoryPgpKeys(privateKey, keyPassphrase) + + sign(publishing.publications) +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 98d4571..2fa94fb 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,6 +4,11 @@ metadata.format.version = "1.1" minestom = "f13a7b49fa" zstd = "1.5.5-3" +nexuspublish = "1.3.0" + [libraries] minestom = { group = "dev.hollowcube", name = "minestom-ce", version.ref = "minestom" } zstd = { group = "com.github.luben", name = "zstd-jni", version.ref = "zstd" } + +[plugins] +nexuspublish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "nexuspublish" } diff --git a/src/main/java/net/hollowcube/polar/AnvilPolar.java b/src/main/java/net/hollowcube/polar/AnvilPolar.java new file mode 100644 index 0000000..ba0f380 --- /dev/null +++ b/src/main/java/net/hollowcube/polar/AnvilPolar.java @@ -0,0 +1,233 @@ +package net.hollowcube.polar; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jglrxavpok.hephaistos.mca.AnvilException; +import org.jglrxavpok.hephaistos.mca.RegionFile; +import org.jglrxavpok.hephaistos.mca.readers.ChunkReader; +import org.jglrxavpok.hephaistos.mca.readers.ChunkSectionReader; +import org.jglrxavpok.hephaistos.nbt.NBTCompound; +import org.jglrxavpok.hephaistos.nbt.NBTString; +import org.jglrxavpok.hephaistos.nbt.mutable.MutableNBTCompound; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class AnvilPolar { + private static final Logger logger = LoggerFactory.getLogger(AnvilPolar.class); + + public static @NotNull PolarWorld anvilToPolar(@NotNull Path path) throws IOException { + return anvilToPolar(path, ChunkSelector.all()); + } + + public static @NotNull PolarWorld anvilToPolar(@NotNull Path path, @NotNull ChunkSelector selector) throws IOException { + int minSection = Integer.MAX_VALUE, maxSection = Integer.MIN_VALUE; + + var chunks = new ArrayList(); + try (var files = Files.walk(path.resolve("region"), 1)) { + for (var regionFile : files.toList()) { + if (!regionFile.getFileName().toString().endsWith(".mca")) continue; + + var nameParts = regionFile.getFileName().toString().split("\\."); + var regionX = Integer.parseInt(nameParts[1]); + var regionZ = Integer.parseInt(nameParts[2]); + + try (var region = new RegionFile(new RandomAccessFile(regionFile.toFile(), "r"), regionX, regionZ)) { + var chunkSet = readAnvilChunks(region, selector); + if (chunkSet.chunks.isEmpty()) continue; + + if (minSection == Integer.MAX_VALUE) { + minSection = chunkSet.minSection(); + maxSection = chunkSet.maxSection(); + } else { + if (minSection != chunkSet.minSection() || maxSection != chunkSet.maxSection()) { + throw new IllegalStateException("Inconsistent world height 2 " + minSection + " " + maxSection + " " + chunkSet.minSection() + " " + chunkSet.maxSection()); + } + } + + chunks.addAll(chunkSet.chunks()); + } + } + } catch (AnvilException e) { + throw new IOException(e); + } + + return new PolarWorld( + PolarWorld.VERSION_MAJOR, + PolarWorld.VERSION_MINOR, + PolarWorld.DEFAULT_COMPRESSION, + (byte) minSection, (byte) maxSection, + chunks + ); + } + + private static @NotNull ChunkSet readAnvilChunks(@NotNull RegionFile regionFile, @NotNull ChunkSelector selector) throws AnvilException, IOException { + int minSection = Integer.MAX_VALUE, maxSection = Integer.MIN_VALUE; + + var chunks = new ArrayList(); + for (int x = 0; x < 32; x++) { + for (int z = 0; z < 32; z++) { + int chunkX = x + (regionFile.getRegionX() * 32); + int chunkZ = z + (regionFile.getRegionZ() * 32); + + if (!selector.test(chunkX, chunkZ)) continue; + + var chunkData = regionFile.getChunkData(chunkX, chunkZ); + if (chunkData == null) continue; + + var chunkReader = new ChunkReader(chunkData); + + //todo check over section min/max everywhere + var yRange = chunkReader.getYRange(); + if (minSection == Integer.MAX_VALUE) { + minSection = yRange.getStart() >> 4; + maxSection = yRange.getEndInclusive() >> 4; + } else { + if (minSection != yRange.getStart() >> 4 || maxSection != yRange.getEndInclusive() >> 4) { + throw new IllegalStateException("Inconsistent world height"); + } + } + + var sections = new PolarSection[maxSection - minSection + 1]; + for (var sectionData : chunkReader.getSections()) { + var sectionReader = new ChunkSectionReader(chunkReader.getMinecraftVersion(), sectionData); + + // Blocks + String[] blockPalette; + int[] blockData = null; + var blockInfo = sectionReader.getBlockPalette(); + if (blockInfo == null) { + logger.warn("Chunk section {}, {}, {} has no block palette", + chunkReader.getChunkX(), sectionReader.getY(), chunkReader.getChunkZ()); + + blockPalette = new String[]{"minecraft:air"}; + } else { +// System.out.println("sec " + chunkReader.getChunkX() + " " + sectionReader.getY() + " " + chunkReader.getChunkZ()); + blockData = sectionReader.getUncompressedBlockStateIDs(); + blockPalette = new String[blockInfo.getSize()]; + for (int i = 0; i < blockPalette.length; i++) { + var paletteEntry = blockInfo.get(i); + var blockName = new StringBuilder(Objects.requireNonNull(paletteEntry.getString("Name"))); + + var propertiesNbt = paletteEntry.getCompound("Properties"); + if (propertiesNbt != null && propertiesNbt.getSize() > 0) { + blockName.append("["); + + for (var property : propertiesNbt) { + blockName.append(property.getKey()) + .append("=") + .append(((NBTString) property.getValue()).getValue()) + .append(","); + } + blockName.deleteCharAt(blockName.length() - 1); + + blockName.append("]"); + } + + blockPalette[i] = blockName.toString(); + } + } + + // Biomes + String[] biomePalette; + int[] biomeData = null; + var biomeInfo = sectionReader.getBiomeInformation(); + if (!biomeInfo.hasBiomeInformation()) { + logger.warn("Chunk section {}, {}, {} has no biome information", + chunkReader.getChunkX(), sectionReader.getY(), chunkReader.getChunkZ()); + + biomePalette = new String[]{"minecraft:plains"}; + } else if (biomeInfo.isFilledWithSingleBiome()) { + biomePalette = new String[]{biomeInfo.getBaseBiome()}; + } else { + var palette = new ArrayList(); + biomeData = new int[PolarSection.BIOME_PALETTE_SIZE]; + for (int i = 0; i < biomeData.length; i++) { + var biome = biomeInfo.getBiomes()[i]; + var paletteId = palette.indexOf(biome); + if (paletteId == -1) { + palette.add(biome); + paletteId = palette.size() - 1; + } + + biomeData[i] = paletteId; + } + biomePalette = palette.toArray(new String[0]); + } + + + byte[] blockLight = null, skyLight = null; + if (sectionReader.getBlockLight() != null && sectionReader.getSkyLight() != null) { + blockLight = sectionReader.getBlockLight().copyArray(); + skyLight = sectionReader.getSkyLight().copyArray(); + } + + sections[sectionReader.getY() - minSection] = new PolarSection( + blockPalette, blockData, + biomePalette, biomeData, + blockLight, skyLight + ); + } + + var blockEntities = new ArrayList(); + for (var blockEntityCompound : chunkReader.getBlockEntities()) { + var blockEntity = convertBlockEntity(blockEntityCompound); + if (blockEntity != null) blockEntities.add(blockEntity); + } + + var heightmaps = new byte[PolarChunk.HEIGHTMAP_BYTE_SIZE][PolarChunk.HEIGHTMAPS.length]; + chunkData.getCompound("Heightmaps"); + //todo: heightmaps +// MOTION_BLOCKING MOTION_BLOCKING_NO_LEAVES +// OCEAN_FLOOR OCEAN_FLOOR_WG +// WORLD_SURFACE WORLD_SURFACE_WG + + chunks.add(new PolarChunk( + chunkReader.getChunkX(), + chunkReader.getChunkZ(), + sections, + blockEntities, + heightmaps + )); + } + } + return new ChunkSet(chunks, minSection, maxSection); + } + + private static @Nullable PolarChunk.BlockEntity convertBlockEntity(@NotNull NBTCompound blockEntityCompound) { + final var x = blockEntityCompound.getInt("x"); + final var y = blockEntityCompound.getInt("y"); + final var z = blockEntityCompound.getInt("z"); + if (x == null || y == null || z == null) { + logger.warn("Block entity could not be converted due to invalid coordinates"); + return null; + } + + final String blockEntityId = blockEntityCompound.getString("id"); + if (blockEntityId == null) { + logger.warn("Block entity could not be converted due to missing id"); + return null; + } + + // Remove anvil tags + MutableNBTCompound mutableCopy = blockEntityCompound.toMutableCompound(); + mutableCopy.remove("id"); + mutableCopy.remove("x"); + mutableCopy.remove("y"); + mutableCopy.remove("z"); + mutableCopy.remove("keepPacked"); + + return new PolarChunk.BlockEntity(x, y, z, blockEntityId, mutableCopy.toCompound()); + } + + private record ChunkSet(List chunks, int minSection, int maxSection) { + } + +} diff --git a/src/main/java/net/hollowcube/polar/ChunkSelector.java b/src/main/java/net/hollowcube/polar/ChunkSelector.java new file mode 100644 index 0000000..5a238ef --- /dev/null +++ b/src/main/java/net/hollowcube/polar/ChunkSelector.java @@ -0,0 +1,31 @@ +package net.hollowcube.polar; + +import org.jetbrains.annotations.NotNull; + +/** + * A {@link ChunkSelector} can be used to select some chunks from a world. This is useful for + * saving or loading only a select portion of a world, ignoring the rest. + *

+ * Polar supports {@link ChunkSelector}s in most loading/saving APIs. + */ +public interface ChunkSelector { + + static @NotNull ChunkSelector all() { + return (x, z) -> true; + } + + static @NotNull ChunkSelector radius(int radius) { + return radius(0, 0, radius); + } + + static @NotNull ChunkSelector radius(int centerX, int centerZ, int radius) { + return (x, z) -> { + int dx = x - centerX; + int dz = z - centerZ; + return dx * dx + dz * dz <= radius * radius; + }; + } + + boolean test(int x, int z); + +} diff --git a/src/main/java/net/hollowcube/polar/PolarChunk.java b/src/main/java/net/hollowcube/polar/PolarChunk.java index acbc993..38d307e 100644 --- a/src/main/java/net/hollowcube/polar/PolarChunk.java +++ b/src/main/java/net/hollowcube/polar/PolarChunk.java @@ -12,13 +12,13 @@ */ public class PolarChunk { - public static final int HEIGHTMAP_NONE = 0x0; - public static final int HEIGHTMAP_MOTION_BLOCKING = 0x1; - public static final int HEIGHTMAP_MOTION_BLOCKING_NO_LEAVES = 0x2; - public static final int HEIGHTMAP_OCEAN_FLOOR = 0x4; - public static final int HEIGHTMAP_OCEAN_FLOOR_WG = 0x8; - public static final int HEIGHTMAP_WORLD_SURFACE = 0x10; - public static final int HEIGHTMAP_WORLD_SURFACE_WG = 0x20; + public static final int HEIGHTMAP_NONE = 0b0; + public static final int HEIGHTMAP_MOTION_BLOCKING = 0b1; + public static final int HEIGHTMAP_MOTION_BLOCKING_NO_LEAVES = 0b10; + public static final int HEIGHTMAP_OCEAN_FLOOR = 0b100; + public static final int HEIGHTMAP_OCEAN_FLOOR_WG = 0b1000; + public static final int HEIGHTMAP_WORLD_SURFACE = 0b10000; + public static final int HEIGHTMAP_WORLD_SURFACE_WG = 0b100000; static final int[] HEIGHTMAPS = new int[]{ HEIGHTMAP_NONE, HEIGHTMAP_MOTION_BLOCKING, @@ -28,6 +28,7 @@ public class PolarChunk { HEIGHTMAP_WORLD_SURFACE, HEIGHTMAP_WORLD_SURFACE_WG, }; + static final int HEIGHTMAP_BYTE_SIZE = 32; private final int x; private final int z; diff --git a/src/main/java/net/hollowcube/polar/PolarLoader.java b/src/main/java/net/hollowcube/polar/PolarLoader.java index 4de42e5..cafea9b 100644 --- a/src/main/java/net/hollowcube/polar/PolarLoader.java +++ b/src/main/java/net/hollowcube/polar/PolarLoader.java @@ -17,6 +17,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -26,7 +30,21 @@ public class PolarLoader implements IChunkLoader { private static final BiomeManager BIOME_MANAGER = MinecraftServer.getBiomeManager(); private static final Logger logger = LoggerFactory.getLogger(PolarLoader.class); - private final PolarWorld worldData = null; + private final PolarWorld worldData; + + public PolarLoader(@NotNull Path path) throws IOException { + this(Files.newInputStream(path)); + } + + public PolarLoader(@NotNull InputStream inputStream) throws IOException { + try (inputStream) { + this.worldData = PolarReader.read(inputStream.readAllBytes()); + } + } + + public PolarLoader(@NotNull PolarWorld world) { + this.worldData = world; + } // Loading diff --git a/src/main/java/net/hollowcube/polar/PolarReader.java b/src/main/java/net/hollowcube/polar/PolarReader.java index 5491f1d..4818fd1 100644 --- a/src/main/java/net/hollowcube/polar/PolarReader.java +++ b/src/main/java/net/hollowcube/polar/PolarReader.java @@ -34,7 +34,7 @@ public class PolarReader { byte minSection = buffer.read(BYTE), maxSection = buffer.read(BYTE); assertThat(minSection < maxSection, "Invalid section range"); - var chunks = buffer.readCollection(b -> readChunk(b, maxSection - minSection)); + var chunks = buffer.readCollection(b -> readChunk(b, maxSection - minSection + 1)); return new PolarWorld(major, minor, compression, minSection, maxSection, chunks); } @@ -50,7 +50,7 @@ public class PolarReader { var blockEntities = buffer.readCollection(PolarReader::readBlockEntity); - var heightmaps = new byte[32][PolarChunk.HEIGHTMAPS.length]; + var heightmaps = new byte[PolarChunk.HEIGHTMAP_BYTE_SIZE][PolarChunk.HEIGHTMAPS.length]; int heightmapMask = buffer.read(INT); for (int i = 0; i < PolarChunk.HEIGHTMAPS.length; i++) { if ((heightmapMask & PolarChunk.HEIGHTMAPS[i]) == 0) @@ -71,7 +71,7 @@ public class PolarReader { // If section is empty exit immediately if (buffer.read(BOOLEAN)) return new PolarSection(); - var blockPalette = buffer.readCollection(b -> b.read(STRING)).toArray(String[]::new); + var blockPalette = buffer.readCollection(STRING).toArray(String[]::new); int[] blockData = null; if (blockPalette.length > 1) { blockData = new int[PolarSection.BLOCK_PALETTE_SIZE]; @@ -81,7 +81,7 @@ public class PolarReader { unpackPaletteData(blockData, rawBlockData, bitsPerEntry); } - var biomePalette = buffer.readCollection(b -> b.read(STRING)).toArray(String[]::new); + var biomePalette = buffer.readCollection(STRING).toArray(String[]::new); int[] biomeData = null; if (biomePalette.length > 1) { biomeData = new int[PolarSection.BIOME_PALETTE_SIZE]; @@ -102,12 +102,13 @@ public class PolarReader { private static @NotNull PolarChunk.BlockEntity readBlockEntity(@NotNull NetworkBuffer buffer) { int posIndex = buffer.read(INT); + var id = buffer.read(STRING); + var nbt = (NBTCompound) buffer.read(NBT); return new PolarChunk.BlockEntity( ChunkUtils.blockIndexToChunkPositionX(posIndex), ChunkUtils.blockIndexToChunkPositionY(posIndex), ChunkUtils.blockIndexToChunkPositionZ(posIndex), - buffer.read(STRING), - (NBTCompound) buffer.read(NBT) + id, nbt ); } @@ -141,6 +142,10 @@ private static void unpackPaletteData(int[] out, long[] in, int bitsPerEntry) { int longIndex = i / intsPerLongCeil; int subIndex = i % intsPerLongCeil; + if (in.length == 0) { + System.out.println("ZERO"); + } + out[i] = (int) ((in[longIndex] >>> (bitsPerEntry * subIndex)) & mask); } } diff --git a/src/main/java/net/hollowcube/polar/PolarSection.java b/src/main/java/net/hollowcube/polar/PolarSection.java index 1930b71..feab39f 100644 --- a/src/main/java/net/hollowcube/polar/PolarSection.java +++ b/src/main/java/net/hollowcube/polar/PolarSection.java @@ -13,7 +13,7 @@ @ApiStatus.Internal public class PolarSection { public static final int BLOCK_PALETTE_SIZE = 4096; - public static final int BIOME_PALETTE_SIZE = 256; + public static final int BIOME_PALETTE_SIZE = 64; private final boolean empty; diff --git a/src/main/java/net/hollowcube/polar/PolarWorld.java b/src/main/java/net/hollowcube/polar/PolarWorld.java index 620946d..b69ff96 100644 --- a/src/main/java/net/hollowcube/polar/PolarWorld.java +++ b/src/main/java/net/hollowcube/polar/PolarWorld.java @@ -6,6 +6,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.Collection; import java.util.List; /** @@ -17,14 +18,16 @@ public class PolarWorld { public static final byte VERSION_MAJOR = 1; public static final byte VERSION_MINOR = 0; + public static CompressionType DEFAULT_COMPRESSION = CompressionType.ZSTD; + // Polar metadata private byte major; private byte minor; private CompressionType compression; // World metadata - private final int minSection; - private final int maxSection; + private final byte minSection; + private final byte maxSection; // Chunk data private final Long2ObjectMap chunks = new Long2ObjectOpenHashMap<>(); @@ -32,7 +35,7 @@ public class PolarWorld { public PolarWorld( byte major, byte minor, @NotNull CompressionType compression, - int minSection, int maxSection, + byte minSection, byte maxSection, @NotNull List chunks ) { this.major = major; @@ -48,10 +51,26 @@ public PolarWorld( } } + public @NotNull CompressionType compression() { + return compression; + } + + public byte minSection() { + return minSection; + } + + public byte maxSection() { + return maxSection; + } + public @Nullable PolarChunk chunkAt(int x, int z) { return chunks.getOrDefault(ChunkUtils.getChunkIndex(x, z), null); } + public @NotNull Collection chunks() { + return chunks.values(); + } + public enum CompressionType { NONE, ZSTD; diff --git a/src/main/java/net/hollowcube/polar/PolarWriter.java b/src/main/java/net/hollowcube/polar/PolarWriter.java new file mode 100644 index 0000000..bde26dc --- /dev/null +++ b/src/main/java/net/hollowcube/polar/PolarWriter.java @@ -0,0 +1,110 @@ +package net.hollowcube.polar; + +import com.github.luben.zstd.Zstd; +import net.minestom.server.network.NetworkBuffer; +import net.minestom.server.utils.chunk.ChunkUtils; +import org.jetbrains.annotations.NotNull; + +import java.nio.ByteBuffer; + +import static net.minestom.server.network.NetworkBuffer.*; + +@SuppressWarnings("UnstableApiUsage") +public class PolarWriter { + + public static byte[] write(@NotNull PolarWorld world) { + // Write the compressed content first + var content = new NetworkBuffer(ByteBuffer.allocate(1024)); + content.write(BYTE, world.minSection()); + content.write(BYTE, world.maxSection()); + content.writeCollection(world.chunks(), PolarWriter::writeChunk); + + // Create final buffer + return NetworkBuffer.makeArray(buffer -> { + buffer.write(INT, PolarWorld.MAGIC_NUMBER); + buffer.write(BYTE, PolarWorld.VERSION_MAJOR); + buffer.write(BYTE, PolarWorld.VERSION_MINOR); + buffer.write(BYTE, (byte) world.compression().ordinal()); + switch (world.compression()) { + case NONE -> { + buffer.write(VAR_INT, content.readableBytes()); + buffer.write(RAW_BYTES, content.readBytes(content.readableBytes())); + } + case ZSTD -> { + buffer.write(VAR_INT, content.readableBytes()); + buffer.write(RAW_BYTES, Zstd.compress(content.readBytes(content.readableBytes()))); + } + } + }); + } + + private static void writeChunk(@NotNull NetworkBuffer buffer, @NotNull PolarChunk chunk) { + buffer.write(VAR_INT, chunk.x()); + buffer.write(VAR_INT, chunk.z()); + + for (var section : chunk.sections()) { + writeSection(buffer, section); + } + buffer.writeCollection(chunk.blockEntities(), PolarWriter::writeBlockEntity); + + //todo heightmaps + buffer.write(INT, PolarChunk.HEIGHTMAP_NONE); + } + + private static void writeSection(@NotNull NetworkBuffer buffer, @NotNull PolarSection section) { + buffer.write(BOOLEAN, section.isEmpty()); + if (section.isEmpty()) return; + + // Blocks + var blockPalette = section.blockPalette(); + buffer.writeCollection(STRING, blockPalette); + if (blockPalette.length > 1) { + var blockData = section.blockData(); + var bitsPerEntry = (int) Math.ceil(Math.log(blockPalette.length) / Math.log(2)); + if (bitsPerEntry < 1) bitsPerEntry = 1; + buffer.write(LONG_ARRAY, pack(blockData, bitsPerEntry)); + } + + // Biomes + var biomePalette = section.biomePalette(); + buffer.writeCollection(STRING, biomePalette); + if (biomePalette.length > 1) { + var biomeData = section.biomeData(); + var bitsPerEntry = (int) Math.ceil(Math.log(biomePalette.length) / Math.log(2)); + if (bitsPerEntry < 1) bitsPerEntry = 1; + buffer.write(LONG_ARRAY, pack(biomeData, bitsPerEntry)); + } + + // Light + buffer.write(BOOLEAN, section.hasLightData()); + if (section.hasLightData()) { + buffer.write(RAW_BYTES, section.blockLight()); + buffer.write(RAW_BYTES, section.skyLight()); + } + } + + private static void writeBlockEntity(@NotNull NetworkBuffer buffer, @NotNull PolarChunk.BlockEntity blockEntity) { + var index = ChunkUtils.getBlockIndex(blockEntity.x(), blockEntity.y(), blockEntity.z()); + buffer.write(INT, index); + buffer.write(STRING, blockEntity.id()); + buffer.write(NBT, blockEntity.data()); + } + + private static long[] pack(int[] ints, int bitsPerEntry) { + int intsPerLong = (int) Math.floor(64d / bitsPerEntry); + long[] longs = new long[(int) Math.ceil(ints.length / (double) intsPerLong)]; + + long mask = (1L << bitsPerEntry) - 1L; + for (int i = 0; i < longs.length; i++) { + for (int intIndex = 0; intIndex < intsPerLong; intIndex++) { + int bitIndex = intIndex * bitsPerEntry; + int intActualIndex = intIndex + i * intsPerLong; + if (intActualIndex < ints.length) { + longs[i] |= (ints[intActualIndex] & mask) << bitIndex; + } + } + } + + return longs; + } +} diff --git a/src/test/java/net/hollowcube/polar/TestAnvilPolar.java b/src/test/java/net/hollowcube/polar/TestAnvilPolar.java new file mode 100644 index 0000000..77d6fd3 --- /dev/null +++ b/src/test/java/net/hollowcube/polar/TestAnvilPolar.java @@ -0,0 +1,23 @@ +package net.hollowcube.polar; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class TestAnvilPolar { + + @Test + void testConvertAnvilWorld() throws Exception { + var world = AnvilPolar.anvilToPolar( + Path.of("./src/test/resources/bench").toRealPath(), + ChunkSelector.radius(5) + ); + assertEquals(-4, world.minSection()); + + var result = PolarWriter.write(world); + System.out.println(result.length); + } + +} diff --git a/src/test/java/net/hollowcube/polar/TestPolarReader.java b/src/test/java/net/hollowcube/polar/TestPolarReader.java index c7e4590..3cf05e8 100644 --- a/src/test/java/net/hollowcube/polar/TestPolarReader.java +++ b/src/test/java/net/hollowcube/polar/TestPolarReader.java @@ -10,7 +10,7 @@ class TestPolarReader { @Test void testReadInvalidMagic() { var e = assertThrows(PolarReader.Error.class, () -> { - PolarReader.read(new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }); + PolarReader.read(new byte[]{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}); }); assertEquals("Invalid magic number", e.getMessage()); } @@ -18,7 +18,7 @@ void testReadInvalidMagic() { @Test void testNewerVersionFail() { var e = assertThrows(PolarReader.Error.class, () -> { - PolarReader.read(new byte[] { + PolarReader.read(new byte[]{ 0x50, 0x6F, 0x6C, 0x72, // magic number Byte.MAX_VALUE, 0x0, // version }); diff --git a/src/test/java/net/hollowcube/polar/demo/DemoServer.java b/src/test/java/net/hollowcube/polar/demo/DemoServer.java new file mode 100644 index 0000000..3f51e45 --- /dev/null +++ b/src/test/java/net/hollowcube/polar/demo/DemoServer.java @@ -0,0 +1,36 @@ +package net.hollowcube.polar.demo; + +import net.hollowcube.polar.PolarLoader; +import net.hollowcube.polar.PolarReader; +import net.minestom.server.MinecraftServer; +import net.minestom.server.coordinate.Pos; +import net.minestom.server.entity.GameMode; +import net.minestom.server.event.player.PlayerLoginEvent; +import net.minestom.server.event.player.PlayerSpawnEvent; +import net.minestom.server.instance.LightingChunk; + +import java.nio.file.Files; +import java.nio.file.Path; + +public class DemoServer { + public static void main(String[] args) throws Exception { + var server = MinecraftServer.init(); + + var instance = MinecraftServer.getInstanceManager().createInstanceContainer(); +// var world = AnvilPolar.anvilToPolar(Path.of("./src/test/resources/testtnt").toRealPath()); + var world = PolarReader.read(Files.readAllBytes(Path.of("./src/test/resources/testtnt.polar"))); + instance.setChunkLoader(new PolarLoader(world)); + instance.setChunkSupplier(LightingChunk::new); + + MinecraftServer.getGlobalEventHandler() + .addListener(PlayerLoginEvent.class, event -> { + event.setSpawningInstance(instance); + event.getPlayer().setRespawnPoint(new Pos(0, 100, 0)); + }) + .addListener(PlayerSpawnEvent.class, event -> { + event.getPlayer().setGameMode(GameMode.CREATIVE); + }); + + server.start("0.0.0.0", 25565); + } +} diff --git a/src/test/resources/bench/bench-nozstd.polar b/src/test/resources/bench/bench-nozstd.polar new file mode 100644 index 0000000..2ecd55c Binary files /dev/null and b/src/test/resources/bench/bench-nozstd.polar differ diff --git a/src/test/resources/bench/bench.polar b/src/test/resources/bench/bench.polar new file mode 100644 index 0000000..50b7a50 Binary files /dev/null and b/src/test/resources/bench/bench.polar differ diff --git a/src/test/resources/bench/region/r.0.0.mca b/src/test/resources/bench/region/r.0.0.mca new file mode 100644 index 0000000..6fddcc8 Binary files /dev/null and b/src/test/resources/bench/region/r.0.0.mca differ