feat: implemented preloading anvil worlds

This commit is contained in:
Andrey Terentev 2024-11-27 02:08:54 +07:00
parent 08c8aa43b8
commit 3bd876a628
10 changed files with 568 additions and 2 deletions

View File

@ -1,7 +1,6 @@
package ru.dragonestia.msb3.api; package ru.dragonestia.msb3.api;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import net.minestom.server.coordinate.Pos;
import net.minestom.server.entity.GameMode; import net.minestom.server.entity.GameMode;
import ru.dragonestia.msb3.api.module.FlatWorldModule; import ru.dragonestia.msb3.api.module.FlatWorldModule;
import ru.dragonestia.msb3.api.module.MotdModule; import ru.dragonestia.msb3.api.module.MotdModule;

View File

@ -32,9 +32,11 @@ public class PreloadedWorldModule {
var loadingWorld = MinecraftServer.getInstanceManager().createInstanceContainer(); var loadingWorld = MinecraftServer.getInstanceManager().createInstanceContainer();
loadingWorld.eventNode().addListener(PlayerMoveEvent.class, event -> event.setCancelled(true)); loadingWorld.eventNode().addListener(PlayerMoveEvent.class, event -> event.setCancelled(true));
var factory = WorldFactory.anvil(worldDir); var factory = WorldFactory.preloadedAnvil(worldDir);
var worlds = new ConcurrentHashMap<UUID, World>(); var worlds = new ConcurrentHashMap<UUID, World>();
factory.load();
MinecraftServer.getGlobalEventHandler().addListener(AsyncPlayerConfigurationEvent.class, event -> { MinecraftServer.getGlobalEventHandler().addListener(AsyncPlayerConfigurationEvent.class, event -> {
var player = event.getPlayer(); var player = event.getPlayer();

View File

@ -3,6 +3,7 @@ package ru.dragonestia.msb3.api.world;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import net.minestom.server.instance.InstanceContainer; import net.minestom.server.instance.InstanceContainer;
import ru.dragonestia.msb3.api.world.factory.AnvilWorldFactory; import ru.dragonestia.msb3.api.world.factory.AnvilWorldFactory;
import ru.dragonestia.msb3.api.world.factory.PreloadedAnvilWorldFactory;
import java.io.File; import java.io.File;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
@ -27,4 +28,8 @@ public abstract class WorldFactory {
public static WorldFactory anvil(File worldDir) { public static WorldFactory anvil(File worldDir) {
return new AnvilWorldFactory(worldDir); return new AnvilWorldFactory(worldDir);
} }
public static PreloadedAnvilWorldFactory preloadedAnvil(File worldDir) {
return new PreloadedAnvilWorldFactory(worldDir);
}
} }

View File

@ -0,0 +1,12 @@
package ru.dragonestia.msb3.api.world.chunk;
import net.minestom.server.instance.DynamicChunk;
import net.minestom.server.instance.Instance;
import org.jetbrains.annotations.NotNull;
public class OutOfBoundsChunk extends DynamicChunk {
public OutOfBoundsChunk(@NotNull Instance instance, int chunkX, int chunkZ) {
super(instance, chunkX, chunkZ);
}
}

View File

@ -0,0 +1,37 @@
package ru.dragonestia.msb3.api.world.chunk;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import lombok.Getter;
import net.minestom.server.instance.DynamicChunk;
import net.minestom.server.instance.Instance;
import net.minestom.server.instance.Section;
import net.minestom.server.instance.block.Block;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.List;
@Getter
public class SharedChunk extends DynamicChunk {
private final Source source;
public SharedChunk(@NotNull Instance instance, int chunkX, int chunkZ, Source source) {
super(instance, chunkX, chunkZ);
this.source = source;
sections = source.sections.stream().map(Section::clone).toList();
entries.putAll(source.entries);
}
public static class Source {
private final List<Section> sections = new ArrayList<>();
private final Int2ObjectOpenHashMap<Block> entries = new Int2ObjectOpenHashMap<>(0);
public Source(List<Section> sections, Int2ObjectOpenHashMap<Block> entries) {
this.sections.addAll(sections);
this.entries.putAll(entries);
}
}
}

View File

@ -0,0 +1,27 @@
package ru.dragonestia.msb3.api.world.factory;
import net.minestom.server.MinecraftServer;
import ru.dragonestia.msb3.api.world.World;
import ru.dragonestia.msb3.api.world.WorldFactory;
import ru.dragonestia.msb3.api.world.loader.PreloadedAnvilChunkLoader;
import java.io.File;
public class PreloadedAnvilWorldFactory extends WorldFactory {
private final PreloadedAnvilChunkLoader.Source source;
public PreloadedAnvilWorldFactory(File worldDir) {
source = new PreloadedAnvilChunkLoader.Source(worldDir);
}
public void load() {
source.load();
}
@Override
protected World loadWorld() {
var instance = MinecraftServer.getInstanceManager().createInstanceContainer(source.create());
return createWorld(instance);
}
}

View File

@ -0,0 +1,92 @@
package ru.dragonestia.msb3.api.world.loader;
import lombok.extern.log4j.Log4j2;
import net.minestom.server.instance.Chunk;
import net.minestom.server.instance.IChunkLoader;
import net.minestom.server.instance.Instance;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import ru.dragonestia.msb3.api.world.chunk.OutOfBoundsChunk;
import ru.dragonestia.msb3.api.world.chunk.SharedChunk;
import ru.dragonestia.msb3.api.world.loader.anvil.AnvilRegionLoader;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@Log4j2
public class PreloadedAnvilChunkLoader implements IChunkLoader {
private final Source source;
private PreloadedAnvilChunkLoader(Source source) {
this.source = source;
}
@Override
public @NotNull CompletableFuture<@Nullable Chunk> loadChunk(@NotNull Instance instance, int chunkX, int chunkZ) {
return CompletableFuture.supplyAsync(() -> {
var sourceChunkData = source.provideChunk(chunkX, chunkZ);
if (sourceChunkData.isEmpty()) {
return new OutOfBoundsChunk(instance, chunkX, chunkZ);
}
return new SharedChunk(instance, chunkX, chunkZ, sourceChunkData.get());
});
}
@Override
public @NotNull CompletableFuture<Void> saveChunk(@NotNull Chunk chunk) {
if (chunk instanceof SharedChunk sharedChunk) {
return CompletableFuture.runAsync(() -> {
// TODO...
});
}
return CompletableFuture.completedFuture(null);
}
public static class Source {
private final File worldDir;
private final Map<ChunkPos, SharedChunk.Source> chunkSources = new HashMap<>();
private boolean loaded = false;
public Source(File worldDir) {
this.worldDir = worldDir;
}
public synchronized void load() {
if (loaded) return;
loaded = true;
log.info("Preloading anvil world '{}'", worldDir);
load0();
log.info("Successfully preloaded anvil world '{}'", worldDir);
}
private void load0() {
var regionLoader = new AnvilRegionLoader(worldDir);
var regions = regionLoader.listRegionFiles();
regions.stream()
.map(regionLoader::readAllChunksFromRegion)
.flatMap(chunks -> regionLoader.initChunkData(chunks).stream())
.forEach(chunkData -> {
var chunkPos = new ChunkPos(chunkData.chunkX(), chunkData.chunkZ());
chunkSources.put(chunkPos, new SharedChunk.Source(chunkData.sections(), chunkData.entries()));
});
log.info("Preloaded {} chunks from {} regions", chunkSources.size(), regions.size());
}
public PreloadedAnvilChunkLoader create() {
load();
return new PreloadedAnvilChunkLoader(this);
}
private Optional<SharedChunk.Source> provideChunk(int chunkX, int chunkZ) {
return Optional.ofNullable(chunkSources.get(new ChunkPos(chunkX, chunkZ)));
}
}
private record ChunkPos(int chunk, int chunkZ) {}
}

View File

@ -0,0 +1,9 @@
package ru.dragonestia.msb3.api.world.loader.anvil;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import net.minestom.server.instance.Section;
import net.minestom.server.instance.block.Block;
import java.util.List;
public record AnvilChunkData(int chunkX, int chunkZ, List<Section> sections, Int2ObjectOpenHashMap<Block> entries) {}

View File

@ -0,0 +1,253 @@
package ru.dragonestia.msb3.api.world.loader.anvil;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import lombok.extern.log4j.Log4j2;
import net.kyori.adventure.nbt.*;
import net.minestom.server.MinecraftServer;
import net.minestom.server.instance.DynamicChunk;
import net.minestom.server.instance.Instance;
import net.minestom.server.instance.Section;
import net.minestom.server.instance.block.Block;
import net.minestom.server.instance.block.BlockHandler;
import net.minestom.server.registry.DynamicRegistry;
import net.minestom.server.utils.ArrayUtils;
import net.minestom.server.utils.MathUtils;
import net.minestom.server.utils.NamespaceID;
import net.minestom.server.utils.validate.Check;
import net.minestom.server.world.biome.Biome;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.util.*;
@Log4j2
public class AnvilRegionLoader {
private static final DynamicRegistry<Biome> BIOME_REGISTRY = MinecraftServer.getBiomeRegistry();
private final static int PLAINS_ID = BIOME_REGISTRY.getId(NamespaceID.from("minecraft:plains"));
private final File worldDir;
public AnvilRegionLoader(File worldDir) {
this.worldDir = worldDir;
}
public Collection<File> listRegionFiles() {
if (!worldDir.exists()) return List.of();
var regionsDir = new File(worldDir, "region");
if (!regionsDir.exists()) return List.of();
return Arrays.stream(Objects.requireNonNull(regionsDir.listFiles(File::isFile)))
.filter(file -> file.getName().startsWith("r.") && file.getName().endsWith(".mca"))
.toList();
}
public Collection<ChunkEntry> readAllChunksFromRegion(File regionFile) {
var list = new ArrayList<ChunkEntry>();
try (var region = new RegionFile(regionFile)) {
for (int x = 0; x < 32; x++) {
for (int z = 0; z < 32; z++) {
int chunkX = (region.getRegionX() << 5) + x;
int chunkZ = (region.getRegionZ() << 5) + z;
var chunkData = region.readChunkData(chunkX, chunkZ);
if (chunkData == null) continue;
list.add(new ChunkEntry(chunkX, chunkZ, chunkData));
}
}
} catch (IOException ex) {
throw new RuntimeException(ex);
}
log.debug("Found {} chunks inside region {}", list.size(), regionFile);
return list;
}
public Collection<AnvilChunkData> initChunkData(Collection<ChunkEntry> chunkEntries) {
var list = new ArrayList<AnvilChunkData>();
var instance = MinecraftServer.getInstanceManager().createInstanceContainer();
for (var entry: chunkEntries) {
var chunk = new Chunk(instance, entry.chunkX(), entry.chunkZ());
var chunkData = entry.binaryData();
final String status = chunkData.getString("status");
if (status.isEmpty() || "minecraft:full".equals(status)) {
loadSections(chunk, chunkData);
loadBlockEntities(chunk, chunkData);
chunk.loadHeightmapsFromNBT(chunkData.getCompound("Heightmaps"));
list.add(new AnvilChunkData(
chunk.getChunkX(),
chunk.getChunkZ(),
chunk.getSections().stream().map(Section::clone).toList(),
chunk.getEntries()
));
} else {
log.warn("Skipping partially generated chunk at {}, {} with status {}", entry.chunkX(), entry.chunkZ(), status);
}
}
MinecraftServer.getInstanceManager().unregisterInstance(instance);
return list;
}
private void loadSections(@NotNull Chunk chunk, @NotNull CompoundBinaryTag chunkData) {
for (BinaryTag sectionTag : chunkData.getList("sections", BinaryTagTypes.COMPOUND)) {
final CompoundBinaryTag sectionData = (CompoundBinaryTag) sectionTag;
final int sectionY = sectionData.getInt("Y", Integer.MIN_VALUE);
Check.stateCondition(sectionY == Integer.MIN_VALUE, "Missing section Y value");
final int yOffset = Chunk.CHUNK_SECTION_SIZE * sectionY;
if (sectionY < chunk.getMinSection() || sectionY >= chunk.getMaxSection()) {
// Vanilla stores a section below and above the world for lighting, throw it out.
continue;
}
final Section section = chunk.getSection(sectionY);
// Lighting
if (sectionData.get("SkyLight") instanceof ByteArrayBinaryTag skyLightTag && skyLightTag.size() == 2048) {
section.setSkyLight(skyLightTag.value());
}
if (sectionData.get("BlockLight") instanceof ByteArrayBinaryTag blockLightTag && blockLightTag.size() == 2048) {
section.setBlockLight(blockLightTag.value());
}
{ // Biomes
final CompoundBinaryTag biomesTag = sectionData.getCompound("biomes");
final ListBinaryTag biomePaletteTag = biomesTag.getList("palette", BinaryTagTypes.STRING);
int[] convertedBiomePalette = loadBiomePalette(biomePaletteTag);
if (convertedBiomePalette.length == 1) {
// One solid block, no need to check the data
section.biomePalette().fill(convertedBiomePalette[0]);
} else if (convertedBiomePalette.length > 1) {
final long[] packedIndices = biomesTag.getLongArray("data");
Check.stateCondition(packedIndices.length == 0, "Missing packed biomes data");
int[] biomeIndices = new int[64];
int bitsPerEntry = packedIndices.length * 64 / biomeIndices.length;
if (bitsPerEntry > 3) bitsPerEntry = MathUtils.bitsToRepresent(convertedBiomePalette.length);
ArrayUtils.unpack(biomeIndices, packedIndices, bitsPerEntry);
section.biomePalette().setAll((x, y, z) -> {
final int index = x + z * 4 + y * 16;
return convertedBiomePalette[biomeIndices[index]];
});
}
}
{ // Blocks
final CompoundBinaryTag blockStatesTag = sectionData.getCompound("block_states");
final ListBinaryTag blockPaletteTag = blockStatesTag.getList("palette", BinaryTagTypes.COMPOUND);
Block[] convertedPalette = loadBlockPalette(blockPaletteTag);
if (blockPaletteTag.size() == 1) {
// One solid block, no need to check the data
section.blockPalette().fill(convertedPalette[0].stateId());
} else if (blockPaletteTag.size() > 1) {
final long[] packedStates = blockStatesTag.getLongArray("data");
Check.stateCondition(packedStates.length == 0, "Missing packed states data");
int[] blockStateIndices = new int[Chunk.CHUNK_SECTION_SIZE * Chunk.CHUNK_SECTION_SIZE * Chunk.CHUNK_SECTION_SIZE];
ArrayUtils.unpack(blockStateIndices, packedStates, packedStates.length * 64 / blockStateIndices.length);
for (int y = 0; y < Chunk.CHUNK_SECTION_SIZE; y++) {
for (int z = 0; z < Chunk.CHUNK_SECTION_SIZE; z++) {
for (int x = 0; x < Chunk.CHUNK_SECTION_SIZE; x++) {
try {
final int blockIndex = y * Chunk.CHUNK_SECTION_SIZE * Chunk.CHUNK_SECTION_SIZE + z * Chunk.CHUNK_SECTION_SIZE + x;
final int paletteIndex = blockStateIndices[blockIndex];
final Block block = convertedPalette[paletteIndex];
chunk.setBlock(x, y + yOffset, z, block);
} catch (Exception e) {
MinecraftServer.getExceptionManager().handleException(e);
}
}
}
}
}
}
}
}
private Block[] loadBlockPalette(@NotNull ListBinaryTag paletteTag) {
Block[] convertedPalette = new Block[paletteTag.size()];
for (int i = 0; i < convertedPalette.length; i++) {
CompoundBinaryTag paletteEntry = paletteTag.getCompound(i);
String blockName = paletteEntry.getString("Name");
if (blockName.equals("minecraft:air")) {
convertedPalette[i] = Block.AIR;
} else {
Block block = Objects.requireNonNull(Block.fromNamespaceId(blockName), "Unknown block " + blockName);
// Properties
final Map<String, String> properties = new HashMap<>();
CompoundBinaryTag propertiesNBT = paletteEntry.getCompound("Properties");
for (var property : propertiesNBT) {
if (property.getValue() instanceof StringBinaryTag propertyValue) {
properties.put(property.getKey(), propertyValue.value());
} else {
log.warn("Fail to parse block state properties {}, expected a string for {}, but contents were {}", propertiesNBT, property.getKey(), TagStringIOExt.writeTag(property.getValue()));
}
}
if (!properties.isEmpty()) block = block.withProperties(properties);
// Handler
final BlockHandler handler = MinecraftServer.getBlockManager().getHandler(block.name());
if (handler != null) block = block.withHandler(handler);
convertedPalette[i] = block;
}
}
return convertedPalette;
}
private int[] loadBiomePalette(@NotNull ListBinaryTag paletteTag) {
int[] convertedPalette = new int[paletteTag.size()];
for (int i = 0; i < convertedPalette.length; i++) {
final String name = paletteTag.getString(i);
int biomeId = BIOME_REGISTRY.getId(NamespaceID.from(name));
if (biomeId == -1) biomeId = PLAINS_ID;
convertedPalette[i] = biomeId;
}
return convertedPalette;
}
private void loadBlockEntities(@NotNull Chunk loadedChunk, @NotNull CompoundBinaryTag chunkData) {
for (BinaryTag blockEntityTag : chunkData.getList("block_entities", BinaryTagTypes.COMPOUND)) {
final CompoundBinaryTag blockEntity = (CompoundBinaryTag) blockEntityTag;
final int x = blockEntity.getInt("x");
final int y = blockEntity.getInt("y");
final int z = blockEntity.getInt("z");
Block block = loadedChunk.getBlock(x, y, z);
// Load the block handler if the id is present
if (blockEntity.get("id") instanceof StringBinaryTag blockEntityId) {
final BlockHandler handler = MinecraftServer.getBlockManager().getHandlerOrDummy(blockEntityId.value());
block = block.withHandler(handler);
}
// Remove anvil tags
CompoundBinaryTag trimmedTag = CompoundBinaryTag.builder().put(blockEntity)
.remove("id").remove("keepPacked")
.remove("x").remove("y").remove("z")
.build();
// Place block
final var finalBlock = trimmedTag.size() > 0 ? block.withNbt(trimmedTag) : block;
loadedChunk.setBlock(x, y, z, finalBlock);
}
}
public record ChunkEntry(int chunkX, int chunkZ, CompoundBinaryTag binaryData) {}
private static class Chunk extends DynamicChunk {
public Chunk(@NotNull Instance instance, int chunkX, int chunkZ) {
super(instance, chunkX, chunkZ);
}
public Int2ObjectOpenHashMap<Block> getEntries() {
return entries;
}
}
}

View File

@ -0,0 +1,130 @@
package ru.dragonestia.msb3.api.world.loader.anvil;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.booleans.BooleanList;
import lombok.Getter;
import lombok.extern.log4j.Log4j2;
import net.kyori.adventure.nbt.*;
import net.minestom.server.utils.chunk.ChunkUtils;
import net.minestom.server.utils.validate.Check;
import org.jetbrains.annotations.Nullable;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.concurrent.locks.ReentrantLock;
@Log4j2
public class RegionFile implements AutoCloseable {
private static final int MAX_ENTRY_COUNT = 1024;
private static final int SECTOR_SIZE = 4096;
private static final int HEADER_LENGTH = MAX_ENTRY_COUNT * 2 * 4; // 2 4-byte fields per entry
private static final int COMPRESSION_ZLIB = 2;
private static final BinaryTagIO.Reader TAG_READER = BinaryTagIO.unlimitedReader();
private final ReentrantLock lock = new ReentrantLock();
private final RandomAccessFile file;
private final int[] locations = new int[MAX_ENTRY_COUNT];
private final BooleanList freeSectors = new BooleanArrayList(2);
@Getter private final int regionX;
@Getter private final int regionZ;
public RegionFile(File regionFile) throws IOException {
this.file = new RandomAccessFile(regionFile, "rw");
var fileName = regionFile.getName();
assert fileName.matches("r\\.-?\\d+\\.-?\\d+\\.mca");
String[] pos = fileName.substring("r.".length(), fileName.length() - ".mca".length()).split("\\.");
regionX = Integer.parseInt(pos[0]);
regionZ = Integer.parseInt(pos[1]);
readHeader();
}
public boolean hasChunkData(int chunkX, int chunkZ) {
lock.lock();
try {
return locations[getChunkIndex(chunkX, chunkZ)] != 0;
} finally {
lock.unlock();
}
}
public @Nullable CompoundBinaryTag readChunkData(int chunkX, int chunkZ) throws IOException {
lock.lock();
try {
if (!hasChunkData(chunkX, chunkZ)) return null;
int location = locations[getChunkIndex(chunkX, chunkZ)];
file.seek((long) (location >> 8) * SECTOR_SIZE); // Move to start of first sector
int length = file.readInt();
int compressionType = file.readByte();
BinaryTagIO.Compression compression = switch (compressionType) {
case 1 -> BinaryTagIO.Compression.GZIP;
case COMPRESSION_ZLIB -> BinaryTagIO.Compression.ZLIB;
case 3 -> BinaryTagIO.Compression.NONE;
default -> throw new IOException("Unsupported compression type: " + compressionType);
};
// Read the raw content
byte[] data = new byte[length - 1];
file.read(data);
// Parse it as a compound tag
return TAG_READER.read(new ByteArrayInputStream(data), compression);
} finally {
lock.unlock();
}
}
@Override
public void close() throws IOException {
file.close();
}
private int getChunkIndex(int chunkX, int chunkZ) {
return (ChunkUtils.toRegionLocal(chunkZ) << 5) | ChunkUtils.toRegionLocal(chunkX);
}
private void readHeader() throws IOException {
file.seek(0);
if (file.length() < HEADER_LENGTH) {
// new file, fill in data
file.write(new byte[HEADER_LENGTH]);
}
final long totalSectors = ((file.length() - 1) / SECTOR_SIZE) + 1; // Round up, last sector does not need to be full size
for (int i = 0; i < totalSectors; i++) freeSectors.add(true);
freeSectors.set(0, false); // First sector is locations
freeSectors.set(1, false); // Second sector is timestamps
// Read locations
file.seek(0);
for (int i = 0; i < MAX_ENTRY_COUNT; i++) {
int location = locations[i] = file.readInt();
if (location != 0) {
markLocation(location, false);
}
}
// Read timestamps
for (int i = 0; i < MAX_ENTRY_COUNT; i++) {
file.readInt();
}
}
private void markLocation(int location, boolean free) {
int sectorCount = location & 0xFF;
int sectorStart = location >> 8;
Check.stateCondition(sectorStart + sectorCount > freeSectors.size(), "Invalid sector count");
for (int i = sectorStart; i < sectorStart + sectorCount; i++) {
freeSectors.set(i, free);
}
}
}