Allow configuring block disk and net compression. Change default disk…

… level.
minetest · Dec 15, 2020 · e638056 · e638056
1 parent d0a38f6
commit e638056
Show file tree

Hide file tree

Showing 10 changed files with 47 additions and 40 deletions.
diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt
@@ -1048,6 +1048,13 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui
 #    client number.
 max_packets_per_iteration (Max. packets per iteration) int 1024
 
+#    ZLib compression level to use when sending mapblocks to the client.
+#    -1 - Zlib's default compression level
+#     0 - no compresson, fastest
+#     9 - best compression, slowest
+#     (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
+map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9
+
 [*Game]
 
 #    Default game when creating a new world.
@@ -1240,6 +1247,13 @@ max_objects_per_block (Maximum objects per block) int 64
 #    See https://www.sqlite.org/pragma.html#pragma_synchronous
 sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2
 
+#    ZLib compression level to use when saving mapblocks to disk.
+#    -1 - Zlib's default compression level
+#     0 - no compresson, fastest
+#     9 - best compression, slowest
+#     (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
+map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9
+
 #    Length of a server tick and the interval at which objects are generally updated over
 #    network.
 dedicated_server_step (Dedicated server step) float 0.09

diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp
@@ -385,6 +385,8 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("chat_message_limit_per_10sec", "8.0");
 	settings->setDefault("chat_message_limit_trigger_kick", "50");
 	settings->setDefault("sqlite_synchronous", "2");
+	settings->setDefault("map_compression_level_disk", "3");
+	settings->setDefault("map_compression_level_net", "-1");
 	settings->setDefault("full_block_send_enable_min_time_from_building", "2.0");
 	settings->setDefault("dedicated_server_step", "0.09");
 	settings->setDefault("active_block_mgmt_interval", "2.0");
@@ -470,6 +472,8 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("fps_max_unfocused", "10");
 	settings->setDefault("max_objects_per_block", "20");
 	settings->setDefault("sqlite_synchronous", "1");
+	settings->setDefault("map_compression_level_disk", "-1");
+	settings->setDefault("map_compression_level_net", "3");
 	settings->setDefault("server_map_save_interval", "15");
 	settings->setDefault("client_mapblock_limit", "1000");
 	settings->setDefault("active_block_range", "2");

diff --git a/src/map.cpp b/src/map.cpp
@@ -1235,6 +1235,8 @@ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef,
 
 	m_save_time_counter = mb->addCounter("minetest_core_map_save_time", "Map save time (in nanoseconds)");
 
+	m_map_compression_level = rangelim(g_settings->getS16("map_compression_level_disk"), -1, 9);
+
 	try {
 		// If directory exists, check contents and load if possible
 		if (fs::PathExists(m_savedir)) {
@@ -1863,10 +1865,10 @@ void ServerMap::endSave()
 
 bool ServerMap::saveBlock(MapBlock *block)
 {
-	return saveBlock(block, dbase);
+	return saveBlock(block, dbase, m_map_compression_level);
 }
 
-bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db)
+bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db, int compression_level)
 {
 	v3s16 p3d = block->getPos();
 
@@ -1886,7 +1888,7 @@ bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db)
 	*/
 	std::ostringstream o(std::ios_base::binary);
 	o.write((char*) &version, 1);
-	block->serialize(o, version, true);
+	block->serialize(o, version, true, compression_level);
 
 	bool ret = db->saveBlock(p3d, o.str());
 	if (ret) {

diff --git a/src/map.h b/src/map.h
@@ -381,7 +381,7 @@ class ServerMap : public Map
 	MapgenParams *getMapgenParams();
 
 	bool saveBlock(MapBlock *block);
-	static bool saveBlock(MapBlock *block, MapDatabase *db);
+	static bool saveBlock(MapBlock *block, MapDatabase *db, int compression_level = -1);
 	MapBlock* loadBlock(v3s16 p);
 	// Database version
 	void loadBlock(std::string *blob, v3s16 p3d, MapSector *sector, bool save_after_load=false);
@@ -416,6 +416,7 @@ class ServerMap : public Map
 	std::string m_savedir;
 	bool m_map_saving_enabled;
 
+	int m_map_compression_level;
 #if 0
 	// Chunk size in MapSectors
 	// If 0, chunks are disabled.

diff --git a/src/mapblock.cpp b/src/mapblock.cpp
@@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes,
 	}
 }
 
-void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
+void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level)
 {
 	if(!ser_ver_supported(version))
 		throw VersionMismatchException("ERROR: MapBlock format not supported");
@@ -394,7 +394,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
 		writeU8(os, content_width);
 		writeU8(os, params_width);
 		MapNode::serializeBulk(os, version, tmp_nodes, nodecount,
-				content_width, params_width, true);
+				content_width, params_width, compression_level);
 		delete[] tmp_nodes;
 	}
 	else
@@ -404,15 +404,15 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
 		writeU8(os, content_width);
 		writeU8(os, params_width);
 		MapNode::serializeBulk(os, version, data, nodecount,
-				content_width, params_width, true);
+				content_width, params_width, compression_level);
 	}
 
 	/*
 		Node metadata
 	*/
 	std::ostringstream oss(std::ios_base::binary);
 	m_node_metadata.serialize(oss, version, disk);
-	compressZlib(oss.str(), os);
+	compressZlib(oss.str(), os, compression_level);
 
 	/*
 		Data that goes to disk, but not the network
@@ -485,7 +485,7 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
 	if(params_width != 2)
 		throw SerializationError("MapBlock::deSerialize(): invalid params_width");
 	MapNode::deSerializeBulk(is, version, data, nodecount,
-			content_width, params_width, true);
+			content_width, params_width);
 
 	/*
 		NodeMetadata

diff --git a/src/mapblock.h b/src/mapblock.h
@@ -482,7 +482,7 @@ class MapBlock
 	// These don't write or read version by itself
 	// Set disk to true for on-disk format, false for over-the-network format
 	// Precondition: version >= SER_FMT_VER_LOWEST_WRITE
-	void serialize(std::ostream &os, u8 version, bool disk);
+	void serialize(std::ostream &os, u8 version, bool disk, int compression_level);
 	// If disk == true: In addition to doing other things, will add
 	// unknown blocks from id-name mapping to wndef
 	void deSerialize(std::istream &is, u8 version, bool disk);

diff --git a/src/mapgen/mg_schematic.cpp b/src/mapgen/mg_schematic.cpp
@@ -334,7 +334,7 @@ bool Schematic::deserializeFromMts(std::istream *is,
 	schemdata = new MapNode[nodecount];
 
 	MapNode::deSerializeBulk(ss, SER_FMT_VER_HIGHEST_READ, schemdata,
-		nodecount, 2, 2, true);
+		nodecount, 2, 2);
 
 	// Fix probability values for nodes that were ignore; removed in v2
 	if (version < 2) {
@@ -376,7 +376,7 @@ bool Schematic::serializeToMts(std::ostream *os,
 
 	// compressed bulk node data
 	MapNode::serializeBulk(ss, SER_FMT_VER_HIGHEST_WRITE,
-		schemdata, size.X * size.Y * size.Z, 2, 2, true);
+		schemdata, size.X * size.Y * size.Z, 2, 2, -1);
 
 	return true;
 }

diff --git a/src/mapnode.cpp b/src/mapnode.cpp
@@ -706,7 +706,7 @@ void MapNode::deSerialize(u8 *source, u8 version)
 }
 void MapNode::serializeBulk(std::ostream &os, int version,
 		const MapNode *nodes, u32 nodecount,
-		u8 content_width, u8 params_width, bool compressed)
+		u8 content_width, u8 params_width, int compression_level)
 {
 	if (!ser_ver_supported(version))
 		throw VersionMismatchException("ERROR: MapNode format not supported");
@@ -737,18 +737,15 @@ void MapNode::serializeBulk(std::ostream &os, int version,
 		Compress data to output stream
 	*/
 
-	if (compressed)
-		compressZlib(databuf, databuf_size, os);
-	else
-		os.write((const char*) &databuf[0], databuf_size);
+	compressZlib(databuf, databuf_size, os, compression_level);
 
 	delete [] databuf;
 }
 
 // Deserialize bulk node data
 void MapNode::deSerializeBulk(std::istream &is, int version,
 		MapNode *nodes, u32 nodecount,
-		u8 content_width, u8 params_width, bool compressed)
+		u8 content_width, u8 params_width)
 {
 	if(!ser_ver_supported(version))
 		throw VersionMismatchException("ERROR: MapNode format not supported");
@@ -760,24 +757,13 @@ void MapNode::deSerializeBulk(std::istream &is, int version,
 
 	// Uncompress or read data
 	u32 len = nodecount * (content_width + params_width);
-	SharedBuffer<u8> databuf(len);
-	if(compressed)
-	{
-		std::ostringstream os(std::ios_base::binary);
-		decompressZlib(is, os);
-		std::string s = os.str();
-		if(s.size() != len)
-			throw SerializationError("deSerializeBulkNodes: "
-					"decompress resulted in invalid size");
-		memcpy(&databuf[0], s.c_str(), len);
-	}
-	else
-	{
-		is.read((char*) &databuf[0], len);
-		if(is.eof() || is.fail())
-			throw SerializationError("deSerializeBulkNodes: "
-					"failed to read bulk node data");
-	}
+	std::ostringstream os(std::ios_base::binary);
+	decompressZlib(is, os);
+	std::string s = os.str();
+	if(s.size() != len)
+		throw SerializationError("deSerializeBulkNodes: "
+				"decompress resulted in invalid size");
+	const u8 *databuf = reinterpret_cast<const u8*>(s.c_str());
 
 	// Deserialize content
 	if(content_width == 1)

diff --git a/src/mapnode.h b/src/mapnode.h
@@ -292,10 +292,10 @@ struct MapNode
 	//   compressed = true to zlib-compress output
 	static void serializeBulk(std::ostream &os, int version,
 			const MapNode *nodes, u32 nodecount,
-			u8 content_width, u8 params_width, bool compressed);
+			u8 content_width, u8 params_width, int compression_level);
 	static void deSerializeBulk(std::istream &is, int version,
 			MapNode *nodes, u32 nodecount,
-			u8 content_width, u8 params_width, bool compressed);
+			u8 content_width, u8 params_width);
 
 private:
 	// Deprecated serialization methods

diff --git a/src/server.cpp b/src/server.cpp
@@ -2332,9 +2332,9 @@ void Server::SendBlockNoLock(session_t peer_id, MapBlock *block, u8 ver,
 	/*
 		Create a packet with the block in the right format
 	*/
-
+	thread_local const int net_compression_level = rangelim(g_settings->getS16("map_compression_level_net"), -1, 9);
 	std::ostringstream os(std::ios_base::binary);
-	block->serialize(os, ver, false);
+	block->serialize(os, ver, false, net_compression_level);
 	block->serializeNetworkSpecific(os);
 	std::string s = os.str();