Skip to content

Commit e638056

Browse files
lhofhansllhofhansl
committedDec 15, 2020
Allow configuring block disk and net compression. Change default disk level.
1 parent d0a38f6 commit e638056

10 files changed

+47
-40
lines changed
 

‎builtin/settingtypes.txt

+14
Original file line numberDiff line numberDiff line change
@@ -1048,6 +1048,13 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui
10481048
# client number.
10491049
max_packets_per_iteration (Max. packets per iteration) int 1024
10501050

1051+
# ZLib compression level to use when sending mapblocks to the client.
1052+
# -1 - Zlib's default compression level
1053+
# 0 - no compresson, fastest
1054+
# 9 - best compression, slowest
1055+
# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
1056+
map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9
1057+
10511058
[*Game]
10521059

10531060
# Default game when creating a new world.
@@ -1240,6 +1247,13 @@ max_objects_per_block (Maximum objects per block) int 64
12401247
# See https://www.sqlite.org/pragma.html#pragma_synchronous
12411248
sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2
12421249

1250+
# ZLib compression level to use when saving mapblocks to disk.
1251+
# -1 - Zlib's default compression level
1252+
# 0 - no compresson, fastest
1253+
# 9 - best compression, slowest
1254+
# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
1255+
map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9
1256+
12431257
# Length of a server tick and the interval at which objects are generally updated over
12441258
# network.
12451259
dedicated_server_step (Dedicated server step) float 0.09

‎src/defaultsettings.cpp

+4
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,8 @@ void set_default_settings(Settings *settings)
385385
settings->setDefault("chat_message_limit_per_10sec", "8.0");
386386
settings->setDefault("chat_message_limit_trigger_kick", "50");
387387
settings->setDefault("sqlite_synchronous", "2");
388+
settings->setDefault("map_compression_level_disk", "3");
389+
settings->setDefault("map_compression_level_net", "-1");
388390
settings->setDefault("full_block_send_enable_min_time_from_building", "2.0");
389391
settings->setDefault("dedicated_server_step", "0.09");
390392
settings->setDefault("active_block_mgmt_interval", "2.0");
@@ -470,6 +472,8 @@ void set_default_settings(Settings *settings)
470472
settings->setDefault("fps_max_unfocused", "10");
471473
settings->setDefault("max_objects_per_block", "20");
472474
settings->setDefault("sqlite_synchronous", "1");
475+
settings->setDefault("map_compression_level_disk", "-1");
476+
settings->setDefault("map_compression_level_net", "3");
473477
settings->setDefault("server_map_save_interval", "15");
474478
settings->setDefault("client_mapblock_limit", "1000");
475479
settings->setDefault("active_block_range", "2");

‎src/map.cpp

+5-3
Original file line numberDiff line numberDiff line change
@@ -1235,6 +1235,8 @@ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef,
12351235

12361236
m_save_time_counter = mb->addCounter("minetest_core_map_save_time", "Map save time (in nanoseconds)");
12371237

1238+
m_map_compression_level = rangelim(g_settings->getS16("map_compression_level_disk"), -1, 9);
1239+
12381240
try {
12391241
// If directory exists, check contents and load if possible
12401242
if (fs::PathExists(m_savedir)) {
@@ -1863,10 +1865,10 @@ void ServerMap::endSave()
18631865

18641866
bool ServerMap::saveBlock(MapBlock *block)
18651867
{
1866-
return saveBlock(block, dbase);
1868+
return saveBlock(block, dbase, m_map_compression_level);
18671869
}
18681870

1869-
bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db)
1871+
bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db, int compression_level)
18701872
{
18711873
v3s16 p3d = block->getPos();
18721874

@@ -1886,7 +1888,7 @@ bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db)
18861888
*/
18871889
std::ostringstream o(std::ios_base::binary);
18881890
o.write((char*) &version, 1);
1889-
block->serialize(o, version, true);
1891+
block->serialize(o, version, true, compression_level);
18901892

18911893
bool ret = db->saveBlock(p3d, o.str());
18921894
if (ret) {

‎src/map.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ class ServerMap : public Map
381381
MapgenParams *getMapgenParams();
382382

383383
bool saveBlock(MapBlock *block);
384-
static bool saveBlock(MapBlock *block, MapDatabase *db);
384+
static bool saveBlock(MapBlock *block, MapDatabase *db, int compression_level = -1);
385385
MapBlock* loadBlock(v3s16 p);
386386
// Database version
387387
void loadBlock(std::string *blob, v3s16 p3d, MapSector *sector, bool save_after_load=false);
@@ -416,6 +416,7 @@ class ServerMap : public Map
416416
std::string m_savedir;
417417
bool m_map_saving_enabled;
418418

419+
int m_map_compression_level;
419420
#if 0
420421
// Chunk size in MapSectors
421422
// If 0, chunks are disabled.

‎src/mapblock.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes,
355355
}
356356
}
357357

358-
void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
358+
void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level)
359359
{
360360
if(!ser_ver_supported(version))
361361
throw VersionMismatchException("ERROR: MapBlock format not supported");
@@ -394,7 +394,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
394394
writeU8(os, content_width);
395395
writeU8(os, params_width);
396396
MapNode::serializeBulk(os, version, tmp_nodes, nodecount,
397-
content_width, params_width, true);
397+
content_width, params_width, compression_level);
398398
delete[] tmp_nodes;
399399
}
400400
else
@@ -404,15 +404,15 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
404404
writeU8(os, content_width);
405405
writeU8(os, params_width);
406406
MapNode::serializeBulk(os, version, data, nodecount,
407-
content_width, params_width, true);
407+
content_width, params_width, compression_level);
408408
}
409409

410410
/*
411411
Node metadata
412412
*/
413413
std::ostringstream oss(std::ios_base::binary);
414414
m_node_metadata.serialize(oss, version, disk);
415-
compressZlib(oss.str(), os);
415+
compressZlib(oss.str(), os, compression_level);
416416

417417
/*
418418
Data that goes to disk, but not the network
@@ -485,7 +485,7 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
485485
if(params_width != 2)
486486
throw SerializationError("MapBlock::deSerialize(): invalid params_width");
487487
MapNode::deSerializeBulk(is, version, data, nodecount,
488-
content_width, params_width, true);
488+
content_width, params_width);
489489

490490
/*
491491
NodeMetadata

‎src/mapblock.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ class MapBlock
482482
// These don't write or read version by itself
483483
// Set disk to true for on-disk format, false for over-the-network format
484484
// Precondition: version >= SER_FMT_VER_LOWEST_WRITE
485-
void serialize(std::ostream &os, u8 version, bool disk);
485+
void serialize(std::ostream &os, u8 version, bool disk, int compression_level);
486486
// If disk == true: In addition to doing other things, will add
487487
// unknown blocks from id-name mapping to wndef
488488
void deSerialize(std::istream &is, u8 version, bool disk);

‎src/mapgen/mg_schematic.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ bool Schematic::deserializeFromMts(std::istream *is,
334334
schemdata = new MapNode[nodecount];
335335

336336
MapNode::deSerializeBulk(ss, SER_FMT_VER_HIGHEST_READ, schemdata,
337-
nodecount, 2, 2, true);
337+
nodecount, 2, 2);
338338

339339
// Fix probability values for nodes that were ignore; removed in v2
340340
if (version < 2) {
@@ -376,7 +376,7 @@ bool Schematic::serializeToMts(std::ostream *os,
376376

377377
// compressed bulk node data
378378
MapNode::serializeBulk(ss, SER_FMT_VER_HIGHEST_WRITE,
379-
schemdata, size.X * size.Y * size.Z, 2, 2, true);
379+
schemdata, size.X * size.Y * size.Z, 2, 2, -1);
380380

381381
return true;
382382
}

‎src/mapnode.cpp

+10-24
Original file line numberDiff line numberDiff line change
@@ -706,7 +706,7 @@ void MapNode::deSerialize(u8 *source, u8 version)
706706
}
707707
void MapNode::serializeBulk(std::ostream &os, int version,
708708
const MapNode *nodes, u32 nodecount,
709-
u8 content_width, u8 params_width, bool compressed)
709+
u8 content_width, u8 params_width, int compression_level)
710710
{
711711
if (!ser_ver_supported(version))
712712
throw VersionMismatchException("ERROR: MapNode format not supported");
@@ -737,18 +737,15 @@ void MapNode::serializeBulk(std::ostream &os, int version,
737737
Compress data to output stream
738738
*/
739739

740-
if (compressed)
741-
compressZlib(databuf, databuf_size, os);
742-
else
743-
os.write((const char*) &databuf[0], databuf_size);
740+
compressZlib(databuf, databuf_size, os, compression_level);
744741

745742
delete [] databuf;
746743
}
747744

748745
// Deserialize bulk node data
749746
void MapNode::deSerializeBulk(std::istream &is, int version,
750747
MapNode *nodes, u32 nodecount,
751-
u8 content_width, u8 params_width, bool compressed)
748+
u8 content_width, u8 params_width)
752749
{
753750
if(!ser_ver_supported(version))
754751
throw VersionMismatchException("ERROR: MapNode format not supported");
@@ -760,24 +757,13 @@ void MapNode::deSerializeBulk(std::istream &is, int version,
760757

761758
// Uncompress or read data
762759
u32 len = nodecount * (content_width + params_width);
763-
SharedBuffer<u8> databuf(len);
764-
if(compressed)
765-
{
766-
std::ostringstream os(std::ios_base::binary);
767-
decompressZlib(is, os);
768-
std::string s = os.str();
769-
if(s.size() != len)
770-
throw SerializationError("deSerializeBulkNodes: "
771-
"decompress resulted in invalid size");
772-
memcpy(&databuf[0], s.c_str(), len);
773-
}
774-
else
775-
{
776-
is.read((char*) &databuf[0], len);
777-
if(is.eof() || is.fail())
778-
throw SerializationError("deSerializeBulkNodes: "
779-
"failed to read bulk node data");
780-
}
760+
std::ostringstream os(std::ios_base::binary);
761+
decompressZlib(is, os);
762+
std::string s = os.str();
763+
if(s.size() != len)
764+
throw SerializationError("deSerializeBulkNodes: "
765+
"decompress resulted in invalid size");
766+
const u8 *databuf = reinterpret_cast<const u8*>(s.c_str());
781767

782768
// Deserialize content
783769
if(content_width == 1)

‎src/mapnode.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -292,10 +292,10 @@ struct MapNode
292292
// compressed = true to zlib-compress output
293293
static void serializeBulk(std::ostream &os, int version,
294294
const MapNode *nodes, u32 nodecount,
295-
u8 content_width, u8 params_width, bool compressed);
295+
u8 content_width, u8 params_width, int compression_level);
296296
static void deSerializeBulk(std::istream &is, int version,
297297
MapNode *nodes, u32 nodecount,
298-
u8 content_width, u8 params_width, bool compressed);
298+
u8 content_width, u8 params_width);
299299

300300
private:
301301
// Deprecated serialization methods

‎src/server.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -2332,9 +2332,9 @@ void Server::SendBlockNoLock(session_t peer_id, MapBlock *block, u8 ver,
23322332
/*
23332333
Create a packet with the block in the right format
23342334
*/
2335-
2335+
thread_local const int net_compression_level = rangelim(g_settings->getS16("map_compression_level_net"), -1, 9);
23362336
std::ostringstream os(std::ios_base::binary);
2337-
block->serialize(os, ver, false);
2337+
block->serialize(os, ver, false, net_compression_level);
23382338
block->serializeNetworkSpecific(os);
23392339
std::string s = os.str();
23402340

0 commit comments

Comments
 (0)
Please sign in to comment.