compressZlib: don't use a SharedBuffer but a raw u8 * pointer

Remove usage of the SharedBuffer in zlib compression which has two problems: * We copied the whole memory block to compress it (not good with mapblocks) * We copied sometimes strings to SharedBuffer to SharedBuffer (2nd time) Use this method in MapNode::serializeBulk + optimize serialization but merging 3 identical loops in a single loop
minetest · Jul 27, 2017 · c27504a · c27504a
1 parent 61e4877
commit c27504a
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 40 deletions.
diff --git a/src/mapnode.cpp b/src/mapnode.cpp
@@ -658,46 +658,41 @@ void MapNode::serializeBulk(std::ostream &os, int version,
 		const MapNode *nodes, u32 nodecount,
 		u8 content_width, u8 params_width, bool compressed)
 {
-	if(!ser_ver_supported(version))
+	if (!ser_ver_supported(version))
 		throw VersionMismatchException("ERROR: MapNode format not supported");
 
 	sanity_check(content_width == 2);
 	sanity_check(params_width == 2);
 
 	// Can't do this anymore; we have 16-bit dynamically allocated node IDs
 	// in memory; conversion just won't work in this direction.
-	if(version < 24)
+	if (version < 24)
 		throw SerializationError("MapNode::serializeBulk: serialization to "
 				"version < 24 not possible");
 
-	SharedBuffer<u8> databuf(nodecount * (content_width + params_width));
-
-	// Serialize content
-	for(u32 i=0; i<nodecount; i++)
-		writeU16(&databuf[i*2], nodes[i].param0);
+	size_t databuf_size = nodecount * (content_width + params_width);
+	u8 *databuf = new u8[databuf_size];
 
-	// Serialize param1
 	u32 start1 = content_width * nodecount;
-	for(u32 i=0; i<nodecount; i++)
-		writeU8(&databuf[start1 + i], nodes[i].param1);
-
-	// Serialize param2
 	u32 start2 = (content_width + 1) * nodecount;
-	for(u32 i=0; i<nodecount; i++)
+
+	// Serialize content
+	for (u32 i = 0; i < nodecount; i++) {
+		writeU16(&databuf[i * 2], nodes[i].param0);
+		writeU8(&databuf[start1 + i], nodes[i].param1);
 		writeU8(&databuf[start2 + i], nodes[i].param2);
+	}
 
 	/*
 		Compress data to output stream
 	*/
 
-	if(compressed)
-	{
-		compressZlib(databuf, os);
-	}
+	if (compressed)
+		compressZlib(databuf, databuf_size, os);
 	else
-	{
-		os.write((const char*) &databuf[0], databuf.getSize());
-	}
+		os.write((const char*) &databuf[0], databuf_size);
+
+	delete [] databuf;
 }
 
 // Deserialize bulk node data

diff --git a/src/serialization.cpp b/src/serialization.cpp
@@ -27,7 +27,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 /* report a zlib or i/o error */
 void zerr(int ret)
-{   
+{
     dstream<<"zerr: ";
     switch (ret) {
     case Z_ERRNO:
@@ -53,7 +53,7 @@ void zerr(int ret)
     }
 }
 
-void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
+void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
 {
 	z_stream z;
 	const s32 bufsize = 16384;
@@ -68,16 +68,16 @@ void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
 	ret = deflateInit(&z, level);
 	if(ret != Z_OK)
 		throw SerializationError("compressZlib: deflateInit failed");
-	
+
 	// Point zlib to our input buffer
 	z.next_in = (Bytef*)&data[0];
-	z.avail_in = data.getSize();
+	z.avail_in = data_size;
 	// And get all output
 	for(;;)
 	{
 		z.next_out = (Bytef*)output_buffer;
 		z.avail_out = bufsize;
-		
+
 		status = deflate(&z, Z_FINISH);
 		if(status == Z_NEED_DICT || status == Z_DATA_ERROR
 				|| status == Z_MEM_ERROR)
@@ -98,8 +98,7 @@ void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
 
 void compressZlib(const std::string &data, std::ostream &os, int level)
 {
-	SharedBuffer<u8> databuf((u8*)data.c_str(), data.size());
-	compressZlib(databuf, os, level);
+	compressZlib((u8*)data.c_str(), data.size(), os, level);
 }
 
 void decompressZlib(std::istream &is, std::ostream &os)
@@ -120,9 +119,9 @@ void decompressZlib(std::istream &is, std::ostream &os)
 	ret = inflateInit(&z);
 	if(ret != Z_OK)
 		throw SerializationError("dcompressZlib: inflateInit failed");
-	
+
 	z.avail_in = 0;
-	
+
 	//dstream<<"initial fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 
 	for(;;)
@@ -143,7 +142,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
 			//dstream<<"z.avail_in == 0"<<std::endl;
 			break;
 		}
-			
+
 		//dstream<<"1 z.avail_in="<<z.avail_in<<std::endl;
 		status = inflate(&z, Z_NO_FLUSH);
 		//dstream<<"2 z.avail_in="<<z.avail_in<<std::endl;
@@ -163,7 +162,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
 		if(status == Z_STREAM_END)
 		{
 			//dstream<<"Z_STREAM_END"<<std::endl;
-			
+
 			//dstream<<"z.avail_in="<<z.avail_in<<std::endl;
 			//dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
 			// Unget all the data that inflate didn't take
@@ -178,31 +177,31 @@ void decompressZlib(std::istream &is, std::ostream &os)
 					throw SerializationError("decompressZlib: unget failed");
 				}
 			}
-			
+
 			break;
 		}
 	}
 
 	inflateEnd(&z);
 }
 
-void compress(SharedBuffer<u8> data, std::ostream &os, u8 version)
+void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version)
 {
 	if(version >= 11)
 	{
-		compressZlib(data, os);
+		compressZlib(*data ,data.getSize(), os);
 		return;
 	}
 
 	if(data.getSize() == 0)
 		return;
-	
+
 	// Write length (u32)
 
 	u8 tmp[4];
 	writeU32(tmp, data.getSize());
 	os.write((char*)tmp, 4);
-	
+
 	// We will be writing 8-bit pairs of more_count and byte
 	u8 more_count = 0;
 	u8 current_byte = data[0];
@@ -242,7 +241,7 @@ void decompress(std::istream &is, std::ostream &os, u8 version)
 	u8 tmp[4];
 	is.read((char*)tmp, 4);
 	u32 len = readU32(tmp);
-	
+
 	// We will be reading 8-bit pairs of more_count and byte
 	u32 count = 0;
 	for(;;)
@@ -251,7 +250,7 @@ void decompress(std::istream &is, std::ostream &os, u8 version)
 		u8 byte=0;
 
 		is.read((char*)&more_count, 1);
-		
+
 		is.read((char*)&byte, 1);
 
 		if(is.eof())

diff --git a/src/serialization.h b/src/serialization.h
@@ -86,12 +86,12 @@ inline bool ser_ver_supported(s32 v) {
 	Misc. serialization functions
 */
 
-void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level = -1);
+void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level = -1);
 void compressZlib(const std::string &data, std::ostream &os, int level = -1);
 void decompressZlib(std::istream &is, std::ostream &os);
 
 // These choose between zlib and a self-made one according to version
-void compress(SharedBuffer<u8> data, std::ostream &os, u8 version);
+void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version);
 //void compress(const std::string &data, std::ostream &os, u8 version);
 void decompress(std::istream &is, std::ostream &os, u8 version);