Improve accuracy and safety of float serialization

kwolekr · kwolekr · commit 42cf5e972d1e · 2015-08-01T19:30:08.000-04:00
Multiplying by a factor of 1/1000.f (rather than dividing by 1000.f) directly
introduces an error of 1 ULP.  With this patch, an exact comparison of a
floating point literal with the deserialized F1000 form representing it is now
guaranteed to be successful.
In addition, the maxmium and minimum safely representible floating point
numbers are now well-defined as constants.
diff --git a/src/unittest/test_serialization.cpp b/src/unittest/test_serialization.cpp
@@ -296,29 +296,22 @@ void TestSerialization::testStreamRead()
 	UASSERT(readS32(is) == -6);
 	UASSERT(readS64(is) == -43);
 
-	UASSERT(fabs(readF1000(is) - 53.534f) < 0.005);
-	UASSERT(fabs(readF1000(is) - -300000.32f) < 0.05);
-	UASSERT(fabs(readF1000(is) - -2147483.f) < 0.05);
-	UASSERT(fabs(readF1000(is) - 2147483.f) < 0.05);
+	UASSERT(readF1000(is) == 53.534f);
+	UASSERT(readF1000(is) == -300000.32f);
+	UASSERT(readF1000(is) == F1000_MIN);
+	UASSERT(readF1000(is) == F1000_MAX);
 
 	UASSERT(deSerializeString(is) == "foobar!");
 
 	UASSERT(readV2S16(is) == v2s16(500, 500));
 	UASSERT(readV3S16(is) == v3s16(4207, 604, -30));
 	UASSERT(readV2S32(is) == v2s32(1920, 1080));
 	UASSERT(readV3S32(is) == v3s32(-400, 6400054, 290549855));
-
-	v2f vec2 = readV2F1000(is);
-	UASSERT(fabs(vec2.X - 500.656f) < 0.005);
-	UASSERT(fabs(vec2.Y - 350.345f) < 0.005);
+	UASSERT(readV2F1000(is) == v2f(500.656f, 350.345f));
 
 	UASSERT(deSerializeWideString(is) == L"\x02~woof~\x5455");
 
-	v3f vec3 = readV3F1000(is);
-	UASSERT(fabs(vec3.X - 500.f) < 0.005);
-	UASSERT(fabs(vec3.Y - 10024.2f) < 0.005);
-	UASSERT(fabs(vec3.Z - -192.54f) < 0.005);
-
+	UASSERT(readV3F1000(is) == v3f(500, 10024.2f, -192.54f));
 	UASSERT(readARGB8(is) == video::SColor(255, 128, 50, 128));
 
 	UASSERT(deSerializeLongString(is) == "some longer string here");
@@ -346,8 +339,8 @@ void TestSerialization::testStreamWrite()
 
 	writeF1000(os, 53.53467f);
 	writeF1000(os, -300000.32f);
-	writeF1000(os, -2147483.f);
-	writeF1000(os, 2147483.f);
+	writeF1000(os, F1000_MIN);
+	writeF1000(os, F1000_MAX);
 
 	os << serializeString("foobar!");
 
diff --git a/src/util/serialize.h b/src/util/serialize.h
@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #define UTIL_SERIALIZE_HEADER
 
 #include "../irrlichttypes_bloated.h"
+#include "../debug.h" // for assert
 #include "config.h"
 #if HAVE_ENDIAN_H
 #include <endian.h>
@@ -30,7 +31,16 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <string>
 
 #define FIXEDPOINT_FACTOR 1000.0f
-#define FIXEDPOINT_INVFACTOR (1.0f/FIXEDPOINT_FACTOR)
+
+// 0x7FFFFFFF / 1000.0f is not serializable.
+// The limited float precision at this magnitude may cause the result to round
+// to a greater value than can be represented by a 32 bit integer when increased
+// by a factor of FIXEDPOINT_FACTOR.  As a result, [F1000_MIN..F1000_MAX] does
+// not represent the full range, but rather the largest safe range, of values on
+// all supported architectures.  Note: This definition makes assumptions on
+// platform float-to-int conversion behavior.
+#define F1000_MIN ((float)(s32)((-0x7FFFFFFF - 1) / FIXEDPOINT_FACTOR))
+#define F1000_MAX ((float)(s32)((0x7FFFFFFF) / FIXEDPOINT_FACTOR))
 
 #define STRING_MAX_LEN 0xFFFF
 #define WIDE_STRING_MAX_LEN 0xFFFF
@@ -163,7 +173,7 @@ inline s64 readS64(const u8 *data)
 
 inline f32 readF1000(const u8 *data)
 {
-	return (f32)readS32(data) * FIXEDPOINT_INVFACTOR;
+	return (f32)readS32(data) / FIXEDPOINT_FACTOR;
 }
 
 inline video::SColor readARGB8(const u8 *data)
@@ -252,6 +262,7 @@ inline void writeS64(u8 *data, s64 i)
 
 inline void writeF1000(u8 *data, f32 i)
 {
+	assert(i >= F1000_MIN && i <= F1000_MAX);
 	writeS32(data, i * FIXEDPOINT_FACTOR);
 }
 
