Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Implement AreaStore serialization
  • Loading branch information
ShadowNinja committed Mar 7, 2016
1 parent c4b7afe commit 821551a
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 92 deletions.
4 changes: 4 additions & 0 deletions doc/lua_api.txt
Expand Up @@ -2737,6 +2737,10 @@ If you chose the parameter-less constructor, a fast implementation will be autom
block_radius = number, -- the radius (in nodes) of the areas the cache generates prefiltered lists for, minimum 16, default 64
limit = number, -- the cache's size, minimum 20, default 1000
}
* `to_string()`: Experimental. Returns area store serialized as a (binary) string.
* `to_file(filename)`: Experimental. Like `to_string()`, but writes the data to a file.
* `from_string(str)`: Experimental. Deserializes string and loads it into the AreaStore. Returns success and, optionally, an error message.
* `from_file(filename)`: Experimental. Like `from_string()`, but reads the data from a file.

### `ItemStack`
An `ItemStack` is a stack of items.
Expand Down
37 changes: 21 additions & 16 deletions src/script/lua_api/l_areastore.cpp
Expand Up @@ -70,6 +70,22 @@ static inline void push_areas(lua_State *L, const std::vector<Area *> &areas,
}
}

// Deserializes value and handles errors
static int deserialization_helper(lua_State *L, AreaStore *as,
std::istream &is)
{
try {
as->deserialize(is);
} catch (const SerializationError &e) {
lua_pushboolean(L, false);
lua_pushstring(L, e.what());
return 2;
}

lua_pushboolean(L, true);
return 1;
}

// garbage collector
int LuaAreaStore::gc_object(lua_State *L)
{
Expand Down Expand Up @@ -217,17 +233,15 @@ int LuaAreaStore::l_set_cache_params(lua_State *L)
return 0;
}

#if 0
// to_string()
int LuaAreaStore::l_to_string(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;

LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;

std::ostringstream os(std::ios_base::binary);
ast->serialize(os);
o->as->serialize(os);
std::string str = os.str();

lua_pushlstring(L, str.c_str(), str.length());
Expand Down Expand Up @@ -258,16 +272,12 @@ int LuaAreaStore::l_from_string(lua_State *L)
NO_MAP_LOCK_REQUIRED;

LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;

size_t len;
const char *str = luaL_checklstring(L, 2, &len);

std::istringstream is(std::string(str, len), std::ios::binary);
bool success = ast->deserialize(is);

lua_pushboolean(L, success);
return 1;
return deserialization_helper(L, o->as, is);
}

// from_file(filename)
Expand All @@ -276,18 +286,13 @@ int LuaAreaStore::l_from_file(lua_State *L)
NO_MAP_LOCK_REQUIRED;

LuaAreaStore *o = checkobject(L, 1);
AreaStore *ast = o->as;

const char *filename = luaL_checkstring(L, 2);
CHECK_SECURE_PATH_OPTIONAL(L, filename);

std::ifstream is(filename, std::ios::binary);
bool success = ast->deserialize(is);

lua_pushboolean(L, success);
return 1;
return deserialization_helper(L, o->as, is);
}
#endif

LuaAreaStore::LuaAreaStore()
{
Expand Down Expand Up @@ -377,9 +382,9 @@ const luaL_reg LuaAreaStore::methods[] = {
luamethod(LuaAreaStore, reserve),
luamethod(LuaAreaStore, remove_area),
luamethod(LuaAreaStore, set_cache_params),
/* luamethod(LuaAreaStore, to_string),
luamethod(LuaAreaStore, to_string),
luamethod(LuaAreaStore, to_file),
luamethod(LuaAreaStore, from_string),
luamethod(LuaAreaStore, from_file),*/
luamethod(LuaAreaStore, from_file),
{0,0}
};
4 changes: 2 additions & 2 deletions src/script/lua_api/l_areastore.h
Expand Up @@ -43,11 +43,11 @@ class LuaAreaStore : public ModApiBase {

static int l_set_cache_params(lua_State *L);

/* static int l_to_string(lua_State *L);
static int l_to_string(lua_State *L);
static int l_to_file(lua_State *L);

static int l_from_string(lua_State *L);
static int l_from_file(lua_State *L); */
static int l_from_file(lua_State *L);

public:
AreaStore *as;
Expand Down
39 changes: 39 additions & 0 deletions src/unittest/test_areastore.cpp
Expand Up @@ -31,6 +31,7 @@ class TestAreaStore : public TestBase {
void genericStoreTest(AreaStore *store);
void testVectorStore();
void testSpatialStore();
void testSerialization();
};

static TestAreaStore g_test_instance;
Expand All @@ -41,6 +42,7 @@ void TestAreaStore::runTests(IGameDef *gamedef)
#if USE_SPATIAL
TEST(testSpatialStore);
#endif
TEST(testSerialization);
}

////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -121,3 +123,40 @@ void TestAreaStore::genericStoreTest(AreaStore *store)
store->removeArea(d.id);
}

void TestAreaStore::testSerialization()
{
VectorAreaStore store;

Area a(v3s16(-1, 0, 1), v3s16(0, 1, 2));
a.data = "Area A";
store.insertArea(&a);

Area b(v3s16(123, 456, 789), v3s16(32000, 100, 10));
b.data = "Area B";
store.insertArea(&b);

std::ostringstream os;
store.serialize(os);
std::string str = os.str();

std::string str_wanted("\x00" // Version
"\x00\x02" // Count
"\xFF\xFF\x00\x00\x00\x01" // Area A min edge
"\x00\x00\x00\x01\x00\x02" // Area A max edge
"\x00\x06" // Area A data length
"Area A" // Area A data
"\x00\x7B\x00\x64\x00\x0A" // Area B min edge (last two swapped with max edge for sorting)
"\x7D\x00\x01\xC8\x03\x15" // Area B max edge (^)
"\x00\x06" // Area B data length
"Area B", // Area B data
1 + 2 +
6 + 6 + 2 + 6 +
6 + 6 + 2 + 6);
UASSERTEQ(std::string, str, str_wanted);

std::istringstream is(str);
store.deserialize(is);

UASSERTEQ(size_t, store.size(), 4); // deserialize() doesn't clear the store
}

91 changes: 28 additions & 63 deletions src/util/areastore.cpp
Expand Up @@ -62,57 +62,42 @@ const Area *AreaStore::getArea(u32 id) const
return &it->second;
}

#if 0
Currently, serialisation is commented out. This is because of multiple reasons:
1. Why do we store the areastore into a file, why not into the database?
2. We don't use libspatial's serialisation, but we should, or perhaps not, because
it would remove the ability to switch. Perhaps write migration routines?
3. Various things need fixing, e.g. the size is serialized as
c++ implementation defined size_t
bool AreaStore::deserialize(std::istream &is)
void AreaStore::serialize(std::ostream &os) const
{
u8 ver = readU8(is);
if (ver != 1)
return false;
u16 count_areas = readU16(is);
for (u16 i = 0; i < count_areas; i++) {
// deserialize an area
Area a;
a.id = readU32(is);
a.minedge = readV3S16(is);
a.maxedge = readV3S16(is);
a.datalen = readU16(is);
a.data = new char[a.datalen];
is.read((char *) a.data, a.datalen);
insertArea(a);
writeU8(os, 0); // Serialisation version

// TODO: Compression?
writeU16(os, areas_map.size());
for (AreaMap::const_iterator it = areas_map.begin();
it != areas_map.end(); ++it) {
const Area &a = it->second;
writeV3S16(os, a.minedge);
writeV3S16(os, a.maxedge);
writeU16(os, a.data.size());
os.write(a.data.data(), a.data.size());
}
return true;
}


static bool serialize_area(void *ostr, Area *a)
void AreaStore::deserialize(std::istream &is)
{
std::ostream &os = *((std::ostream *) ostr);
writeU32(os, a->id);
writeV3S16(os, a->minedge);
writeV3S16(os, a->maxedge);
writeU16(os, a->datalen);
os.write(a->data, a->datalen);

return false;
}

u8 ver = readU8(is);
if (ver != 0)
throw SerializationError("Unknown AreaStore "
"serialization version!");

void AreaStore::serialize(std::ostream &os) const
{
// write initial data
writeU8(os, 1); // serialisation version
writeU16(os, areas_map.size()); //DANGER: not platform independent
forEach(&serialize_area, &os);
u16 num_areas = readU16(is);
for (u32 i = 0; i < num_areas; ++i) {
Area a;
a.minedge = readV3S16(is);
a.maxedge = readV3S16(is);
u16 data_len = readU16(is);
char *data = new char[data_len];
is.read(data, data_len);
a.data = std::string(data, data_len);
insertArea(&a);
}
}

#endif

void AreaStore::invalidateCache()
{
if (m_cache_enabled) {
Expand Down Expand Up @@ -226,18 +211,6 @@ void VectorAreaStore::getAreasInArea(std::vector<Area *> *result,
}
}

#if 0
bool SimpleAreaStore::forEach(ForEachCallback callback, void *arg) const
{
for (size_t i = 0; i < m_areas.size(); ++i) {
if (callback(m_areas[i], arg)) {
return true;
}
}
return false;
}
#endif

#if USE_SPATIAL

static inline SpatialIndex::Region get_spatial_region(const v3s16 minedge,
Expand Down Expand Up @@ -301,14 +274,6 @@ void SpatialAreaStore::getAreasInArea(std::vector<Area *> *result,
}
}

#if 0
bool SpatialAreaStore::forEach(ForEachCallback callback, void *arg) const
{
// TODO ?? (this is only needed for serialisation, but libspatial has its own serialisation)
return false;
}
#endif

SpatialAreaStore::~SpatialAreaStore()
{
delete m_tree;
Expand Down
18 changes: 7 additions & 11 deletions src/util/areastore.h
Expand Up @@ -92,16 +92,14 @@ class AreaStore {
/// or NULL if it doesn't exist.
const Area *getArea(u32 id) const;

#if 0
typedef bool (*ForEachCallback)(const Area *a, void *arg);
/// Calls a passed function for every stored area, until the
/// callback returns true. If that happens, it returns true,
/// if the search is exhausted, it returns false.
virtual bool forEach(ForEachCallback, void *arg=NULL) const = 0;

/// Serializes the store's areas to a binary ostream.
void serialize(std::ostream &is) const;
bool deserialize(std::istream &is);
#endif

/// Deserializes the Areas from a binary istream.
/// This does not currently clear the AreaStore before adding the
/// areas, making it possible to deserialize multiple serialized
/// AreaStores.
void deserialize(std::istream &is);

protected:
/// Invalidates the getAreasForPos cache.
Expand Down Expand Up @@ -141,7 +139,6 @@ class VectorAreaStore : public AreaStore {
virtual bool removeArea(u32 id);
virtual void getAreasInArea(std::vector<Area *> *result,
v3s16 minedge, v3s16 maxedge, bool accept_overlap);
//virtual bool forEach(ForEachCallback, void *arg) const;

protected:
virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);
Expand All @@ -162,7 +159,6 @@ class SpatialAreaStore : public AreaStore {
virtual bool removeArea(u32 id);
virtual void getAreasInArea(std::vector<Area *> *result,
v3s16 minedge, v3s16 maxedge, bool accept_overlap);
//virtual bool forEach(ForEachCallback, void *arg) const;

protected:
virtual void getAreasForPosImpl(std::vector<Area *> *result, v3s16 pos);
Expand Down

0 comments on commit 821551a

Please sign in to comment.