Skip to content

Commit

Permalink
Optimize database access further by allowing "brute-force" queries in…
Browse files Browse the repository at this point in the history 
…stead of listing available blocks

Also adds a heuristic that will enable this behaviour automatically.
  • Loading branch information
@sfan5
sfan5 committed Mar 27, 2020
1 parent 5b264fd commit 7ff2288
Show file tree
Hide file tree
Showing 14 changed files with 318 additions and 90 deletions.
4 changes: 4 additions & 0 deletions README.rst
View file
Expand Up @@ -106,3 +106,7 @@ colors:

scales:
Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``

exhaustive:
Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.
193 changes: 133 additions & 60 deletions TileGenerator.cpp
View file
Expand Up @@ -96,6 +96,7 @@ TileGenerator::TileGenerator():
m_geomY(-2048),
m_geomX2(2048),
m_geomY2(2048),
m_exhaustiveSearch(EXH_AUTO),
m_zoom(1),
m_scales(SCALE_LEFT | SCALE_TOP)
{
Expand Down Expand Up @@ -206,6 +207,11 @@ void TileGenerator::setMaxY(int y)
std::swap(m_yMin, m_yMax);
}

void TileGenerator::setExhaustiveSearch(int mode)
{
m_exhaustiveSearch = mode;
}

void TileGenerator::parseColorsFile(const std::string &fileName)
{
ifstream in;
Expand All @@ -222,6 +228,7 @@ void TileGenerator::printGeometry(const std::string &input)
input_path += PATH_SEPARATOR;
}

setExhaustiveSearch(EXH_NEVER);
openDb(input_path);
loadBlocks();

Expand All @@ -247,6 +254,8 @@ void TileGenerator::generate(const std::string &input, const std::string &output
input_path += PATH_SEPARATOR;
}

if (m_dontWriteEmpty) // FIXME: possible too, just needs to be done differently
setExhaustiveSearch(EXH_NEVER);
openDb(input_path);
loadBlocks();

Expand Down Expand Up @@ -329,6 +338,35 @@ void TileGenerator::openDb(const std::string &input)
#endif
else
throw std::runtime_error(((std::string) "Unknown map backend: ") + backend);

// Determine how we're going to traverse the database (heuristic)
if (m_exhaustiveSearch == EXH_AUTO) {
using u64 = uint64_t;
u64 y_range = (m_yMax / 16 + 1) - (m_yMin / 16);
u64 blocks = (u64)(m_geomX2 - m_geomX) * y_range * (u64)(m_geomY2 - m_geomY);
#ifndef NDEBUG
std::cout << "Heuristic parameters:"
<< " preferRangeQueries()=" << m_db->preferRangeQueries()
<< " y_range=" << y_range << " blocks=" << blocks << std::endl;
#endif
if (m_db->preferRangeQueries())
m_exhaustiveSearch = EXH_NEVER;
else if (blocks < 200000)
m_exhaustiveSearch = EXH_FULL;
else if (y_range < 600)
m_exhaustiveSearch = EXH_Y;
else
m_exhaustiveSearch = EXH_NEVER;
} else if (m_exhaustiveSearch == EXH_FULL || m_exhaustiveSearch == EXH_Y) {
if (m_db->preferRangeQueries()) {
std::cerr << "Note: The current database backend supports efficient "
"range queries, forcing exhaustive search should always result "
" in worse performance." << std::endl;
}
}
if (m_exhaustiveSearch == EXH_Y)
m_exhaustiveSearch = EXH_NEVER; // (TODO remove when implemented)
assert(m_exhaustiveSearch != EXH_AUTO);
}

void TileGenerator::closeDatabase()
Expand All @@ -340,37 +378,40 @@ void TileGenerator::closeDatabase()
void TileGenerator::loadBlocks()
{
const int16_t yMax = m_yMax / 16 + 1;
std::vector<BlockPos> vec = m_db->getBlockPos(
BlockPos(m_geomX, m_yMin / 16, m_geomY),
BlockPos(m_geomX2, yMax, m_geomY2)
);

for (auto pos : vec) {
assert(pos.x >= m_geomX && pos.x < m_geomX2);
assert(pos.y >= m_yMin / 16 && pos.y < yMax);
assert(pos.z >= m_geomY && pos.z < m_geomY2);

// Adjust minimum and maximum positions to the nearest block
if (pos.x < m_xMin)
m_xMin = pos.x;
if (pos.x > m_xMax)
m_xMax = pos.x;

if (pos.z < m_zMin)
m_zMin = pos.z;
if (pos.z > m_zMax)
m_zMax = pos.z;

m_positions[pos.z].emplace(pos.x);
}

if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
std::vector<BlockPos> vec = m_db->getBlockPos(
BlockPos(m_geomX, m_yMin / 16, m_geomY),
BlockPos(m_geomX2, yMax, m_geomY2)
);

for (auto pos : vec) {
assert(pos.x >= m_geomX && pos.x < m_geomX2);
assert(pos.y >= m_yMin / 16 && pos.y < yMax);
assert(pos.z >= m_geomY && pos.z < m_geomY2);

// Adjust minimum and maximum positions to the nearest block
if (pos.x < m_xMin)
m_xMin = pos.x;
if (pos.x > m_xMax)
m_xMax = pos.x;

if (pos.z < m_zMin)
m_zMin = pos.z;
if (pos.z > m_zMax)
m_zMax = pos.z;

m_positions[pos.z].emplace(pos.x);
}

#ifndef NDEBUG
int count = 0;
for (const auto &it : m_positions)
count += it.second.size();
std::cout << "Loaded " << count
<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
int count = 0;
for (const auto &it : m_positions)
count += it.second.size();
std::cout << "Loaded " << count
<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
#endif
}
}

void TileGenerator::createImage()
Expand Down Expand Up @@ -422,44 +463,76 @@ void TileGenerator::renderMap()
BlockDecoder blk;
const int16_t yMax = m_yMax / 16 + 1;

for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
int16_t zPos = it->first;
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
int16_t xPos = *it2;

m_readPixels.reset();
m_readInfo.reset();
for (int i = 0; i < 16; i++) {
for (int j = 0; j < 16; j++) {
m_color[i][j] = m_bgColor; // This will be drawn by renderMapBlockBottom() for y-rows with only 'air', 'ignore' or unknown nodes if --drawalpha is used
m_color[i][j].a = 0; // ..but set alpha to 0 to tell renderMapBlock() not to use this color to mix a shade
m_thickness[i][j] = 0;
}
auto renderSingle = [&] (int16_t xPos, int16_t zPos, BlockList &blockStack) {
m_readPixels.reset();
m_readInfo.reset();
for (int i = 0; i < 16; i++) {
for (int j = 0; j < 16; j++) {
m_color[i][j] = m_bgColor; // This will be drawn by renderMapBlockBottom() for y-rows with only 'air', 'ignore' or unknown nodes if --drawalpha is used
m_color[i][j].a = 0; // ..but set alpha to 0 to tell renderMapBlock() not to use this color to mix a shade
m_thickness[i][j] = 0;
}
}

BlockList blockStack;
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
blockStack.sort();
for (const auto &it : blockStack) {
const BlockPos pos = it.first;
assert(pos.x == xPos && pos.z == zPos);
assert(pos.y >= m_yMin / 16 && pos.y < yMax);

blk.reset();
blk.decode(it.second);
if (blk.isEmpty())
continue;
renderMapBlock(blk, pos);
for (const auto &it : blockStack) {
const BlockPos pos = it.first;
assert(pos.x == xPos && pos.z == zPos);
assert(pos.y >= m_yMin / 16 && pos.y < yMax);

// Exit out if all pixels for this MapBlock are covered
if (m_readPixels.full())
break;
}
if (!m_readPixels.full())
renderMapBlockBottom(blockStack.begin()->first);
blk.reset();
blk.decode(it.second);
if (blk.isEmpty())
continue;
renderMapBlock(blk, pos);

// Exit out if all pixels for this MapBlock are covered
if (m_readPixels.full())
break;
}
if (!m_readPixels.full())
renderMapBlockBottom(blockStack.begin()->first);
};
auto postRenderRow = [&] (int16_t zPos) {
if (m_shading)
renderShading(zPos);
};

if (m_exhaustiveSearch == EXH_NEVER) {
for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
int16_t zPos = it->first;
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
int16_t xPos = *it2;

BlockList blockStack;
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
blockStack.sort();

renderSingle(xPos, zPos, blockStack);
}
postRenderRow(zPos);
}
} else if (m_exhaustiveSearch == EXH_FULL) {
#ifndef NDEBUG
std::cerr << "Exhaustively searching "
<< (m_geomX2 - m_geomX) << "x" << (yMax - (m_yMin / 16)) << "x"
<< (m_geomY2 - m_geomY) << " blocks" << std::endl;
#endif
std::vector<BlockPos> positions;
positions.reserve(yMax - (m_yMin / 16));
for (int16_t zPos = m_geomY2 - 1; zPos >= m_geomY; zPos--) {
for (int16_t xPos = m_geomX2 - 1; xPos >= m_geomX; xPos--) {
positions.clear();
for (int16_t yPos = m_yMin / 16; yPos < yMax; yPos++)
positions.emplace_back(xPos, yPos, zPos);

BlockList blockStack;
m_db->getBlocksByPos(blockStack, positions);
blockStack.sort();

renderSingle(xPos, zPos, blockStack);
}
postRenderRow(zPos);
}
}
}

Expand Down
16 changes: 16 additions & 0 deletions db-leveldb.cpp
View file
Expand Up @@ -96,3 +96,19 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
}
}
}

void DBLevelDB::getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions)
{
std::string datastr;
leveldb::Status status;

for (auto pos : positions) {
status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
if (status.ok()) {
blocks.emplace_back(
pos, ustring((unsigned char *) datastr.data(), datastr.size())
);
}
}
}
45 changes: 44 additions & 1 deletion db-postgresql.cpp
View file
Expand Up @@ -38,6 +38,11 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
" posX = $1::int4 AND posZ = $2::int4"
" AND (posY BETWEEN $3::int4 AND $4::int4)"
);
prepareStatement(
"get_block_exact",
"SELECT data FROM blocks WHERE"
" posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4"
);

checkResults(PQexec(db, "START TRANSACTION;"));
checkResults(PQexec(db, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"));
Expand All @@ -48,12 +53,13 @@ DBPostgreSQL::~DBPostgreSQL()
{
try {
checkResults(PQexec(db, "COMMIT;"));
} catch (std::exception& caught) {
} catch (const std::exception& caught) {
std::cerr << "could not finalize: " << caught.what() << std::endl;
}
PQfinish(db);
}


std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
{
int32_t const x1 = htonl(min.x);
Expand Down Expand Up @@ -124,6 +130,43 @@ void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
PQclear(results);
}


void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions)
{
int32_t x, y, z;

const void *args[] = { &x, &y, &z };
const int argLen[] = { 4, 4, 4 };
const int argFmt[] = { 1, 1, 1 };

for (auto pos : positions) {
x = htonl(pos.x);
y = htonl(pos.y);
z = htonl(pos.z);

PGresult *results = execPrepared(
"get_block_exact", ARRLEN(args), args,
argLen, argFmt, false
);

if (PQntuples(results) > 0) {
blocks.emplace_back(
pos,
ustring(
reinterpret_cast<unsigned char*>(
PQgetvalue(results, 0, 0)
),
PQgetlength(results, 0, 0)
)
);
}

PQclear(results);
}
}


PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
{
ExecStatusType statusType = PQresultStatus(res);
Expand Down

0 comments on commit 7ff2288

Please sign in to comment.