Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: mockingbirdnest/Principia
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 82fd37437123
Choose a base ref
...
head repository: mockingbirdnest/Principia
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 51190d84c05f
Choose a head ref
  • 11 commits
  • 7 files changed
  • 1 contributor

Commits on Mar 14, 2020

  1. Templatized N-dimensional zfp.

    pleroy committed Mar 14, 2020
    Copy the full SHA
    8af3131 View commit details
  2. Proper 2D support.

    pleroy committed Mar 14, 2020
    Copy the full SHA
    dbe848a View commit details
  3. Test passing.

    pleroy committed Mar 14, 2020
    Copy the full SHA
    d1dc3e5 View commit details

Commits on Mar 15, 2020

  1. 1, 3 and 4 dimensions.

    pleroy committed Mar 15, 2020
    Copy the full SHA
    a95b16e View commit details
  2. Copy the full SHA
    fcc9b2e View commit details
  3. Renaming and tolerances.

    pleroy committed Mar 15, 2020
    Copy the full SHA
    754e172 View commit details
  4. Bring 'em back alive.

    pleroy committed Mar 15, 2020
    Copy the full SHA
    475edbd View commit details
  5. Copy the full SHA
    5efcd25 View commit details
  6. Lint.

    pleroy committed Mar 15, 2020
    Copy the full SHA
    6844fb4 View commit details

Commits on Mar 16, 2020

  1. After egg's review.

    pleroy committed Mar 16, 2020
    Copy the full SHA
    405fe79 View commit details
  2. Merge pull request #2496 from pleroy/HiDim

    Add support for D-dimensional zfp compression
    pleroy authored Mar 16, 2020
    Copy the full SHA
    51190d8 View commit details
55 changes: 8 additions & 47 deletions base/zfp_compressor.cpp
Original file line number Diff line number Diff line change
@@ -11,52 +11,13 @@ namespace principia {
namespace base {
namespace zfp_compressor_internal {

ZfpCompressor::ZfpCompressor(double const accuracy) : accuracy_(accuracy) {}

void ZfpCompressor::WriteToMessage2D(
std::vector<double>& v,
not_null<std::string*> const message) const {
if (v.empty()) {
return;
}

// Round up the size of the vector to a multiple of the block size. This will
// lead to poor compression at the end, but there is no support for "ignored"
// data in zfp at this point.
v.resize(((v.size() + block_ - 1) / block_) * block_, 0);
CHECK_EQ(0, v.size() % block_);

// Beware nx and ny! (And the Jabberwock, my son!)
// See https://zfp.readthedocs.io/en/release0.5.5/tutorial.html#high-level-c-interface
std::unique_ptr<zfp_field, std::function<void(zfp_field*)>> const field(
zfp_field_2d(v.data(),
/*type=*/zfp_type_double,
/*nx=*/block_,
/*ny=*/v.size() / block_),
[](zfp_field* const field) { zfp_field_free(field); });
WriteToMessage(field.get(), message);
}
// ZFP headers limit the dimensions to 2^(48 / N). For N = 4, this is way too
// small for our purposes. Therefore, we must not include the bit
// ZFP_HEADER_META. We still need a header, though, to record the compression
// parameters.
static constexpr uint header_mask = ZFP_HEADER_MODE | ZFP_HEADER_MAGIC;

void ZfpCompressor::ReadFromMessage2D(std::vector<double>& v,
std::string_view& message) const {
if (v.empty()) {
return;
}

// Make sure that we have enough space in the vector to decompress the
// padding.
v.resize(((v.size() + block_ - 1) / block_) * block_, 0);
CHECK_EQ(0, v.size() % block_);

std::unique_ptr<zfp_field, std::function<void(zfp_field*)>> const field(
zfp_field_2d(v.data(),
/*type=*/zfp_type_double,
/*nx=*/block_,
/*ny=*/v.size() / block_),
[](zfp_field* const field) { zfp_field_free(field); });

ReadFromMessage(field.get(), message);
}
ZfpCompressor::ZfpCompressor(double const accuracy) : accuracy_(accuracy) {}

void ZfpCompressor::WriteToMessage(const zfp_field* const field,
not_null<std::string*> message) const {
@@ -76,7 +37,7 @@ void ZfpCompressor::WriteToMessage(const zfp_field* const field,
check_not_null(stream_open(buffer.data.get(), buffer_size));
zfp_stream_set_bit_stream(zfp.get(), &*stream);

zfp_write_header(zfp.get(), field, ZFP_HEADER_FULL);
zfp_write_header(zfp.get(), field, header_mask);
size_t const compressed_size = zfp_compress(zfp.get(), field);
CHECK_LT(0, compressed_size);
message->append(static_cast<char const*>(stream_data(stream)),
@@ -92,7 +53,7 @@ void ZfpCompressor::ReadFromMessage(zfp_field* const field,
not_null<bitstream*> const stream = check_not_null(
stream_open(const_cast<char*>(&message.front()), message.size()));
zfp_stream_set_bit_stream(zfp.get(), &*stream);
size_t const header_bits = zfp_read_header(zfp.get(), field, ZFP_HEADER_FULL);
size_t const header_bits = zfp_read_header(zfp.get(), field, header_mask);
CHECK_LT(0, header_bits);

size_t const compressed_size = zfp_decompress(zfp.get(), field);
22 changes: 11 additions & 11 deletions base/zfp_compressor.hpp
Original file line number Diff line number Diff line change
@@ -33,15 +33,17 @@ class ZfpCompressor {
template<typename Message>
static void ReadVersion(Message const& message);

// Serialization/deserialization of a vector of double into a message using a
// 2D encoding. This encodes blocks of 16 doubles and therefore takes
// advantage of any correlation across these doubles. The vector |v| may be
// modified by padding it with zeroes. When reading, the |message| parameter
// is updated to reflect the data that was consumed.
void WriteToMessage2D(std::vector<double>& v,
not_null<std::string*> message) const;
void ReadFromMessage2D(std::vector<double>& v,
std::string_view& message) const;
// Serialization/deserialization of a vector of double into a message using an
// D-dimensional encoding. This encodes blocks of 4^D doubles and therefore
// takes advantage of any correlation across these doubles. The vector |v|
// may be modified by padding it with zeroes. When reading, the |message|
// parameter is updated to reflect the data that was consumed.
template<int D>
void WriteToMessageMultidimensional(std::vector<double>& v,
not_null<std::string*> message) const;
template<int D>
void ReadFromMessageMultidimensional(std::vector<double>& v,
std::string_view& message) const;

// Low-level API: serialization/deserialization of a field (allocated and
// owned by the caller) into a message (which is expected to by a bytes field
@@ -53,8 +55,6 @@ class ZfpCompressor {
std::string_view& message) const;

private:
static constexpr int block_ = 4;

std::optional<double> const accuracy_;
};

98 changes: 98 additions & 0 deletions base/zfp_compressor_body.hpp
Original file line number Diff line number Diff line change
@@ -2,10 +2,82 @@

#include "base/zfp_compressor.hpp"

#include <functional>
#include <vector>
#include <string>

namespace principia {
namespace base {
namespace zfp_compressor_internal {

template<int D>
class NDimensionalHelper {
public:
using unique_zfp_field =
std::unique_ptr<zfp_field, std::function<void(zfp_field*)>>;

static unique_zfp_field NewField(zfp_type type, std::vector<double>& v);

private:
// Returns the size rounded up to a multiple of 4^(D - 1).
static constexpr std::int64_t RoundUp(std::int64_t size);

// The ZFP block size.
static constexpr int block_ = 4;
// Data must be padded to a multiple of that value, which is 4^(D - 1).
static constexpr int padding_ = 1 << (2 * (D - 1));
};

template<int D>
typename NDimensionalHelper<D>::unique_zfp_field
NDimensionalHelper<D>::NewField(zfp_type const type, std::vector<double>& v) {
auto free = [](zfp_field* const field) { zfp_field_free(field); };

// On compression: Round up the size of the vector to a multiple of the block
// size. This will lead to poor compression at the end, but there is no
// support for "ignored" data in zfp at this point.
// On decompression: Make sure that we have enough space in the vector to
// decompress the padding.
v.resize(RoundUp(v.size()), 0);
CHECK_EQ(0, v.size() % padding_);

// Beware nx, ny and friends! (And the Jabberwock, my son!)
// See
// https://zfp.readthedocs.io/en/release0.5.5/tutorial.html#high-level-c-interface
if constexpr (D == 1) {
return unique_zfp_field(
zfp_field_1d(v.data(), type, /*nx=*/v.size() / padding_),
std::move(free));
} else if constexpr (D == 2) {
return unique_zfp_field(
zfp_field_2d(v.data(), type, /*nx=*/block_, /*ny=*/v.size() / padding_),
std::move(free));
} else if constexpr (D == 3) {
return unique_zfp_field(zfp_field_3d(v.data(),
type,
/*nx=*/block_,
/*ny=*/block_,
/*nz=*/v.size() / padding_),
std::move(free));
} else if constexpr (D == 4) {
return unique_zfp_field(zfp_field_4d(v.data(),
type,
/*nx=*/block_,
/*ny=*/block_,
/*nz=*/block_,
/*nw=*/v.size() / padding_),
std::move(free));
} else {
static_assert(false, "Unsupported dimension");
}
}

template<int D>
constexpr std::int64_t NDimensionalHelper<D>::RoundUp(std::int64_t const size) {
// [War03], section 3-1.
return (size + (padding_ - 1)) & (-padding_);
}

template<typename Message>
void ZfpCompressor::WriteVersion(not_null<Message*> message) {
// For future compatibility, record the version of ZFP used to compress.
@@ -21,6 +93,32 @@ void ZfpCompressor::ReadVersion(Message const& message) {
CHECK_EQ(ZFP_VERSION, message.zfp().library_version());
}

template<int D>
void ZfpCompressor::WriteToMessageMultidimensional(
std::vector<double>& v,
not_null<std::string*> const message) const {
if (v.empty()) {
return;
}

auto const field =
NDimensionalHelper<D>::NewField(/*type=*/zfp_type_double, v);
WriteToMessage(field.get(), message);
}

template<int D>
void ZfpCompressor::ReadFromMessageMultidimensional(
std::vector<double>& v,
std::string_view& message) const {
if (v.empty()) {
return;
}

auto const field =
NDimensionalHelper<D>::NewField(/*type=*/zfp_type_double, v);
ReadFromMessage(field.get(), message);
}

} // namespace zfp_compressor_internal

using zfp_compressor_internal::ZfpCompressor;
8 changes: 8 additions & 0 deletions documentation/bibliography.bib
Original file line number Diff line number Diff line change
@@ -718,6 +718,14 @@ @book{NistHMF2010
title = {NIST Handbook of Mathematical Functions},
}

@book{Warren2003,
author = {Warren, Jr., Henry S.},
publisher = {Addison-Wesley},
date = {2003},
isbn = {0-201-91465-4},
title = {Hacker's Delight},
}

@incollection{BlanesCasasRos2001a,
author = {Blanes, S. and Casas, F. and Ros, J.},
editor = {Vulkov, Lubin and Yalamov, Plamen and Waśniewski, Jerzy},
Binary file modified documentation/bibliography.pdf
Binary file not shown.
28 changes: 14 additions & 14 deletions physics/discrete_trajectory_body.hpp
Original file line number Diff line number Diff line change
@@ -568,13 +568,13 @@ void DiscreteTrajectory<Frame>::WriteSubTreeToMessage(
(Metre / Second));

ZfpCompressor::WriteVersion(message);
time_compressor.WriteToMessage2D(t, zfp_timeline);
length_compressor.WriteToMessage2D(qx, zfp_timeline);
length_compressor.WriteToMessage2D(qy, zfp_timeline);
length_compressor.WriteToMessage2D(qz, zfp_timeline);
speed_compressor.WriteToMessage2D(px, zfp_timeline);
speed_compressor.WriteToMessage2D(py, zfp_timeline);
speed_compressor.WriteToMessage2D(pz, zfp_timeline);
time_compressor.WriteToMessageMultidimensional<2>(t, zfp_timeline);
length_compressor.WriteToMessageMultidimensional<2>(qx, zfp_timeline);
length_compressor.WriteToMessageMultidimensional<2>(qy, zfp_timeline);
length_compressor.WriteToMessageMultidimensional<2>(qz, zfp_timeline);
speed_compressor.WriteToMessageMultidimensional<2>(px, zfp_timeline);
speed_compressor.WriteToMessageMultidimensional<2>(py, zfp_timeline);
speed_compressor.WriteToMessageMultidimensional<2>(pz, zfp_timeline);
}

if (downsampling_.has_value()) {
@@ -612,13 +612,13 @@ void DiscreteTrajectory<Frame>::FillSubTreeFromMessage(

ZfpCompressor decompressor;
ZfpCompressor::ReadVersion(message);
decompressor.ReadFromMessage2D(t, zfp_timeline);
decompressor.ReadFromMessage2D(qx, zfp_timeline);
decompressor.ReadFromMessage2D(qy, zfp_timeline);
decompressor.ReadFromMessage2D(qz, zfp_timeline);
decompressor.ReadFromMessage2D(px, zfp_timeline);
decompressor.ReadFromMessage2D(py, zfp_timeline);
decompressor.ReadFromMessage2D(pz, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(t, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(qx, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(qy, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(qz, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(px, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(py, zfp_timeline);
decompressor.ReadFromMessageMultidimensional<2>(pz, zfp_timeline);

for (int i = 0; i < timeline_size; ++i) {
Position<Frame> const q =
6 changes: 3 additions & 3 deletions physics/ephemeris_test.cpp
Original file line number Diff line number Diff line change
@@ -1111,16 +1111,16 @@ TEST(EphemerisTestNoFixture, DiscreteTrajectoryCompression) {
trajectory1.WriteToMessage(&message, /*forks=*/{});
std::string uncompressed;
message.SerializePartialToString(&uncompressed);
EXPECT_EQ(18'433, uncompressed.size());
EXPECT_EQ(18'377, uncompressed.size());

std::string compressed;
auto compressor = google::compression::NewGipfeliCompressor();
compressor->Compress(uncompressed, &compressed);

// We want a change detector, but the actual compressed size varies depending
// on the exact numerical values, and therefore on the mathematical library.
EXPECT_LE(16'974, compressed.size());
EXPECT_GE(16'974, compressed.size());
EXPECT_LE(16'935, compressed.size());
EXPECT_GE(16'935, compressed.size());

auto const trajectory2 =
DiscreteTrajectory<ICRS>::ReadFromMessage(message, /*forks=*/{});