NixOS · Mar 10, 2021 · Apr 9, 2021 · Apr 13, 2021 · Apr 15, 2021 · Apr 15, 2021
diff --git a/Makefile.config.in b/Makefile.config.in
@@ -15,7 +15,6 @@ LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
-LIBLZMA_LIBS = @LIBLZMA_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@

diff --git a/configure.ac b/configure.ac
@@ -172,11 +172,6 @@ fi
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
-# Look for libbz2, a required dependency.
-AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
-AC_CHECK_HEADERS([bzlib.h], [true],
-  [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://sourceware.org/bzip2/.])])
 # Checks for libarchive
 PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
@@ -205,16 +200,6 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
 # Look for libsodium, an optional dependency.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
-# Look for liblzma, a required dependency.
-PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
-AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
-  [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
-
-# Look for zlib, a required dependency.
-PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
-AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
-LDFLAGS="-lz $LDFLAGS"
-
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
@@ -300,17 +285,6 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
-# Do we have GNU tar?
-AC_MSG_CHECKING([if you have a recent GNU tar])
-if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then
-    AC_MSG_RESULT(yes)
-    tarFlags="--warning=no-timestamp"
-else
-    AC_MSG_RESULT(no)
-fi
-AC_SUBST(tarFlags)
-
-
 AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   [path of a statically-linked shell to use as /bin/sh in sandboxes]),
   sandbox_shell=$withval)

diff --git a/flake.nix b/flake.nix
@@ -82,7 +82,7 @@
 
         buildDeps =
           [ curl
-            bzip2 xz brotli zlib editline
+            bzip2 xz brotli editline
             openssl sqlite
             libarchive
             boost

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
@@ -179,6 +179,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
+           compression == "zstd" ? ".zst" :
+           compression == "lzip" ? ".lzip" :
+           compression == "lz4" ? ".lz4" :
            compression == "br" ? ".br" :
            "");
 

diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
@@ -148,7 +148,7 @@ struct curlFileTransfer : public FileTransfer
         }
 
         LambdaSink finalSink;
-        std::shared_ptr<CompressionSink> decompressionSink;
+        std::shared_ptr<FinishSink> decompressionSink;
         std::optional<StringSink> errorSink;
 
         std::exception_ptr writeException;

diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
@@ -8,14 +8,18 @@
 
 namespace nix {
 
-struct CompressionSink : BufferedSink
+struct CompressionSink : BufferedSink, FinishSink
 {
-    virtual void finish() = 0;
+    using BufferedSink::operator ();
+    using BufferedSink::write;
+    using FinishSink::finish;
 };
 
+std::unique_ptr<Source> makeDecompressionSource(Source & prev);
+
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
 
 ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
 

diff --git a/src/libutil/local.mk b/src/libutil/local.mk
@@ -6,7 +6,7 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS = -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	libutil_LDFLAGS += -lcpuid

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
@@ -201,6 +201,61 @@ static DefaultStackAllocator defaultAllocatorSingleton;
 StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
 
 
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
+{
+    struct SourceToSink : FinishSink
+    {
+        typedef boost::coroutines2::coroutine<bool> coro_t;
+
+        std::function<void(Source &)> fun;
+        std::optional<coro_t::push_type> coro;
+
+        SourceToSink(std::function<void(Source &)> fun) : fun(fun)
+        {
+        }
+
+        std::string_view cur;
+
+        void operator () (std::string_view in) override
+        {
+            if (in.empty()) return;
+            cur = in;
+
+            if (!coro)
+                coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
+                    LambdaSource source([&](char *out, size_t out_len) {
+                        if (cur.empty()) {
+                            yield();
+                            if (yield.get()) {
+                                return (size_t)0;
+                            }
+                        }
+
+                        size_t n = std::min(cur.size(), out_len);
+                        memcpy(out, cur.data(), n);
+                        cur.remove_prefix(n);
+                        return n;
+                    });
+                    fun(source);
+                });
+
+            if (!*coro) { abort(); }
+
+            if (!cur.empty()) (*coro)(false);
+        }
+
+        void finish() {
+            if (!coro) return;
+            if (!*coro) abort();
+            (*coro)(true);
+            if (*coro) abort();
+        }
+    };
+
+    return std::make_unique<SourceToSink>(fun);
+}
+
+
 std::unique_ptr<Source> sinkToSource(
     std::function<void(Sink &)> fun,
     std::function<void()> eof)
@@ -212,7 +267,6 @@ std::unique_ptr<Source> sinkToSource(
         std::function<void(Sink &)> fun;
         std::function<void()> eof;
         std::optional<coro_t::pull_type> coro;
-        bool started = false;
 
         SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
             : fun(fun), eof(eof)

diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
@@ -25,6 +25,13 @@ struct NullSink : Sink
     { }
 };
 
+
+struct FinishSink : virtual Sink
+{
+    virtual void finish() = 0;
+};
+
+
 /* A buffered abstract sink. Warning: a BufferedSink should not be
    used from multiple threads concurrently. */
 struct BufferedSink : virtual Sink
@@ -281,6 +288,7 @@ struct ChainSource : Source
     size_t read(char * data, size_t len) override;
 };
 
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
 
 /* Convert a function that feeds data into a Sink into a Source. The
    Source executes the function as a coroutine. */

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
@@ -2,83 +2,78 @@
 #include <archive_entry.h>
 
 #include "serialise.hh"
+#include "tarfile.hh"
 
 namespace nix {
 
-struct TarArchive {
-    struct archive * archive;
-    Source * source;
-    std::vector<unsigned char> buffer;
-
-    void check(int err, const char * reason = "failed to extract archive: %s")
-    {
-        if (err == ARCHIVE_EOF)
-            throw EndOfFile("reached end of archive");
-        else if (err != ARCHIVE_OK)
-            throw Error(reason, archive_error_string(this->archive));
+static int callback_open(struct archive *, void * self)
+{
+    return ARCHIVE_OK;
+}
+
+static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
+{
+    auto self = (TarArchive *) _self;
+    *buffer = self->buffer.data();
+
+    try {
+        return self->source->read((char *) self->buffer.data(), 4096);
+    } catch (EndOfFile &) {
+        return 0;
+    } catch (std::exception & err) {
+        archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
+        return -1;
     }
+}
 
-    TarArchive(Source & source) : buffer(4096)
-    {
-        this->archive = archive_read_new();
-        this->source = &source;
+static int callback_close(struct archive *, void * self)
+{
+    return ARCHIVE_OK;
+}
 
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_all(archive);
-        check(archive_read_open(archive,
-                (void *)this,
-                TarArchive::callback_open,
-                TarArchive::callback_read,
-                TarArchive::callback_close),
-            "failed to open archive: %s");
-    }
+void TarArchive::check(int err, const std::string & reason)
+{
+    if (err == ARCHIVE_EOF)
+        throw EndOfFile("reached end of archive");
+    else if (err != ARCHIVE_OK)
+        throw Error(reason, archive_error_string(this->archive));
+}
 
-    TarArchive(const Path & path)
-    {
-        this->archive = archive_read_new();
+TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
+{
+    this->archive = archive_read_new();
+    this->source = &source;
 
+    if (!raw) {
         archive_read_support_filter_all(archive);
         archive_read_support_format_all(archive);
-        check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
-    }
-
-    TarArchive(const TarArchive &) = delete;
-
-    void close()
-    {
-        check(archive_read_close(archive), "failed to close archive: %s");
+    } else {
+        archive_read_support_filter_all(archive);
+        archive_read_support_format_raw(archive);
+        archive_read_support_format_empty(archive);
     }
+    check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
+}
 
-    ~TarArchive()
-    {
-        if (this->archive) archive_read_free(this->archive);
-    }
 
-private:
+TarArchive::TarArchive(const Path & path)
+{
+    this->archive = archive_read_new();
 
-    static int callback_open(struct archive *, void * self) {
-        return ARCHIVE_OK;
-    }
+    archive_read_support_filter_all(archive);
+    archive_read_support_format_all(archive);
+    check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+}
 
-    static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
-    {
-        auto self = (TarArchive *)_self;
-        *buffer = self->buffer.data();
-
-        try {
-            return self->source->read((char *) self->buffer.data(), 4096);
-        } catch (EndOfFile &) {
-            return 0;
-        } catch (std::exception & err) {
-            archive_set_error(archive, EIO, "source threw exception: %s", err.what());
-            return -1;
-        }
-    }
+void TarArchive::close()
+{
+    check(archive_read_close(this->archive), "Failed to close archive (%s)");
+}
 
-    static int callback_close(struct archive *, void * self) {
-        return ARCHIVE_OK;
-    }
-};
+TarArchive::~TarArchive()
+{
+    if (this->archive) archive_read_free(this->archive);
+}
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {

diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
@@ -1,7 +1,26 @@
 #include "serialise.hh"
+#include <archive.h>
 
 namespace nix {
 
+struct TarArchive {
+    struct archive * archive;
+    Source * source;
+    std::vector<unsigned char> buffer;
+
+    void check(int err, const std::string & reason = "failed to extract archive (%s)");
+
+    TarArchive(Source & source, bool raw = false);
+
+    TarArchive(const Path & path);
+
+    // disable copy constructor
+    TarArchive(const TarArchive &) = delete;
+
+    void close();
+
+    ~TarArchive();
+};
 void unpackTarfile(Source & source, const Path & destDir);
 
 void unpackTarfile(const Path & tarFile, const Path & destDir);

diff --git a/tests/local.mk b/tests/local.mk
@@ -30,6 +30,7 @@ nix_tests = \
   signing.sh \
   shell.sh \
   brotli.sh \
+  zstd.sh \
   pure-eval.sh \
   check.sh \
   plugins.sh \

diff --git a/tests/zstd.sh b/tests/zstd.sh
@@ -0,0 +1,28 @@
+source common.sh
+
+clearStore
+clearCache
+
+cacheURI="file://$cacheDir?compression=zstd"
+
+outPath=$(nix-build dependencies.nix --no-out-link)
+
+nix copy --to $cacheURI $outPath
+
+HASH=$(nix hash path $outPath)
+
+clearStore
+clearCacheCache
+
+nix copy --from $cacheURI $outPath --no-check-sigs
+
+if ls $cacheDir/nar/*.zst &> /dev/null; then
+    echo "files do exist"
+else
+    echo "nars do not exist"
+    exit 1
+fi
+
+HASH2=$(nix hash path $outPath)
+
+[[ $HASH = $HASH2 ]]