NixOS · Oct 21, 2019 · Oct 21, 2019 · Oct 21, 2019 · Oct 21, 2019 · Oct 21, 2019
diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
@@ -1,6 +1,8 @@
 #include "lockfile.hh"
 #include "store-api.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix::flake {
 
 LockedInput::LockedInput(const nlohmann::json & json)

diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
@@ -2,7 +2,7 @@
 
 #include "flakeref.hh"
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 class Store;

diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
@@ -286,11 +286,21 @@ GitInfo exportGitHub(
             return *gitInfo;
     }
 
+    if (!rev) {
+        auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
+            owner, repo, ref ? *ref : "master");
+        CachedDownloadRequest request(url);
+        request.ttl = rev ? 1000000000 : settings.tarballTtl;
+        auto result = getDownloader()->downloadCached(store, request);
+        auto json = nlohmann::json::parse(readFile(result.path));
+        rev = Hash(json["sha"], htSHA1);
+    }
+
     // FIXME: use regular /archive URLs instead? api.github.com
     // might have stricter rate limits.
 
     auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
-        owner, repo, rev ? rev->to_string(Base16, false) : ref ? *ref : "master");
+        owner, repo, rev->to_string(Base16, false));
 
     std::string accessToken = settings.githubAccessToken.get();
     if (accessToken != "")
@@ -299,21 +309,15 @@ GitInfo exportGitHub(
     CachedDownloadRequest request(url);
     request.unpack = true;
     request.name = "source";
-    request.ttl = rev ? 1000000000 : settings.tarballTtl;
+    request.ttl = 1000000000;
     request.getLastModified = true;
     auto result = getDownloader()->downloadCached(store, request);
 
-    if (!result.etag)
-        throw Error("did not receive an ETag header from '%s'", url);
-
-    if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
-        throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
-
     assert(result.lastModified);
 
     GitInfo gitInfo;
     gitInfo.storePath = result.storePath;
-    gitInfo.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
+    gitInfo.rev = *rev;
     gitInfo.lastModified = *result.lastModified;
 
     // FIXME: this can overwrite a cache file that contains a revCount.

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "builtins/buildenv.hh"
 #include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
@@ -727,23 +728,6 @@ HookInstance::~HookInstance()
 //////////////////////////////////////////////////////////////////////
 
 
-typedef map<std::string, std::string> StringRewrites;
-
-
-std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
-{
-    for (auto & i : rewrites) {
-        size_t j = 0;
-        while ((j = s.find(i.first, j)) != string::npos)
-            s.replace(j, i.first.size(), i.second);
-    }
-    return s;
-}
-
-
-//////////////////////////////////////////////////////////////////////
-
-
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
 class SubstitutionGoal;
@@ -865,7 +849,7 @@ class DerivationGoal : public Goal
 #endif
 
     /* Hash rewriting. */
-    StringRewrites inputRewrites, outputRewrites;
+    StringMap inputRewrites, outputRewrites;
     typedef map<Path, Path> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
@@ -2537,7 +2521,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
+    auto structuredAttrs = parsedDrv->getStructuredAttrs();
     if (!structuredAttrs) return;
 
     auto json = *structuredAttrs;

diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh
@@ -6,6 +6,5 @@ namespace nix {
 
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
-void builtinBuildenv(const BasicDerivation & drv);
 
 }
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
@@ -1,4 +1,4 @@
-#include "builtins.hh"
+#include "buildenv.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -7,16 +7,14 @@
 
 namespace nix {
 
-typedef std::map<Path,int> Priorities;
-
-// FIXME: change into local variables.
-
-static Priorities priorities;
-
-static unsigned long symlinks;
+struct State
+{
+    std::map<Path, int> priorities;
+    unsigned long symlinks = 0;
+};
 
 /* For each activated package, create symlinks */
-static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
+static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
 {
     DirEntries srcFiles;
 
@@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISDIR(dstSt.st_mode)) {
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 } else if (S_ISLNK(dstSt.st_mode)) {
                     auto target = canonPath(dstFile, true);
@@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                         throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
                         throw SysError(format("creating directory '%1%'"));
-                    createLinks(target, dstFile, priorities[dstFile]);
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, target, dstFile, state.priorities[dstFile]);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
@@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISLNK(dstSt.st_mode)) {
-                    auto prevPriority = priorities[dstFile];
+                    auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(
                                 "packages '%1%' and '%2%' have the same priority %3%; "
@@ -109,41 +107,57 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
         }
 
         createSymlink(srcFile, dstFile);
-        priorities[dstFile] = priority;
-        symlinks++;
+        state.priorities[dstFile] = priority;
+        state.symlinks++;
     }
 }
 
-typedef std::set<Path> FileProp;
+void buildProfile(const Path & out, Packages && pkgs)
+{
+    State state;
 
-static FileProp done;
-static FileProp postponed = FileProp{};
+    std::set<Path> done, postponed;
 
-static Path out;
+    auto addPkg = [&](const Path & pkgDir, int priority) {
+        if (!done.insert(pkgDir).second) return;
+        createLinks(state, pkgDir, out, priority);
 
-static void addPkg(const Path & pkgDir, int priority)
-{
-    if (!done.insert(pkgDir).second) return;
-    createLinks(pkgDir, out, priority);
+        try {
+            for (const auto & p : tokenizeString<std::vector<string>>(
+                    readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
+                if (!done.count(p))
+                    postponed.insert(p);
+        } catch (SysError & e) {
+            if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
+        }
+    };
 
-    try {
-        for (const auto & p : tokenizeString<std::vector<string>>(
-                readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
-            if (!done.count(p))
-                postponed.insert(p);
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
-    }
-}
+    /* Symlink to the packages that have been installed explicitly by the
+     * user. Process in priority order to reduce unnecessary
+     * symlink/unlink steps.
+     */
+    std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
+        return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
+    });
+    for (const auto & pkg : pkgs)
+        if (pkg.active)
+            addPkg(pkg.path, pkg.priority);
 
-struct Package {
-    Path path;
-    bool active;
-    int priority;
-    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
-};
+    /* Symlink to the packages that have been "propagated" by packages
+     * installed by the user (i.e., package X declares that it wants Y
+     * installed as well). We do these later because they have a lower
+     * priority in case of collisions.
+     */
+    auto priorityCounter = 1000;
+    while (!postponed.empty()) {
+        std::set<Path> pkgDirs;
+        postponed.swap(pkgDirs);
+        for (const auto & pkgDir : pkgDirs)
+            addPkg(pkgDir, priorityCounter++);
+    }
 
-typedef std::vector<Package> Packages;
+    printError("created %d symlinks in user environment", state.symlinks);
+}
 
 void builtinBuildenv(const BasicDerivation & drv)
 {
@@ -153,7 +167,7 @@ void builtinBuildenv(const BasicDerivation & drv)
         return i->second;
     };
 
-    out = getAttr("out");
+    Path out = getAttr("out");
     createDirs(out);
 
     /* Convert the stuff we get from the environment back into a
@@ -171,31 +185,7 @@ void builtinBuildenv(const BasicDerivation & drv)
         }
     }
 
-    /* Symlink to the packages that have been installed explicitly by the
-     * user. Process in priority order to reduce unnecessary
-     * symlink/unlink steps.
-     */
-    std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) {
-        return a.priority < b.priority || (a.priority == b.priority && a.path < b.path);
-    });
-    for (const auto & pkg : pkgs)
-        if (pkg.active)
-            addPkg(pkg.path, pkg.priority);
-
-    /* Symlink to the packages that have been "propagated" by packages
-     * installed by the user (i.e., package X declares that it wants Y
-     * installed as well). We do these later because they have a lower
-     * priority in case of collisions.
-     */
-    auto priorityCounter = 1000;
-    while (!postponed.empty()) {
-        auto pkgDirs = postponed;
-        postponed = FileProp{};
-        for (const auto & pkgDir : pkgDirs)
-            addPkg(pkgDir, priorityCounter++);
-    }
-
-    printError("created %d symlinks in user environment", symlinks);
+    buildProfile(out, std::move(pkgs));
 
     createSymlink(getAttr("manifest"), out + "/manifest.nix");
 }

diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "derivations.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct Package {
+    Path path;
+    bool active;
+    int priority;
+    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
+};
+
+typedef std::vector<Package> Packages;
+
+void buildProfile(const Path & out, Packages && pkgs);
+
+void builtinBuildenv(const BasicDerivation & drv);
+
+}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
@@ -5,6 +5,7 @@
 #include "worker-protocol.hh"
 #include "derivations.hh"
 #include "nar-info.hh"
+#include "references.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1002,17 +1003,24 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
-            HashSink hashSink(htSHA256);
+            std::unique_ptr<AbstractHashSink> hashSink;
+            if (info.ca == "")
+                hashSink = std::make_unique<HashSink>(htSHA256);
+            else {
+                if (!info.references.empty())
+                    settings.requireExperimentalFeature("ca-references");
+                hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
+            }
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
-                hashSink(data, n);
+                (*hashSink)(data, n);
                 return n;
             });
 
             restorePath(realPath, wrapperSource);
 
-            auto hashResult = hashSink.finish();
+            auto hashResult = hashSink->finish();
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  wanted: %s\n  got:    %s",
@@ -1234,7 +1242,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
-                HashResult current = hashPath(info->narHash.type, toRealPath(i));
+
+                std::unique_ptr<AbstractHashSink> hashSink;
+                if (info->ca == "")
+                    hashSink = std::make_unique<HashSink>(info->narHash.type);
+                else
+                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
+
+                dumpPath(toRealPath(i), *hashSink);
+                auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError(format("path '%1%' was modified! "