NixOS · May 17, 2021 · May 17, 2021 · May 18, 2021 · May 18, 2021 · May 18, 2021
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
@@ -23,6 +23,12 @@ struct App
     // FIXME: add args, sandbox settings, metadata, ...
 };
 
+struct UnresolvedApp
+{
+    App unresolved;
+    App resolve(ref<Store>);
+};
+
 struct Installable
 {
     virtual ~Installable() { }
@@ -33,7 +39,7 @@ struct Installable
 
     DerivedPath toDerivedPath();
 
-    App toApp(EvalState & state);
+    UnresolvedApp toApp(EvalState & state);
 
     virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
@@ -927,6 +927,7 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
+            newRealisation.drvOutputDeps = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
         } else {

diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql
@@ -3,10 +3,19 @@
 -- is enabled
 
 create table if not exists Realisations (
+    id integer primary key autoincrement not null,
     drvPath text not null,
     outputName text not null, -- symbolic output id, usually "out"
     outputPath integer not null,
     signatures text, -- space-separated list
-    primary key (drvPath, outputName),
     foreign key (outputPath) references ValidPaths(id) on delete cascade
 );
+
+create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+create table if not exists RealisationsRefs (
+    referrer integer not null,
+    realisationReference integer,
+    foreign key (referrer) references Realisations(id) on delete cascade,
+    foreign key (realisationReference) references Realisations(id) on delete restrict
+);
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
@@ -885,10 +885,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopRegisterDrvOutput: {
         logger->startWork();
-        auto outputId = DrvOutput::parse(readString(from));
-        auto outputPath = StorePath(readString(from));
-        store->registerDrvOutput(Realisation{
-            .id = outputId, .outPath = outputPath});
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            auto outputId = DrvOutput::parse(readString(from));
+            auto outputPath = StorePath(readString(from));
+            store->registerDrvOutput(Realisation{
+                .id = outputId, .outPath = outputPath});
+        } else {
+            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
+            store->registerDrvOutput(realisation);
+        }
         logger->stopWork();
         break;
     }
@@ -898,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto outputId = DrvOutput::parse(readString(from));
         auto info = store->queryRealisation(outputId);
         logger->stopWork();
-        std::set<StorePath> outPaths;
-        if (info) outPaths.insert(info->outPath);
-        worker_proto::write(*store, to, outPaths);
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            std::set<StorePath> outPaths;
+            if (info) outPaths.insert(info->outPath);
+            worker_proto::write(*store, to, outPaths);
+        } else {
+            std::set<Realisation> realisations;
+            if (info) realisations.insert(*info);
+            worker_proto::write(*store, to, realisations);
+        }
         break;
     }
 

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
@@ -59,6 +59,8 @@ struct LocalStore::State::Stmts {
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
+    SQLiteStmt QueryRealisationRealisationReferences;
+    SQLiteStmt AddRealisationRealisationReference;
 };
 
 int getSchema(Path schemaPath)
@@ -316,7 +318,7 @@ LocalStore::LocalStore(const Params & params)
             )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
-                select Output.path, Realisations.signatures from Realisations
+                select Realisations.id, Output.path, Realisations.signatures from Realisations
                     inner join ValidPaths as Output on Output.id = Realisations.outputPath
                     where drvPath = ? and outputName = ?
                     ;
@@ -328,6 +330,19 @@ LocalStore::LocalStore(const Params & params)
                     where drvPath = ?
                     ;
             )");
+        state->stmts->QueryRealisationRealisationReferences.create(state->db,
+            R"(
+                select drvPath, outputName from Realisations
+                    join RealisationsRefs on realisationReference = Realisations.id
+                    where referrer = ?;
+            )");
+        state->stmts->AddRealisationRealisationReference.create(state->db,
+            R"(
+                insert or replace into RealisationsRefs (referrer, realisationReference)
+                values (
+                    ?,
+                    (select id from Realisations where drvPath = ? and outputName = ?));
+            )");
     }
 }
 
@@ -666,13 +681,17 @@ void LocalStore::registerDrvOutput(const Realisation & info)
     settings.requireExperimentalFeature("ca-derivations");
     auto state(_state.lock());
     retrySQLite<void>([&]() {
-        state->stmts->RegisterRealisedOutput.use()
-            (info.id.strHash())
-            (info.id.outputName)
-            (printStorePath(info.outPath))
-            (concatStringsSep(" ", info.signatures))
+        state->stmts->RegisterRealisedOutput
+            .use()(info.id.strHash())(info.id.outputName)(printStorePath(
+                info.outPath))(concatStringsSep(" ", info.signatures))
             .exec();
     });
+    uint64_t myId = state->db.getLastInsertedRowId();
+    for (auto& outputId : info.drvOutputDeps) {
+        state->stmts->AddRealisationRealisationReference
+            .use()(myId)(outputId.strHash())(outputId.outputName)
+            .exec();
+    }
 }
 
 void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
@@ -1670,14 +1689,32 @@ std::optional<const Realisation> LocalStore::queryRealisation(
     typedef std::optional<const Realisation> Ret;
     return retrySQLite<Ret>([&]() -> Ret {
         auto state(_state.lock());
-        auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
-            id.outputName));
-        if (!use.next())
+        auto useQueryRealisedOutput(state->stmts->QueryRealisedOutput.use()(
+            id.strHash())(id.outputName));
+        if (!useQueryRealisedOutput.next())
             return std::nullopt;
-        auto outputPath = parseStorePath(use.getStr(0));
-        auto signatures = tokenizeString<StringSet>(use.getStr(1));
+        auto realisationDbId = useQueryRealisedOutput.getInt(0);
+        auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
+        auto signatures =
+            tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
+
+        std::set<DrvOutput> drvOutputDeps;
+        auto useRealisationRefs(
+            state->stmts->QueryRealisationRealisationReferences.use()(
+                realisationDbId));
+        while (useRealisationRefs.next())
+            drvOutputDeps.insert(DrvOutput{
+                    Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
+                useRealisationRefs.getStr(1),
+            }
+        );
+
         return Ret{Realisation{
-            .id = id, .outPath = outputPath, .signatures = signatures}};
+            .id = id,
+            .outPath = outputPath,
+            .signatures = signatures,
+            .drvOutputDeps = drvOutputDeps,
+        }};
     });
 }
 }  // namespace nix
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
@@ -6,98 +6,73 @@
 #include "thread-pool.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
+#include "closure.hh"
 
 namespace nix {
 
-
 void Store::computeFSClosure(const StorePathSet & startPaths,
     StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
-    struct State
-    {
-        size_t pending;
-        StorePathSet & paths;
-        std::exception_ptr exc;
-    };
-
-    Sync<State> state_(State{0, paths_, 0});
-
-    std::function<void(const StorePath &)> enqueue;
-
-    std::condition_variable done;
-
-    enqueue = [&](const StorePath & path) -> void {
-        {
-            auto state(state_.lock());
-            if (state->exc) return;
-            if (!state->paths.insert(path).second) return;
-            state->pending++;
-        }
-
-        queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) {
-            // FIXME: calls to isValidPath() should be async
-
-            try {
-                auto info = fut.get();
-
-                if (flipDirection) {
-
-                    StorePathSet referrers;
-                    queryReferrers(path, referrers);
-                    for (auto & ref : referrers)
-                        if (ref != path)
-                            enqueue(ref);
-
-                    if (includeOutputs)
-                        for (auto & i : queryValidDerivers(path))
-                            enqueue(i);
-
-                    if (includeDerivers && path.isDerivation())
-                        for (auto & i : queryDerivationOutputs(path))
-                            if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                                enqueue(i);
-
-                } else {
-
-                    for (auto & ref : info->references)
-                        if (ref != path)
-                            enqueue(ref);
-
-                    if (includeOutputs && path.isDerivation())
-                        for (auto & i : queryDerivationOutputs(path))
-                            if (isValidPath(i)) enqueue(i);
-
-                    if (includeDerivers && info->deriver && isValidPath(*info->deriver))
-                        enqueue(*info->deriver);
-
-                }
-
-                {
-                    auto state(state_.lock());
-                    assert(state->pending);
-                    if (!--state->pending) done.notify_one();
-                }
-
-            } catch (...) {
-                auto state(state_.lock());
-                if (!state->exc) state->exc = std::current_exception();
-                assert(state->pending);
-                if (!--state->pending) done.notify_one();
-            };
-        }});
-    };
-
-    for (auto & startPath : startPaths)
-        enqueue(startPath);
-
-    {
-        auto state(state_.lock());
-        while (state->pending) state.wait(done);
-        if (state->exc) std::rethrow_exception(state->exc);
-    }
+    std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
+    if (flipDirection)
+        queryDeps = [&](const StorePath& path,
+                        std::future<ref<const ValidPathInfo>> & fut) {
+            StorePathSet res;
+            StorePathSet referrers;
+            queryReferrers(path, referrers);
+            for (auto& ref : referrers)
+                if (ref != path)
+                    res.insert(ref);
+
+            if (includeOutputs)
+                for (auto& i : queryValidDerivers(path))
+                    res.insert(i);
+
+            if (includeDerivers && path.isDerivation())
+                for (auto& i : queryDerivationOutputs(path))
+                    if (isValidPath(i) && queryPathInfo(i)->deriver == path)
+                        res.insert(i);
+            return res;
+        };
+    else
+        queryDeps = [&](const StorePath& path,
+                        std::future<ref<const ValidPathInfo>> & fut) {
+            StorePathSet res;
+            auto info = fut.get();
+            for (auto& ref : info->references)
+                if (ref != path)
+                    res.insert(ref);
+
+            if (includeOutputs && path.isDerivation())
+                for (auto& i : queryDerivationOutputs(path))
+                    if (isValidPath(i))
+                        res.insert(i);
+
+            if (includeDerivers && info->deriver && isValidPath(*info->deriver))
+                res.insert(*info->deriver);
+            return res;
+        };
+
+    computeClosure<StorePath>(
+        startPaths, paths_,
+        [&](const StorePath& path,
+            std::function<void(std::promise<std::set<StorePath>>&)>
+                processEdges) {
+            std::promise<std::set<StorePath>> promise;
+            std::function<void(std::future<ref<const ValidPathInfo>>)>
+                getDependencies =
+                    [&](std::future<ref<const ValidPathInfo>> fut) {
+                        try {
+                            promise.set_value(queryDeps(path, fut));
+                        } catch (...) {
+                            promise.set_exception(std::current_exception());
+                        }
+                    };
+            queryPathInfo(path, getDependencies);
+            processEdges(promise);
+        });
 }
 
-
 void Store::computeFSClosure(const StorePath & startPath,
     StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
@@ -279,5 +254,47 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         }});
 }
 
+std::set<DrvOutput> drvOutputReferences(
+    const std::set<Realisation> inputRealisations,
+    const StorePathSet pathReferences)
+{
+    std::set<DrvOutput> res;
 
+    std::map<StorePath, std::set<DrvOutput>> inputsByOutputPath;
+    for (const auto & input : inputRealisations)
+        inputsByOutputPath[input.outPath].insert(input.id);
+
+    for (const auto & path : pathReferences) {
+        auto theseInputs = inputsByOutputPath[path];
+        res.insert(theseInputs.begin(), theseInputs.end());
+    }
+
+    return res;
+}
+
+std::set<DrvOutput> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath)
+{
+    std::set<Realisation> inputRealisations;
+
+    for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
+        auto outputHashes =
+            staticOutputHashes(store, store.readDerivation(inputDrv));
+        for (const auto& outputName : outputNames) {
+            auto thisRealisation = store.queryRealisation(
+                DrvOutput{outputHashes.at(outputName), outputName});
+            if (!thisRealisation)
+                throw Error(
+                    "output '%s' of derivation '%s' isn’t built", outputName,
+                    store.printStorePath(inputDrv));
+            inputRealisations.insert(*thisRealisation);
+        }
+    }
+
+    auto info = store.queryPathInfo(outputPath);
+
+    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
+}
 }