9 files changed, 738 insertions, 4 deletions

diff --git a/src/nix/command.cc b/src/nix/command.cc
index 9c80f4309..c8d91737d 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -5,6 +5,21 @@ namespace nix {
 
 Commands * RegisterCommand::commands = 0;
 
+void Command::printHelp(const string & programName, std::ostream & out)
+{
+    Args::printHelp(programName, out);
+
+    auto exs = examples();
+    if (!exs.empty()) {
+        out << "\n";
+        out << "Examples:\n";
+        for (auto & ex : exs)
+            out << "\n"
+                << "  " << ex.description << "\n" // FIXME: wrap
+                << "  $ " << ex.command << "\n";
+    }
+}
+
 MultiCommand::MultiCommand(const Commands & _commands)
     : commands(_commands)
 {
@@ -57,9 +72,47 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+StoreCommand::StoreCommand()
+{
+    storeUri = getEnv("NIX_REMOTE");
+
+    mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri);
+}
+
 void StoreCommand::run()
 {
-    run(openStore());
+    run(openStoreAt(storeUri));
+}
+
+StorePathsCommand::StorePathsCommand()
+{
+    expectArgs("paths", &storePaths);
+    mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive);
+    mkFlag(0, "all", "apply operation to the entire store", &all);
+}
+
+void StorePathsCommand::run(ref<Store> store)
+{
+    if (all) {
+        if (storePaths.size())
+            throw UsageError("‘--all’ does not expect arguments");
+        for (auto & p : store->queryAllValidPaths())
+            storePaths.push_back(p);
+    }
+
+    else {
+        for (auto & storePath : storePaths)
+            storePath = followLinksToStorePath(storePath);
+
+        if (recursive) {
+            PathSet closure;
+            for (auto & storePath : storePaths)
+                store->computeFSClosure(storePath, closure, false, false);
+            storePaths = store->topoSortPaths(closure);
+        }
+    }
+
+    run(store, storePaths);
 }
 
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 27c3ab7f2..34affc43d 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -11,6 +11,18 @@ struct Command : virtual Args
     virtual std::string name() = 0;
     virtual void prepare() { };
     virtual void run() = 0;
+
+    struct Example
+    {
+        std::string description;
+        std::string command;
+    };
+
+    typedef std::list<Example> Examples;
+
+    virtual Examples examples() { return Examples(); }
+
+    void printHelp(const string & programName, std::ostream & out) override;
 };
 
 class Store;
@@ -18,13 +30,30 @@ class Store;
 /* A command that require a Nix store. */
 struct StoreCommand : virtual Command
 {
-    bool reserveSpace;
-    StoreCommand(bool reserveSpace = true)
-        : reserveSpace(reserveSpace) { };
+    std::string storeUri;
+    StoreCommand();
     void run() override;
     virtual void run(ref<Store>) = 0;
 };
 
+/* A command that operates on zero or more store paths. */
+struct StorePathsCommand : public StoreCommand
+{
+private:
+
+    Paths storePaths;
+    bool recursive = false;
+    bool all = false;
+
+public:
+
+    StorePathsCommand();
+
+    virtual void run(ref<Store> store, Paths storePaths) = 0;
+
+    void run(ref<Store> store) override;
+};
+
 typedef std::map<std::string, ref<Command>> Commands;
 
 /* An argument parser that supports multiple subcommands,
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
new file mode 100644
index 000000000..be51fee62
--- /dev/null
+++ b/src/nix/copy.cc
@@ -0,0 +1,83 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "sync.hh"
+#include "thread-pool.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdCopy : StorePathsCommand
+{
+    std::string srcUri, dstUri;
+
+    CmdCopy()
+    {
+        mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri);
+        mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri);
+    }
+
+    std::string name() override
+    {
+        return "copy";
+    }
+
+    std::string description() override
+    {
+        return "copy paths between Nix stores";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To copy Firefox to the local store to a binary cache in file:///tmp/cache:",
+                "nix copy --to file:///tmp/cache -r $(type -p firefox)"
+            },
+        };
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        if (srcUri.empty() && dstUri.empty())
+            throw UsageError("you must pass ‘--from’ and/or ‘--to’");
+
+        ref<Store> srcStore = srcUri.empty() ? store : openStoreAt(srcUri);
+        ref<Store> dstStore = dstUri.empty() ? store : openStoreAt(dstUri);
+
+        std::string copiedLabel = "copied";
+
+        logger->setExpected(copiedLabel, storePaths.size());
+
+        ThreadPool pool;
+
+        processGraph<Path>(pool,
+            PathSet(storePaths.begin(), storePaths.end()),
+
+            [&](const Path & storePath) {
+                return srcStore->queryPathInfo(storePath)->references;
+            },
+
+            [&](const Path & storePath) {
+                checkInterrupt();
+
+                if (!dstStore->isValidPath(storePath)) {
+                    Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
+
+                    StringSink sink;
+                    srcStore->exportPaths({storePath}, false, sink);
+
+                    StringSource source(*sink.s);
+                    dstStore->importPaths(false, source, 0);
+
+                    logger->incProgress(copiedLabel);
+                } else
+                    logger->incExpected(copiedLabel, -1);
+            });
+
+        pool.process();
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdCopy>());
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 2005ec5f9..440ced97d 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -7,6 +7,7 @@
 #include "legacy.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "progress-bar.hh"
 
 namespace nix {
 
@@ -26,6 +27,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
 void mainWrapped(int argc, char * * argv)
 {
+    settings.verboseBuild = false;
+
     initNix();
     initGC();
 
@@ -42,6 +45,8 @@ void mainWrapped(int argc, char * * argv)
 
     assert(args.command);
 
+    StartProgressBar bar;
+
     args.command->prepare();
     args.command->run();
 }
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
new file mode 100644
index 000000000..c61fe7ff1
--- /dev/null
+++ b/src/nix/path-info.cc
@@ -0,0 +1,85 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+#include <iomanip>
+#include <algorithm>
+
+using namespace nix;
+
+struct CmdPathInfo : StorePathsCommand
+{
+    bool showSize = false;
+    bool showClosureSize = false;
+    bool showSigs = false;
+
+    CmdPathInfo()
+    {
+        mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
+        mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
+        mkFlag(0, "sigs", "show signatures", &showSigs);
+    }
+
+    std::string name() override
+    {
+        return "path-info";
+    }
+
+    std::string description() override
+    {
+        return "query information about store paths";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To show the closure sizes of every path in the current NixOS system closure, sorted by size:",
+                "nix path-info -rS /run/current-system | sort -nk2"
+            },
+            Example{
+                "To check the existence of a path in a binary cache:",
+                "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/"
+            },
+        };
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        size_t pathLen = 0;
+        for (auto & storePath : storePaths)
+            pathLen = std::max(pathLen, storePath.size());
+
+        for (auto storePath : storePaths) {
+            auto info = store->queryPathInfo(storePath);
+            storePath = info->path; // FIXME: screws up padding
+
+            std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
+
+            if (showSize) {
+                std::cout << '\t' << std::setw(11) << info->narSize;
+            }
+
+            if (showClosureSize) {
+                size_t totalSize = 0;
+                PathSet closure;
+                store->computeFSClosure(storePath, closure, false, false);
+                for (auto & p : closure)
+                    totalSize += store->queryPathInfo(p)->narSize;
+                std::cout << '\t' << std::setw(11) << totalSize;
+            }
+
+            if (showSigs) {
+                std::cout << '\t';
+                Strings ss;
+                if (info->ultimate) ss.push_back("ultimate");
+                for (auto & sig : info->sigs) ss.push_back(sig);
+                std::cout << concatStringsSep(" ", ss);
+            }
+
+            std::cout << std::endl;
+        }
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdPathInfo>());
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
new file mode 100644
index 000000000..659d6572a
--- /dev/null
+++ b/src/nix/progress-bar.cc
@@ -0,0 +1,157 @@
+#include "progress-bar.hh"
+#include "util.hh"
+#include "sync.hh"
+
+#include <map>
+
+namespace nix {
+
+class ProgressBar : public Logger
+{
+private:
+
+    struct ActInfo
+    {
+        Activity * activity;
+        Verbosity lvl;
+        std::string s;
+    };
+
+    struct Progress
+    {
+        uint64_t expected = 0, progress = 0;
+    };
+
+    struct State
+    {
+        std::list<ActInfo> activities;
+        std::map<Activity *, std::list<ActInfo>::iterator> its;
+        std::map<std::string, Progress> progress;
+    };
+
+    Sync<State> state_;
+
+public:
+
+    ~ProgressBar()
+    {
+        auto state(state_.lock());
+        assert(state->activities.empty());
+        writeToStderr("\r\e[K");
+    }
+
+    void log(Verbosity lvl, const FormatOrString & fs) override
+    {
+        auto state(state_.lock());
+        log(*state, lvl, fs.s);
+    }
+
+    void log(State & state, Verbosity lvl, const std::string & s)
+    {
+        writeToStderr("\r\e[K" + s + "\n");
+        update(state);
+    }
+
+    void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override
+    {
+        if (lvl > verbosity) return;
+        auto state(state_.lock());
+        state->activities.emplace_back(ActInfo{&activity, lvl, fs.s});
+        state->its.emplace(&activity, std::prev(state->activities.end()));
+        update(*state);
+    }
+
+    void stopActivity(Activity & activity) override
+    {
+        auto state(state_.lock());
+        auto i = state->its.find(&activity);
+        if (i == state->its.end()) return;
+        state->activities.erase(i->second);
+        state->its.erase(i);
+        update(*state);
+    }
+
+    void setExpected(const std::string & label, uint64_t value) override
+    {
+        auto state(state_.lock());
+        state->progress[label].expected = value;
+    }
+
+    void setProgress(const std::string & label, uint64_t value) override
+    {
+        auto state(state_.lock());
+        state->progress[label].progress = value;
+    }
+
+    void incExpected(const std::string & label, uint64_t value) override
+    {
+        auto state(state_.lock());
+        state->progress[label].expected += value;
+    }
+
+    void incProgress(const std::string & label, uint64_t value)
+    {
+        auto state(state_.lock());
+        state->progress[label].progress += value;
+    }
+
+    void update()
+    {
+        auto state(state_.lock());
+    }
+
+    void update(State & state)
+    {
+        std::string line = "\r";
+
+        std::string status = getStatus(state);
+        if (!status.empty()) {
+            line += '[';
+            line += status;
+            line += "]";
+        }
+
+        if (!state.activities.empty()) {
+            if (!status.empty()) line += " ";
+            line += state.activities.rbegin()->s;
+        }
+
+        line += "\e[K";
+        writeToStderr(line);
+    }
+
+    std::string getStatus(State & state)
+    {
+        std::string res;
+        for (auto & p : state.progress)
+            if (p.second.expected || p.second.progress) {
+                if (!res.empty()) res += ", ";
+                res += std::to_string(p.second.progress);
+                if (p.second.expected) {
+                    res += "/";
+                    res += std::to_string(p.second.expected);
+                }
+                res += " "; res += p.first;
+            }
+        return res;
+    }
+};
+
+StartProgressBar::StartProgressBar()
+{
+    if (isatty(STDERR_FILENO)) {
+        prev = logger;
+        logger = new ProgressBar();
+    }
+}
+
+StartProgressBar::~StartProgressBar()
+{
+    if (prev) {
+        auto bar = logger;
+        logger = prev;
+        delete bar;
+    }
+}
+
+}
diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh
new file mode 100644
index 000000000..d2e44f7c4
--- /dev/null
+++ b/src/nix/progress-bar.hh
@@ -0,0 +1,15 @@
+#pragma once
+
+#include "logging.hh"
+
+namespace nix {
+
+class StartProgressBar
+{
+    Logger * prev = 0;
+public:
+    StartProgressBar();
+    ~StartProgressBar();
+};
+
+}
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
new file mode 100644
index 000000000..9932aa4a9
--- /dev/null
+++ b/src/nix/sigs.cc
@@ -0,0 +1,139 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "thread-pool.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdCopySigs : StorePathsCommand
+{
+    Strings substituterUris;
+
+    CmdCopySigs()
+    {
+        mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1,
+            [&](Strings ss) { substituterUris.push_back(ss.front()); });
+    }
+
+    std::string name() override
+    {
+        return "copy-sigs";
+    }
+
+    std::string description() override
+    {
+        return "copy path signatures from substituters (like binary caches)";
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        if (substituterUris.empty())
+            throw UsageError("you must specify at least one substituter using ‘-s’");
+
+        // FIXME: factor out commonality with MixVerify.
+        std::vector<ref<Store>> substituters;
+        for (auto & s : substituterUris)
+            substituters.push_back(openStoreAt(s));
+
+        ThreadPool pool;
+
+        std::string doneLabel = "done";
+        std::atomic<size_t> added{0};
+
+        logger->setExpected(doneLabel, storePaths.size());
+
+        auto doPath = [&](const Path & storePath) {
+            Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath);
+
+            checkInterrupt();
+
+            auto info = store->queryPathInfo(storePath);
+
+            StringSet newSigs;
+
+            for (auto & store2 : substituters) {
+                try {
+                    auto info2 = store2->queryPathInfo(storePath);
+
+                    /* Don't import signatures that don't match this
+                       binary. */
+                    if (info->narHash != info2->narHash ||
+                        info->narSize != info2->narSize ||
+                        info->references != info2->references)
+                        continue;
+
+                    for (auto & sig : info2->sigs)
+                        if (!info->sigs.count(sig))
+                            newSigs.insert(sig);
+                } catch (InvalidPath &) {
+                }
+            }
+
+            if (!newSigs.empty()) {
+                store->addSignatures(storePath, newSigs);
+                added += newSigs.size();
+            }
+
+            logger->incProgress(doneLabel);
+        };
+
+        for (auto & storePath : storePaths)
+            pool.enqueue(std::bind(doPath, storePath));
+
+        pool.process();
+
+        printMsg(lvlInfo, format("imported %d signatures") % added);
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdCopySigs>());
+
+struct CmdSignPaths : StorePathsCommand
+{
+    Path secretKeyFile;
+
+    CmdSignPaths()
+    {
+        mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile);
+    }
+
+    std::string name() override
+    {
+        return "sign-paths";
+    }
+
+    std::string description() override
+    {
+        return "sign the specified paths";
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        if (secretKeyFile.empty())
+            throw UsageError("you must specify a secret key file using ‘-k’");
+
+        SecretKey secretKey(readFile(secretKeyFile));
+
+        size_t added{0};
+
+        for (auto & storePath : storePaths) {
+            auto info = store->queryPathInfo(storePath);
+
+            auto info2(*info);
+            info2.sigs.clear();
+            info2.sign(secretKey);
+            assert(!info2.sigs.empty());
+
+            if (!info->sigs.count(*info2.sigs.begin())) {
+                store->addSignatures(storePath, info2.sigs);
+                added++;
+            }
+        }
+
+        printMsg(lvlInfo, format("added %d signatures") % added);
+    }
+};
+
+static RegisterCommand r3(make_ref<CmdSignPaths>());
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
new file mode 100644
index 000000000..fd904f465
--- /dev/null
+++ b/src/nix/verify.cc
@@ -0,0 +1,168 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "sync.hh"
+#include "thread-pool.hh"
+
+#include <atomic>
+
+using namespace nix;
+
+struct CmdVerify : StorePathsCommand
+{
+    bool noContents = false;
+    bool noTrust = false;
+    Strings substituterUris;
+    size_t sigsNeeded;
+
+    CmdVerify()
+    {
+        mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
+        mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
+        mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1,
+            [&](Strings ss) { substituterUris.push_back(ss.front()); });
+        mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
+    }
+
+    std::string name() override
+    {
+        return "verify";
+    }
+
+    std::string description() override
+    {
+        return "verify the integrity of store paths";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To verify the entire Nix store:",
+                "nix verify --all"
+            },
+            Example{
+                "To check whether each path in the closure of Firefox has at least 2 signatures:",
+                "nix verify -r -n2 --no-contents $(type -p firefox)"
+            },
+        };
+    }
+
+    void run(ref<Store> store, Paths storePaths) override
+    {
+        std::vector<ref<Store>> substituters;
+        for (auto & s : substituterUris)
+            substituters.push_back(openStoreAt(s));
+
+        auto publicKeys = getDefaultPublicKeys();
+
+        std::atomic<size_t> done{0};
+        std::atomic<size_t> untrusted{0};
+        std::atomic<size_t> corrupted{0};
+        std::atomic<size_t> failed{0};
+
+        std::string doneLabel("paths checked");
+        std::string untrustedLabel("untrusted");
+        std::string corruptedLabel("corrupted");
+        std::string failedLabel("failed");
+        logger->setExpected(doneLabel, storePaths.size());
+
+        ThreadPool pool;
+
+        auto doPath = [&](const Path & storePath) {
+            try {
+                checkInterrupt();
+
+                Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath);
+
+                auto info = store->queryPathInfo(storePath);
+
+                if (!noContents) {
+
+                    HashSink sink(info->narHash.type);
+                    store->narFromPath(info->path, sink);
+
+                    auto hash = sink.finish();
+
+                    if (hash.first != info->narHash) {
+                        logger->incProgress(corruptedLabel);
+                        corrupted = 1;
+                        printMsg(lvlError,
+                            format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’")
+                            % info->path % printHash(info->narHash) % printHash(hash.first));
+                    }
+
+                }
+
+                if (!noTrust) {
+
+                    bool good = false;
+
+                    if (info->ultimate && !sigsNeeded)
+                        good = true;
+
+                    else {
+
+                        StringSet sigsSeen;
+                        size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1;
+                        size_t validSigs = 0;
+
+                        auto doSigs = [&](StringSet sigs) {
+                            for (auto sig : sigs) {
+                                if (sigsSeen.count(sig)) continue;
+                                sigsSeen.insert(sig);
+                                if (info->checkSignature(publicKeys, sig))
+                                    validSigs++;
+                            }
+                        };
+
+                        doSigs(info->sigs);
+
+                        for (auto & store2 : substituters) {
+                            if (validSigs >= actualSigsNeeded) break;
+                            try {
+                                doSigs(store2->queryPathInfo(info->path)->sigs);
+                            } catch (InvalidPath &) {
+                            } catch (Error & e) {
+                                printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
+                            }
+                        }
+
+                        if (validSigs >= actualSigsNeeded)
+                            good = true;
+                    }
+
+                    if (!good) {
+                        logger->incProgress(untrustedLabel);
+                        untrusted++;
+                        printMsg(lvlError, format("path ‘%s’ is untrusted") % info->path);
+                    }
+
+                }
+
+                logger->incProgress(doneLabel);
+                done++;
+
+            } catch (Error & e) {
+                printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
+                logger->incProgress(failedLabel);
+                failed++;
+            }
+        };
+
+        for (auto & storePath : storePaths)
+            pool.enqueue(std::bind(doPath, storePath));
+
+        pool.process();
+
+        printMsg(lvlInfo, format("%d paths checked, %d untrusted, %d corrupted, %d failed")
+            % done % untrusted % corrupted % failed);
+
+        throw Exit(
+            (corrupted ? 1 : 0) |
+            (untrusted ? 2 : 0) |
+            (failed ? 4 : 0));
+    }
+};
+
+static RegisterCommand r1(make_ref<CmdVerify>());