Merge remote-tracking branch 'origin/master' into coerce-string

14 files changed, 264 insertions, 566 deletions

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 17701c3a3..057719e34 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -1,97 +1,114 @@
-{ command }:
+{ toplevel }:
 
 with builtins;
 with import ./utils.nix;
 
 let
 
-  showCommand =
-    { command, def, filename }:
-    ''
-      **Warning**: This program is **experimental** and its interface is subject to change.
-    ''
-    + "# Name\n\n"
-    + "`${command}` - ${def.description}\n\n"
-    + "# Synopsis\n\n"
-    + showSynopsis { inherit command; args = def.args; }
-    + (if def.commands or {} != {}
-       then
-         let
-           categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
-           listCommands = cmds:
-             concatStrings (map (name:
-               "* "
-               + "[`${command} ${name}`](./${appendName filename name}.md)"
-               + " - ${cmds.${name}.description}\n")
-               (attrNames cmds));
-         in
-         "where *subcommand* is one of the following:\n\n"
-         # FIXME: group by category
-         + (if length categories > 1
-            then
-              concatStrings (map
-                (cat:
-                  "**${toString cat.description}:**\n\n"
-                  + listCommands (filterAttrs (n: v: v.category == cat) def.commands)
-                  + "\n"
-                ) categories)
-              + "\n"
-            else
-              listCommands def.commands
-              + "\n")
-       else "")
-    + (if def ? doc
-       then def.doc + "\n\n"
-       else "")
-    + (let s = showOptions def.flags; in
-       if s != ""
-       then "# Options\n\n${s}"
-       else "")
-  ;
+  showCommand = { command, details, filename, toplevel }:
+    let
+      result = ''
+        > **Warning** \
+        > This program is **experimental** and its interface is subject to change.
+
+        # Name
+
+        `${command}` - ${details.description}
+
+        # Synopsis
+
+        ${showSynopsis command details.args}
+
+        ${maybeSubcommands}
+
+        ${maybeDocumentation}
+
+        ${maybeOptions}
+      '';
+      showSynopsis = command: args:
+        let
+          showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "...");
+          arguments = concatStringsSep " " (map showArgument args);
+        in ''
+         `${command}` [*option*...] ${arguments}
+        '';
+      maybeSubcommands = if details ? commands && details.commands != {}
+        then ''
+           where *subcommand* is one of the following:
+
+           ${subcommands}
+         ''
+        else "";
+      subcommands = if length categories > 1
+        then listCategories
+        else listSubcommands details.commands;
+      categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues details.commands)));
+      listCategories = concatStrings (map showCategory categories);
+      showCategory = cat: ''
+        **${toString cat.description}:**
+
+        ${listSubcommands (filterAttrs (n: v: v.category == cat) details.commands)}
+      '';
+      listSubcommands = cmds: concatStrings (attrValues (mapAttrs showSubcommand cmds));
+      showSubcommand = name: subcmd: ''
+        * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
+      '';
+      maybeDocumentation = if details ? doc then details.doc else "";
+      maybeOptions = if details.flags == {} then "" else ''
+        # Options
+
+        ${showOptions details.flags toplevel.flags}
+      '';
+      showOptions = options: commonOptions:
+        let
+          allOptions = options // commonOptions;
+          showCategory = cat: ''
+            ${if cat != "" then "**${cat}:**" else ""}
+
+            ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)}
+            '';
+          listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
+          showOption = name: option:
+            let
+              shortName = if option ? shortName then "/ `-${option.shortName}`" else "";
+              labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else "";
+            in trim ''
+              - `--${name}` ${shortName} ${labels}
+
+                ${option.description}
+            '';
+          categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues allOptions)));
+        in concatStrings (map showCategory categories);
+    in squash result;
 
   appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
 
-  showOptions = flags:
+  processCommand = { command, details, filename, toplevel }:
     let
-      categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues flags)));
-    in
-      concatStrings (map
-        (cat:
-          (if cat != ""
-           then "**${cat}:**\n\n"
-           else "")
-          + concatStrings
-            (map (longName:
-              let
-                flag = flags.${longName};
-              in
-                "  - `--${longName}`"
-                + (if flag ? shortName then " / `-${flag.shortName}`" else "")
-                + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
-                + "  \n"
-                + "    " + flag.description + "\n\n"
-            ) (attrNames (filterAttrs (n: v: v.category == cat) flags))))
-        categories);
-
-  showSynopsis =
-    { command, args }:
-    "`${command}` [*option*...] ${concatStringsSep " "
-      (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
-
-  processCommand = { command, def, filename }:
-    [ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ]
-    ++ concatMap
-      (name: processCommand {
-        filename = appendName filename name;
-        command = command + " " + name;
-        def = def.commands.${name};
-      })
-      (attrNames def.commands or {});
-
-in
+      cmd = {
+        inherit command;
+        name = filename + ".md";
+        value = showCommand { inherit command details filename toplevel; };
+      };
+      subcommand = subCmd: processCommand {
+        command = command + " " + subCmd;
+        details = details.commands.${subCmd};
+        filename = appendName filename subCmd;
+        inherit toplevel;
+      };
+    in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
 
-let
-  manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; };
-  summary = concatStrings (map (manpage: "    - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
-in
-(listToAttrs manpages) // { "SUMMARY.md" = summary; }
+  parsedToplevel = builtins.fromJSON toplevel;
+  manpages = processCommand {
+    command = "nix";
+    details = parsedToplevel;
+    filename = "nix";
+    toplevel = parsedToplevel;
+  };
+
+  tableOfContents = let
+    showEntry = page:
+      "    - [${page.command}](command-ref/new-cli/${page.name})";
+    in concatStringsSep "\n" (map showEntry manpages) + "\n";
+
+in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 364e02967..486dbd7a2 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -50,7 +50,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; }'
+	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 9b66ec3db..908e7e3d9 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -59,14 +59,6 @@
 @manpages@
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
-<!--
-- [Architecture](architecture/architecture.md)
-  - [Store](architecture/store/store.md)
-    - [Closure](architecture/store/store/closure.md)
-    - [Build system terminology](architecture/store/store/build-system-terminology.md)
-  - [Store Path](architecture/store/path.md)
-  - [File System Object](architecture/store/fso.md)
--->
 - [Glossary](glossary.md)
 - [Contributing](contributing/contributing.md)
   - [Hacking](contributing/hacking.md)
diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md
deleted file mode 100644
index 41deb07af..000000000
--- a/doc/manual/src/architecture/architecture.md
+++ /dev/null
@@ -1,79 +0,0 @@
-# Architecture
-
-*(This chapter is unstable and a work in progress. Incoming links may rot.)*
-
-This chapter describes how Nix works.
-It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools.
-
-## Overview
-
-Nix consists of [hierarchical layers][layer-architecture].
-
-```
-+-----------------------------------------------------------------+
-| Nix                                                             |
-|                  [ commmand line interface ]------,             |
-|                               |                   |             |
-|                           evaluates               |             |
-|                               |                manages          |
-|                               V                   |             |
-|                  [ configuration language  ]      |             |
-|                               |                   |             |
-| +-----------------------------|-------------------V-----------+ |
-| | store                  evaluates to                         | |
-| |                             |                               | |
-| |             referenced by   V       builds                  | |
-| |  [ build input ] ---> [ build plan ] ---> [ build result ]  | |
-| |                                                             | |
-| +-------------------------------------------------------------+ |
-+-----------------------------------------------------------------+
-```
-
-At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers.
-
-Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language.
-It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*.
-
-The command line and Nix language are what users interact with most.
-
-> **Note**
-> The Nix language itself does not have a notion of *packages* or *configurations*.
-> As far as we are concerned here, the inputs and results of a build plan are just data.
-
-Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them.
-It can also execute build plans to produce new data.
-
-A build plan is a series of *build tasks*.
-Each build task has a special build input which is used as *build instructions*.
-The result of a build task can be input to another build task.
-
-```
-+-----------------------------------------------------------------------------------------+
-| store                                                                                   |
-|                   .................................................                     |
-|                   :  build plan                                   :                     |
-|                   :                                               :                     |
-|  [ build input ]-----instructions-,                               :                     |
-|                   :               |                               :                     |
-|                   :               v                               :                     |
-|  [ build input ]----------->[ build task ]--instructions-,        :                     |
-|                   :                                      |        :                     |
-|                   :                                      |        :                     |
-|                   :                                      v        :                     |
-|                   :                               [ build task ]----->[ build result ]  |
-|  [ build input ]-----instructions-,                      ^        :                     |
-|                   :               |                      |        :                     |
-|                   :               v                      |        :                     |
-|  [ build input ]----------->[ build task ]---------------'        :                     |
-|                   :               ^                               :                     |
-|                   :               |                               :                     |
-|  [ build input ]------------------'                               :                     |
-|                   :                                               :                     |
-|                   :                                               :                     |
-|                   :...............................................:                     |
-|                                                                                         |
-+-----------------------------------------------------------------------------------------+
-```
-
-[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers
-[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming
diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md
deleted file mode 100644
index e0eb69f60..000000000
--- a/doc/manual/src/architecture/store/fso.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# File System Object
-
-The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object).
-
-Every file system object is one of the following:
-
- - File: an executable flag, and arbitrary data for contents
- - Directory: mapping of names to child file system objects
- - [Symbolic link][symlink]: may point anywhere.
-
-We call a store object's outermost file system object the *root*.
-
-    data FileSystemObject
-      = File      { isExecutable :: Bool, contents :: Bytes }
-      | Directory { entries :: Map FileName FileSystemObject }
-      | SymLink   { target :: Path }
-
-Examples:
-
-- a directory with contents
-
-      /nix/store/<hash>-hello-2.10
-      ├── bin
-      │   └── hello
-      └── share
-          ├── info
-          │   └── hello.info
-          └── man
-              └── man1
-                  └── hello.1.gz
-
-- a directory with relative symlink and other contents
-
-      /nix/store/<hash>-go-1.16.9
-      ├── bin -> share/go/bin
-      ├── nix-support/
-      └── share/
-
-- a directory with absolute symlink
-
-      /nix/store/d3k...-nodejs
-      └── nix_node -> /nix/store/f20...-nodejs-10.24.
-
-A bare file or symlink can be a root file system object.
-Examples:
-
-    /nix/store/<hash>-hello-2.10.tar.gz
-
-    /nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc
-
-Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended.
-Examples:
-
-- an arbitrarily symlinked file may change or not exist at all
-
-      /nix/store/<hash>-foo
-      └── foo -> /home/foo
-
-- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted
-
-      /nix/store/<hash>-bar
-      └── bar -> /nix/store/abc...-foo
-
-Nix file system objects do not support [hard links][hardlink]:
-each file system object which is not the root has exactly one parent and one name.
-However, as store objects are immutable, an underlying file system can use hard links for optimization.
-
-[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link
-[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link
diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md
deleted file mode 100644
index 663f04f46..000000000
--- a/doc/manual/src/architecture/store/path.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# Store Path
-
-Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*.
-
-Store paths are pairs of
-
-- a 20-byte [digest](#digest) for identification
-- a symbolic name for people to read.
-
-Example:
-
-- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
-- name:   `firefox-33.1`
-
-It is rendered to a file system path as the concatenation of
-
-  - [store directory](#store-directory)
-  - path-separator (`/`)
-  - [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
-  - hyphen (`-`)
-  - name
-
-Example:
-
-      /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
-      |--------| |------------------------------| |----------|
-    store directory            digest                 name
-
-## Store Directory
-
-Every [store](./store.md) has a store directory.
-
-If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path).
-
-This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
-
-> **Note**
-> The store directory defaults to `/nix/store`, but is in principle arbitrary.
-
-It is important which store a given store object belongs to:
-Files in the store object can contain store paths, and processes may read these paths.
-Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries.
-
-Therefore one can only copy store objects to a different store if
-
-- the source and target stores' directories match
-
-  or
-
-- the store object in question has no references, that is, contains no store paths.
-
-One cannot copy a store object to a store with a different store directory.
-Instead, it has to be rebuilt, together with all its dependencies.
-It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums.
-
-# Digest
-
-In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*.
-
-Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing).
-
-> **Historical Note**
-> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes.
-> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility.
-
-[digest]: https://en.m.wiktionary.org/wiki/digest#Noun
-[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function
-[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1
-[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256
-
-### Reference scanning
-
-When a new store object is built, Nix scans its file contents for store paths to construct its set of references.
-
-The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data.
-Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives.
-
-This way, scanning files captures run time dependencies without the user having to declare them explicitly.
-Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation.
-
-> **Note**
-> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure.
-This depends on the specifics of the software to build and run.
->
-> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them.
-
-## Input Addressing
-
-Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan.
-
-To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes.
-
-Nix has a custom serialisation format called Nix Archive (NAR)
-
-Store object references of this sort can *not* be validated from the content of the store object.
-Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest.
-
-## Content Addressing
-
-Content addressing means that the digest derives from the store object's contents, namely its file system objects and references.
-If one knows content addressing was used, one can recalculate the reference and thus verify the store object.
-
-Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance.
-Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md).
-
diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md
deleted file mode 100644
index 08b6701d5..000000000
--- a/doc/manual/src/architecture/store/store.md
+++ /dev/null
@@ -1,151 +0,0 @@
-# Store
-
-A Nix store is a collection of *store objects* with references between them.
-It supports operations to manipulate that collection.
-
-The following concept map is a graphical outline of this chapter.
-Arrows indicate suggested reading order.
-
-```
-                      ,--------------[ store ]----------------,
-                      |                  |                    |
-                      v                  v                    v
-               [ store object ]     [ closure ]--,      [ operations ]
-                      |               |   |      |        |        |
-                      v               |   |      v        v        |
-           [ files and processes ]    |   | [ garbage collection ] |
-               /          \           |   |                        |
-              v            v          |   v                        v
-[ file system object ] [ store path ] | [ derivation ]--->[ building ]
-                  |        ^      |   |                         |
-                  v        |      v   v                         |
-             [ digest ]----' [ reference scanning ]<------------'
-              /      \
-             v        v
-[ input addressing ] [ content addressing ]
-```
-
-## Store Object
-
-A store object can hold
-
-- arbitrary *data*
-- *references* to other store objects.
-
-Store objects can be build inputs, build results, or build tasks.
-
-Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted.
-
-## Reference
-
-A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]:
-The only way to obtain references is by adding or building store objects.
-A reference will always point to exactly one store object.
-
-## Operations
-
-A Nix store can *add*, *retrieve*, and *delete* store objects.
-
-                [ data ]
-                    |
-                    V
-    [ store ] ---> add ----> [ store' ]
-                    |
-                    V
-              [ reference ]
-
-<!-- -->
-
-              [ reference ]
-                    |
-                    V
-    [ store ] ---> get
-                    |
-                    V
-             [ store object ]
-
-<!-- -->
-
-              [ reference ]
-                    |
-                    V
-    [ store ] --> delete --> [ store' ]
-
-
-It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks.
-
-
-              [ reference ]
-                    |
-                    V
-    [ store ] --> build --(maybe)--> [ store' ]
-                             |
-                             V
-                       [ reference ]
-
-
-As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects.
-
-
-    [ store ] --> collect garbage --> [ store' ]
-
-## Files and Processes
-
-Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths.
-That allows processes to resolve references contained in files and thus access the contents of store objects.
-
-Store objects are therefore implemented as the pair of
-
-  - a [file system object](fso.md) for data
-  - a set of [store paths](path.md) for references.
-
-[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file
-[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor
-
-The following diagram shows a radical simplification of how Nix interacts with the operating system:
-It uses files as build inputs, and build outputs are files again.
-On the operating system, files can be run as processes, which in turn operate on files.
-A build function also amounts to an operating system process (not depicted).
-
-```
-+-----------------------------------------------------------------+
-| Nix                                                             |
-|                  [ commmand line interface ]------,             |
-|                               |                   |             |
-|                           evaluates               |             |
-|                               |                manages          |
-|                               V                   |             |
-|                  [ configuration language  ]      |             |
-|                               |                   |             |
-| +-----------------------------|-------------------V-----------+ |
-| | store                  evaluates to                         | |
-| |                             |                               | |
-| |             referenced by   V       builds                  | |
-| |  [ build input ] ---> [ build plan ] ---> [ build result ]  | |
-| |         ^                                        |          | |
-| +---------|----------------------------------------|----------+ |
-+-----------|----------------------------------------|------------+
-            |                                        |
-    file system object                          store path
-            |                                        |
-+-----------|----------------------------------------|------------+
-| operating system        +------------+             |            |
-|           '------------ |            | <-----------'            |
-|                         |    file    |                          |
-|                     ,-- |            | <-,                      |
-|                     |   +------------+   |                      |
-|          execute as |                    | read, write, execute |
-|                     |   +------------+   |                      |
-|                     '-> |  process   | --'                      |
-|                         +------------+                          |
-+-----------------------------------------------------------------+
-```
-
-There exist different types of stores, which all follow this model.
-Examples:
-- store on the local file system
-- remote store accessible via SSH
-- binary cache store accessible via HTTP
-
-To make store objects accessible to processes, stores ultimately have to expose store objects through the file system.
-
diff --git a/doc/manual/src/architecture/store/store/build-system-terminology.md b/doc/manual/src/architecture/store/store/build-system-terminology.md
deleted file mode 100644
index eefbaa630..000000000
--- a/doc/manual/src/architecture/store/store/build-system-terminology.md
+++ /dev/null
@@ -1,32 +0,0 @@
-# A [Rosetta stone][rosetta-stone] for build system terminology
-
-The Nix store's design is comparable to other build systems.
-Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change.
-
-The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future.
-
-| generic build system             | Nix              | [Bazel][bazel]                                                       | [Build Systems à la Carte][bsalc] | programming language     |
-| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ |
-| data (build input, build result) | store object     | [artifact][bazel-artifact]                                           | value                             | value                    |
-| build instructions               | builder          | ([depends on action type][bazel-actions])                            | function                          | function                 |
-| build task                       | derivation       | [action][bazel-action]                                               | `Task`                            | [thunk][thunk]           |
-| build plan                       | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks`                           | [call graph][call-graph] |
-| build                            | build            | build                                                                | application of `Build`            | evaluation               |
-| persistence layer                | store            | [action cache][bazel-action-cache]                                   | `Store`                           | heap                     |
-
-All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018).
-
-[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone
-[bazel]: https://bazel.build/start/bazel-intro
-[bazel-artifact]: https://bazel.build/reference/glossary#artifact
-[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html
-[bazel-action]: https://bazel.build/reference/glossary#action
-[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph
-[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph
-[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache
-[thunk]: https://en.m.wikipedia.org/wiki/Thunk
-[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph
-[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming
-[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf
-[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf
-[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf
diff --git a/doc/manual/src/architecture/store/store/closure.md b/doc/manual/src/architecture/store/store/closure.md
deleted file mode 100644
index 065b95ffc..000000000
--- a/doc/manual/src/architecture/store/store/closure.md
+++ /dev/null
@@ -1,29 +0,0 @@
-# Closure
-
-Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store.
-
-The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*.
-
-Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity:
-
-- A newly added store object cannot have references, unless it is a build task.
-
-- Build results must only refer to store objects in the closure of the build inputs.
-
-  Building a store object will add appropriate references, according to the build task.
-
-- Store objects being copied must refer to objects already in the destination store.
-
-  Recursive copying must either proceed in dependency order or be atomic.
-
-- We can only safely delete store objects which are not reachable from any reference still in use.
-
-  <!-- more details in section on garbage collection, link to it once it exists -->
-
-[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity
-[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)
-[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object
-[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type
-[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier
-
-
diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md
index 7047d3012..9a29030bd 100644
--- a/doc/manual/src/command-ref/nix-copy-closure.md
+++ b/doc/manual/src/command-ref/nix-copy-closure.md
@@ -30,8 +30,8 @@ Since `nix-copy-closure` calls `ssh`, you may be asked to type in the
 appropriate password or passphrase.  In fact, you may be asked _twice_
 because `nix-copy-closure` currently connects twice to the remote
 machine, first to get the set of paths missing on the target machine,
-and second to send the dump of those paths.  If this bothers you, use
-`ssh-agent`.
+and second to send the dump of those paths.  When using public key
+authentication, you can avoid typing the passphrase with `ssh-agent`.
 
 # Options
 
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 59ce5cac7..9f7d5057b 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -42,7 +42,7 @@ $ nix develop
 ```
 
 To get a shell with a different compilation environment (e.g. stdenv,
-gccStdenv, clangStdenv, clang11Stdenv):
+gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv):
 
 ```console
 $ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages
@@ -54,6 +54,9 @@ or if you have a flake-enabled nix:
 $ nix develop .#clang11StdenvPackages
 ```
 
+Note: you can use `ccacheStdenv` to drastically improve rebuild
+time. By default, ccache keeps artifacts in `~/.cache/ccache/`.
+
 To build Nix itself in this shell:
 
 ```console
@@ -83,9 +86,7 @@ by:
 $ nix develop
 ```
 
-## Testing
-
-Nix comes with three different flavors of tests: unit, functional and integration.
+## Running tests
 
 ### Unit-tests
 
@@ -108,3 +109,72 @@ These tests include everything that needs to interact with external services or
 Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
 
 You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
+
+### Installer tests
+
+After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.
+
+Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91):
+
+- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache:
+  - `x86_64-linux`
+  - `armv6l-linux`
+  - `armv7l-linux`
+  - `x86_64-darwin`
+
+- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
+
+#### One-time setup
+
+1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix).
+2. At cachix.org:
+    - Create or log in to an account.
+    - Create a Cachix cache using the format `<github-username>-nix-install-tests`.
+    - Navigate to the new cache > Settings > Auth Tokens.
+    - Generate a new Cachix auth token and copy the generated value.
+3. At github.com:
+    - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret.
+    - Name the secret `CACHIX_AUTH_TOKEN`.
+    - Paste the copied value of the Cachix cache auth token.
+
+#### Using the CI-generated installer for manual testing
+
+After the CI run completes, you can check the output to extract the installer URL:
+1. Click into the detailed view of the CI run.
+2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them).
+3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`)
+4. Copy the value of `install_url`
+5. To generate an install command, plug this `install_url` and your GitHub username into this template:
+
+    ```console
+    sh <(curl -L <install_url>) --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
+    ```
+
+<!-- #### Manually generating test installers
+
+There's obviously a manual way to do this, and it's still the only way for
+platforms that lack GA runners.
+
+I did do this back in Fall 2020 (before the GA approach encouraged here). I'll
+sketch what I recall in case it encourages someone to fill in detail, but: I
+didn't know what I was doing at the time and had to fumble/ask around a lot--
+so I don't want to uphold any of it as "right". It may have been dumb or
+the _hard_ way from the getgo. Fundamentals may have changed since.
+
+Here's the build command I used to do this on and for x86_64-darwin:
+nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball"
+
+I used the stable out-link to make it easier to script the next steps:
+link=$(readlink /tmp/foo)
+cp $link/*-darwin.tar.xz ~/somewheres
+
+I've lost the last steps and am just going from memory:
+
+From here, I think I had to extract and modify the `install` script to point
+it at this tarball (which I scped to my own site, but it might make more sense
+to just share them locally). I extracted this script once and then just
+search/replaced in it for each new build.
+
+The installer now supports a `--tarball-url-prefix` flag which _may_ have
+solved this need?
+-->
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index aa0ac78cb..70a0eb994 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -7,10 +7,44 @@
     translated into low-level *store derivations* (implicitly by
     `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
+  - [content-addressed derivation]{#gloss-content-addressed-derivation}\
+    A derivation which has the
+    [`__contentAddressed`](language/advanced-attributes.md#adv-attr-__contentAddressed)
+    attribute set to `true`.
+
+  - [fixed-output derivation]{#gloss-fixed-output-derivation}\
+    A derivation which includes the
+    [`outputHash`](language/advanced-attributes.md#adv-attr-outputHash) attribute.
+
   - [store]{#gloss-store}\
     The location in the file system where store objects live. Typically
     `/nix/store`.
 
+    From   the  perspective   of   the  location   where  Nix   is
+    invoked, the  Nix store can be  referred to
+    as a "_local_" or a "_remote_" one:
+
+    + A *local  store* exists  on the filesystem of
+      the machine where Nix is  invoked. You can use other
+      local stores  by passing  the `--store` flag  to the
+      `nix` command.  Local stores can be used for building derivations.
+
+    + A  *remote store*  exists  anywhere  other than  the
+      local  filesystem. One  example is  the `/nix/store`
+      directory on another machine,  accessed via `ssh` or
+      served by the `nix-serve` Perl script.
+
+  - [chroot store]{#gloss-chroot-store}\
+    A local store whose canonical path is anything other than `/nix/store`.
+
+  - [binary cache]{#gloss-binary-cache}\
+    A *binary cache* is a Nix store which uses a different format: its
+    metadata and signatures are kept in `.narinfo` files rather than in a
+    Nix database.  This different format simplifies serving store objects
+    over the network, but cannot host builds.  Examples of binary caches
+    include S3 buckets and the [NixOS binary
+    cache](https://cache.nixos.org).
+
   - [store path]{#gloss-store-path}\
     The location in the file system of a store object, i.e., an
     immediate child of the Nix store directory.
@@ -22,6 +56,19 @@
     derivation outputs (objects produced by running a build action), or
     derivations (files describing a build action).
 
+  - [input-addressed store object]{#gloss-input-addressed-store-object}\
+    A store object produced by building a
+    non-[content-addressed](#gloss-content-addressed-derivation),
+    non-[fixed-output](#gloss-fixed-output-derivation)
+    derivation.
+
+  - [output-addressed store object]{#gloss-output-addressed-store-object}\
+    A store object whose store path hashes its content.  This
+    includes derivations, the outputs of
+    [content-addressed derivations](#gloss-content-addressed-derivation),
+    and the outputs of
+    [fixed-output derivations](#gloss-fixed-output-derivation).
+
   - [substitute]{#gloss-substitute}\
     A substitute is a command invocation stored in the Nix database that
     describes how to build a store object, bypassing the normal build
@@ -29,6 +76,11 @@
     store object by downloading a pre-built version of the store object
     from some server.
 
+  - [substituter]{#gloss-substituter}\
+    A *substituter* is an additional store from which Nix will
+    copy store objects it doesn't have.  For details, see the
+    [`substituters` option](command-ref/conf-file.html#conf-substituters).
+
   - [purity]{#gloss-purity}\
     The assumption that equal Nix derivations when run always produce
     the same output. This cannot be guaranteed in general (e.g., a
diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index f25fce758..9c43132d4 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,5 +1,11 @@
 # Release X.Y (202?-??-??)
 
+* `<nix/fetchurl.nix>` now accepts an additional argument `impure` which
+  defaults to `false`.  If it is set to `true`, the `hash` and `sha256`
+  arguments will be ignored and the resulting derivation will have
+  `__impure` set to `true`, making it an impure derivation.
 * Error traces have been reworked to provide detailed explanations and more
   accurate error locations. A short excerpt of the trace is now shown by
   default when an error occurs.
+
+>>>>>>> origin/master
diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix
index d4b18472f..d0643ef46 100644
--- a/doc/manual/utils.nix
+++ b/doc/manual/utils.nix
@@ -5,6 +5,32 @@ rec {
 
   concatStrings = concatStringsSep "";
 
+  replaceStringsRec = from: to: string:
+    # recursively replace occurrences of `from` with `to` within `string`
+    # example:
+    #     replaceStringRec "--" "-" "hello-----world"
+    #     => "hello-world"
+    let
+      replaced = replaceStrings [ from ] [ to ] string;
+    in
+      if replaced == string then string else replaceStringsRec from to replaced;
+
+  squash = replaceStringsRec "\n\n\n" "\n\n";
+
+  trim = string:
+    # trim trailing spaces and squash non-leading spaces
+    let
+      trimLine = line:
+        let
+          # separate leading spaces from the rest
+          parts = split "(^ *)" line;
+          spaces = head (elemAt parts 1);
+          rest = elemAt parts 2;
+          # drop trailing spaces
+          body = head (split " *$" rest);
+        in spaces + replaceStringsRec "  " " " body;
+    in concatStringsSep "\n" (map trimLine (splitLines string));
+
   # FIXME: O(n^2)
   unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) [];