diff options
73 files changed, 1353 insertions, 1047 deletions
diff --git a/.gitignore b/.gitignore index ba8e95191..0c1b89ace 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ perl/Makefile.config /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/expressions/builtins.md +/doc/manual/src/language/builtins.md # /scripts/ /scripts/nix-profile.sh @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to +See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources diff --git a/configure.ac b/configure.ac index f0210ab78..64fa12fc7 100644 --- a/configure.ac +++ b/configure.ac @@ -296,15 +296,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) -# This is needed if bzip2 is a static library, and the Nix libraries -# are dynamic. -case "${host_os}" in - darwin*) - LDFLAGS="-all_load $LDFLAGS" - ;; -esac - - AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 371ed6f21..66a8cb7de 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,9 @@ ifeq ($(doc_generate),yes) +MANUAL_SRCS := \ + $(call rwildcard, $(d)/src, *.md) \ + $(call rwildcard, $(d)/src, */*.md) + # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ @@ -61,10 +65,10 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix - @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp +$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp - @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp + @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -97,7 +101,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 19f928c7e..167e221b8 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -132,113 +132,106 @@ var redirects = { "#sec-common-options": "command-ref/opt-common.html", "#ch-utilities": "command-ref/utilities.html", "#chap-hacking": "contributing/hacking.html", - "#adv-attr-allowSubstitutes": "expressions/advanced-attributes.html#adv-attr-allowSubstitutes", - "#adv-attr-allowedReferences": "expressions/advanced-attributes.html#adv-attr-allowedReferences", - "#adv-attr-allowedRequisites": "expressions/advanced-attributes.html#adv-attr-allowedRequisites", - "#adv-attr-disallowedReferences": "expressions/advanced-attributes.html#adv-attr-disallowedReferences", - "#adv-attr-disallowedRequisites": "expressions/advanced-attributes.html#adv-attr-disallowedRequisites", - "#adv-attr-exportReferencesGraph": "expressions/advanced-attributes.html#adv-attr-exportReferencesGraph", - "#adv-attr-impureEnvVars": "expressions/advanced-attributes.html#adv-attr-impureEnvVars", - "#adv-attr-outputHash": "expressions/advanced-attributes.html#adv-attr-outputHash", - "#adv-attr-outputHashAlgo": "expressions/advanced-attributes.html#adv-attr-outputHashAlgo", - "#adv-attr-outputHashMode": "expressions/advanced-attributes.html#adv-attr-outputHashMode", - "#adv-attr-passAsFile": "expressions/advanced-attributes.html#adv-attr-passAsFile", - "#adv-attr-preferLocalBuild": "expressions/advanced-attributes.html#adv-attr-preferLocalBuild", - "#fixed-output-drvs": "expressions/advanced-attributes.html#adv-attr-outputHash", - "#sec-advanced-attributes": "expressions/advanced-attributes.html", - "#sec-arguments": "expressions/arguments-variables.html", - "#sec-build-script": "expressions/build-script.html", - "#builtin-abort": "expressions/builtins.html#builtins-abort", - "#builtin-add": "expressions/builtins.html#builtins-add", - "#builtin-all": "expressions/builtins.html#builtins-all", - "#builtin-any": "expressions/builtins.html#builtins-any", - "#builtin-attrNames": "expressions/builtins.html#builtins-attrNames", - "#builtin-attrValues": "expressions/builtins.html#builtins-attrValues", - "#builtin-baseNameOf": "expressions/builtins.html#builtins-baseNameOf", - "#builtin-bitAnd": "expressions/builtins.html#builtins-bitAnd", - "#builtin-bitOr": "expressions/builtins.html#builtins-bitOr", - "#builtin-bitXor": "expressions/builtins.html#builtins-bitXor", - "#builtin-builtins": "expressions/builtins.html#builtins-builtins", - "#builtin-compareVersions": "expressions/builtins.html#builtins-compareVersions", - "#builtin-concatLists": "expressions/builtins.html#builtins-concatLists", - "#builtin-concatStringsSep": "expressions/builtins.html#builtins-concatStringsSep", - "#builtin-currentSystem": "expressions/builtins.html#builtins-currentSystem", - "#builtin-deepSeq": "expressions/builtins.html#builtins-deepSeq", - "#builtin-derivation": "expressions/builtins.html#builtins-derivation", - "#builtin-dirOf": "expressions/builtins.html#builtins-dirOf", - "#builtin-div": "expressions/builtins.html#builtins-div", - "#builtin-elem": "expressions/builtins.html#builtins-elem", - "#builtin-elemAt": "expressions/builtins.html#builtins-elemAt", - "#builtin-fetchGit": "expressions/builtins.html#builtins-fetchGit", - "#builtin-fetchTarball": "expressions/builtins.html#builtins-fetchTarball", - "#builtin-fetchurl": "expressions/builtins.html#builtins-fetchurl", - "#builtin-filterSource": "expressions/builtins.html#builtins-filterSource", - "#builtin-foldl-prime": "expressions/builtins.html#builtins-foldl-prime", - "#builtin-fromJSON": "expressions/builtins.html#builtins-fromJSON", - "#builtin-functionArgs": "expressions/builtins.html#builtins-functionArgs", - "#builtin-genList": "expressions/builtins.html#builtins-genList", - "#builtin-getAttr": "expressions/builtins.html#builtins-getAttr", - "#builtin-getEnv": "expressions/builtins.html#builtins-getEnv", - "#builtin-hasAttr": "expressions/builtins.html#builtins-hasAttr", - "#builtin-hashFile": "expressions/builtins.html#builtins-hashFile", - "#builtin-hashString": "expressions/builtins.html#builtins-hashString", - "#builtin-head": "expressions/builtins.html#builtins-head", - "#builtin-import": "expressions/builtins.html#builtins-import", - "#builtin-intersectAttrs": "expressions/builtins.html#builtins-intersectAttrs", - "#builtin-isAttrs": "expressions/builtins.html#builtins-isAttrs", - "#builtin-isBool": "expressions/builtins.html#builtins-isBool", - "#builtin-isFloat": "expressions/builtins.html#builtins-isFloat", - "#builtin-isFunction": "expressions/builtins.html#builtins-isFunction", - "#builtin-isInt": "expressions/builtins.html#builtins-isInt", - "#builtin-isList": "expressions/builtins.html#builtins-isList", - "#builtin-isNull": "expressions/builtins.html#builtins-isNull", - "#builtin-isString": "expressions/builtins.html#builtins-isString", - "#builtin-length": "expressions/builtins.html#builtins-length", - "#builtin-lessThan": "expressions/builtins.html#builtins-lessThan", - "#builtin-listToAttrs": "expressions/builtins.html#builtins-listToAttrs", - "#builtin-map": "expressions/builtins.html#builtins-map", - "#builtin-match": "expressions/builtins.html#builtins-match", - "#builtin-mul": "expressions/builtins.html#builtins-mul", - "#builtin-parseDrvName": "expressions/builtins.html#builtins-parseDrvName", - "#builtin-path": "expressions/builtins.html#builtins-path", - "#builtin-pathExists": "expressions/builtins.html#builtins-pathExists", - "#builtin-placeholder": "expressions/builtins.html#builtins-placeholder", - "#builtin-readDir": "expressions/builtins.html#builtins-readDir", - "#builtin-readFile": "expressions/builtins.html#builtins-readFile", - "#builtin-removeAttrs": "expressions/builtins.html#builtins-removeAttrs", - "#builtin-replaceStrings": "expressions/builtins.html#builtins-replaceStrings", - "#builtin-seq": "expressions/builtins.html#builtins-seq", - "#builtin-sort": "expressions/builtins.html#builtins-sort", - "#builtin-split": "expressions/builtins.html#builtins-split", - "#builtin-splitVersion": "expressions/builtins.html#builtins-splitVersion", - "#builtin-stringLength": "expressions/builtins.html#builtins-stringLength", - "#builtin-sub": "expressions/builtins.html#builtins-sub", - "#builtin-substring": "expressions/builtins.html#builtins-substring", - "#builtin-tail": "expressions/builtins.html#builtins-tail", - "#builtin-throw": "expressions/builtins.html#builtins-throw", - "#builtin-toFile": "expressions/builtins.html#builtins-toFile", - "#builtin-toJSON": "expressions/builtins.html#builtins-toJSON", - "#builtin-toPath": "expressions/builtins.html#builtins-toPath", - "#builtin-toString": "expressions/builtins.html#builtins-toString", - "#builtin-toXML": "expressions/builtins.html#builtins-toXML", - "#builtin-trace": "expressions/builtins.html#builtins-trace", - "#builtin-tryEval": "expressions/builtins.html#builtins-tryEval", - "#builtin-typeOf": "expressions/builtins.html#builtins-typeOf", - "#ssec-builtins": "expressions/builtins.html", - "#attr-system": "expressions/derivations.html#attr-system", - "#ssec-derivation": "expressions/derivations.html", - "#ch-expression-language": "expressions/expression-language.html", - "#sec-expression-syntax": "expressions/expression-syntax.html", - "#sec-generic-builder": "expressions/generic-builder.html", - "#sec-constructs": "expressions/language-constructs.html", - "#sect-let-expressions": "expressions/language-constructs.html#let-expressions", - "#ss-functions": "expressions/language-constructs.html#functions", - "#sec-language-operators": "expressions/language-operators.html", - "#table-operators": "expressions/language-operators.html", - "#ssec-values": "expressions/language-values.html", - "#sec-building-simple": "expressions/simple-building-testing.html", - "#ch-simple-expression": "expressions/simple-expression.html", - "#chap-writing-nix-expressions": "expressions/writing-nix-expressions.html", + "#adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", + "#adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", + "#adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", + "#adv-attr-disallowedReferences": "language/advanced-attributes.html#adv-attr-disallowedReferences", + "#adv-attr-disallowedRequisites": "language/advanced-attributes.html#adv-attr-disallowedRequisites", + "#adv-attr-exportReferencesGraph": "language/advanced-attributes.html#adv-attr-exportReferencesGraph", + "#adv-attr-impureEnvVars": "language/advanced-attributes.html#adv-attr-impureEnvVars", + "#adv-attr-outputHash": "language/advanced-attributes.html#adv-attr-outputHash", + "#adv-attr-outputHashAlgo": "language/advanced-attributes.html#adv-attr-outputHashAlgo", + "#adv-attr-outputHashMode": "language/advanced-attributes.html#adv-attr-outputHashMode", + "#adv-attr-passAsFile": "language/advanced-attributes.html#adv-attr-passAsFile", + "#adv-attr-preferLocalBuild": "language/advanced-attributes.html#adv-attr-preferLocalBuild", + "#fixed-output-drvs": "language/advanced-attributes.html#adv-attr-outputHash", + "#sec-advanced-attributes": "language/advanced-attributes.html", + "#builtin-abort": "language/builtins.html#builtins-abort", + "#builtin-add": "language/builtins.html#builtins-add", + "#builtin-all": "language/builtins.html#builtins-all", + "#builtin-any": "language/builtins.html#builtins-any", + "#builtin-attrNames": "language/builtins.html#builtins-attrNames", + "#builtin-attrValues": "language/builtins.html#builtins-attrValues", + "#builtin-baseNameOf": "language/builtins.html#builtins-baseNameOf", + "#builtin-bitAnd": "language/builtins.html#builtins-bitAnd", + "#builtin-bitOr": "language/builtins.html#builtins-bitOr", + "#builtin-bitXor": "language/builtins.html#builtins-bitXor", + "#builtin-builtins": "language/builtins.html#builtins-builtins", + "#builtin-compareVersions": "language/builtins.html#builtins-compareVersions", + "#builtin-concatLists": "language/builtins.html#builtins-concatLists", + "#builtin-concatStringsSep": "language/builtins.html#builtins-concatStringsSep", + "#builtin-currentSystem": "language/builtins.html#builtins-currentSystem", + "#builtin-deepSeq": "language/builtins.html#builtins-deepSeq", + "#builtin-derivation": "language/builtins.html#builtins-derivation", + "#builtin-dirOf": "language/builtins.html#builtins-dirOf", + "#builtin-div": "language/builtins.html#builtins-div", + "#builtin-elem": "language/builtins.html#builtins-elem", + "#builtin-elemAt": "language/builtins.html#builtins-elemAt", + "#builtin-fetchGit": "language/builtins.html#builtins-fetchGit", + "#builtin-fetchTarball": "language/builtins.html#builtins-fetchTarball", + "#builtin-fetchurl": "language/builtins.html#builtins-fetchurl", + "#builtin-filterSource": "language/builtins.html#builtins-filterSource", + "#builtin-foldl-prime": "language/builtins.html#builtins-foldl-prime", + "#builtin-fromJSON": "language/builtins.html#builtins-fromJSON", + "#builtin-functionArgs": "language/builtins.html#builtins-functionArgs", + "#builtin-genList": "language/builtins.html#builtins-genList", + "#builtin-getAttr": "language/builtins.html#builtins-getAttr", + "#builtin-getEnv": "language/builtins.html#builtins-getEnv", + "#builtin-hasAttr": "language/builtins.html#builtins-hasAttr", + "#builtin-hashFile": "language/builtins.html#builtins-hashFile", + "#builtin-hashString": "language/builtins.html#builtins-hashString", + "#builtin-head": "language/builtins.html#builtins-head", + "#builtin-import": "language/builtins.html#builtins-import", + "#builtin-intersectAttrs": "language/builtins.html#builtins-intersectAttrs", + "#builtin-isAttrs": "language/builtins.html#builtins-isAttrs", + "#builtin-isBool": "language/builtins.html#builtins-isBool", + "#builtin-isFloat": "language/builtins.html#builtins-isFloat", + "#builtin-isFunction": "language/builtins.html#builtins-isFunction", + "#builtin-isInt": "language/builtins.html#builtins-isInt", + "#builtin-isList": "language/builtins.html#builtins-isList", + "#builtin-isNull": "language/builtins.html#builtins-isNull", + "#builtin-isString": "language/builtins.html#builtins-isString", + "#builtin-length": "language/builtins.html#builtins-length", + "#builtin-lessThan": "language/builtins.html#builtins-lessThan", + "#builtin-listToAttrs": "language/builtins.html#builtins-listToAttrs", + "#builtin-map": "language/builtins.html#builtins-map", + "#builtin-match": "language/builtins.html#builtins-match", + "#builtin-mul": "language/builtins.html#builtins-mul", + "#builtin-parseDrvName": "language/builtins.html#builtins-parseDrvName", + "#builtin-path": "language/builtins.html#builtins-path", + "#builtin-pathExists": "language/builtins.html#builtins-pathExists", + "#builtin-placeholder": "language/builtins.html#builtins-placeholder", + "#builtin-readDir": "language/builtins.html#builtins-readDir", + "#builtin-readFile": "language/builtins.html#builtins-readFile", + "#builtin-removeAttrs": "language/builtins.html#builtins-removeAttrs", + "#builtin-replaceStrings": "language/builtins.html#builtins-replaceStrings", + "#builtin-seq": "language/builtins.html#builtins-seq", + "#builtin-sort": "language/builtins.html#builtins-sort", + "#builtin-split": "language/builtins.html#builtins-split", + "#builtin-splitVersion": "language/builtins.html#builtins-splitVersion", + "#builtin-stringLength": "language/builtins.html#builtins-stringLength", + "#builtin-sub": "language/builtins.html#builtins-sub", + "#builtin-substring": "language/builtins.html#builtins-substring", + "#builtin-tail": "language/builtins.html#builtins-tail", + "#builtin-throw": "language/builtins.html#builtins-throw", + "#builtin-toFile": "language/builtins.html#builtins-toFile", + "#builtin-toJSON": "language/builtins.html#builtins-toJSON", + "#builtin-toPath": "language/builtins.html#builtins-toPath", + "#builtin-toString": "language/builtins.html#builtins-toString", + "#builtin-toXML": "language/builtins.html#builtins-toXML", + "#builtin-trace": "language/builtins.html#builtins-trace", + "#builtin-tryEval": "language/builtins.html#builtins-tryEval", + "#builtin-typeOf": "language/builtins.html#builtins-typeOf", + "#ssec-builtins": "language/builtins.html", + "#attr-system": "language/derivations.html#attr-system", + "#ssec-derivation": "language/derivations.html", + "#ch-expression-language": "language/index.html", + "#sec-constructs": "language/constructs.html", + "#sect-let-language": "language/constructs.html#let-language", + "#ss-functions": "language/constructs.html#functions", + "#sec-language-operators": "language/operators.html", + "#table-operators": "language/operators.html", + "#ssec-values": "language/values.html", "#gloss-closure": "glossary.html#gloss-closure", "#gloss-derivation": "glossary.html#gloss-derivation", "#gloss-deriver": "glossary.html#gloss-deriver", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9728728aa..8fbb59716 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,21 +26,14 @@ - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - [Serving a Nix store via S3](package-management/s3-substituter.md) -- [Writing Nix Expressions](expressions/writing-nix-expressions.md) - - [A Simple Nix Expression](expressions/simple-expression.md) - - [Expression Syntax](expressions/expression-syntax.md) - - [Build Script](expressions/build-script.md) - - [Arguments and Variables](expressions/arguments-variables.md) - - [Building and Testing](expressions/simple-building-testing.md) - - [Generic Builder Syntax](expressions/generic-builder.md) - - [Writing Nix Expressions](expressions/expression-language.md) - - [Values](expressions/language-values.md) - - [Language Constructs](expressions/language-constructs.md) - - [Operators](expressions/language-operators.md) - - [Derivations](expressions/derivations.md) - - [Advanced Attributes](expressions/advanced-attributes.md) - - [Built-in Constants](expressions/builtin-constants.md) - - [Built-in Functions](expressions/builtins.md) +- [Nix Language](language/index.md) + - [Data Types](language/values.md) + - [Language Constructs](language/constructs.md) + - [Operators](language/operators.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Built-in Constants](language/builtin-constants.md) + - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) @@ -66,6 +59,14 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) +<!-- +- [Architecture](architecture/architecture.md) + - [Store](architecture/store/store.md) + - [Closure](architecture/store/store/closure.md) + - [Build system terminology](architecture/store/store/build-system-terminology.md) + - [Store Path](architecture/store/path.md) + - [File System Object](architecture/store/fso.md) +--> - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index b0d7fbf1a..fefd10100 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix ping-store --store ssh://mac +$ nix store ping --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md new file mode 100644 index 000000000..41deb07af --- /dev/null +++ b/doc/manual/src/architecture/architecture.md @@ -0,0 +1,79 @@ +# Architecture + +*(This chapter is unstable and a work in progress. Incoming links may rot.)* + +This chapter describes how Nix works. +It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. + +## Overview + +Nix consists of [hierarchical layers][layer-architecture]. + +``` ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | | | +| +-------------------------------------------------------------+ | ++-----------------------------------------------------------------+ +``` + +At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. + +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. + +The command line and Nix language are what users interact with most. + +> **Note** +> The Nix language itself does not have a notion of *packages* or *configurations*. +> As far as we are concerned here, the inputs and results of a build plan are just data. + +Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. +It can also execute build plans to produce new data. + +A build plan is a series of *build tasks*. +Each build task has a special build input which is used as *build instructions*. +The result of a build task can be input to another build task. + +``` ++-----------------------------------------------------------------------------------------+ +| store | +| ................................................. | +| : build plan : | +| : : | +| [ build input ]-----instructions-, : | +| : | : | +| : v : | +| [ build input ]----------->[ build task ]--instructions-, : | +| : | : | +| : | : | +| : v : | +| : [ build task ]----->[ build result ] | +| [ build input ]-----instructions-, ^ : | +| : | | : | +| : v | : | +| [ build input ]----------->[ build task ]---------------' : | +| : ^ : | +| : | : | +| [ build input ]------------------' : | +| : : | +| : : | +| :...............................................: | +| | ++-----------------------------------------------------------------------------------------+ +``` + +[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers +[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md new file mode 100644 index 000000000..e0eb69f60 --- /dev/null +++ b/doc/manual/src/architecture/store/fso.md @@ -0,0 +1,69 @@ +# File System Object + +The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object). + +Every file system object is one of the following: + + - File: an executable flag, and arbitrary data for contents + - Directory: mapping of names to child file system objects + - [Symbolic link][symlink]: may point anywhere. + +We call a store object's outermost file system object the *root*. + + data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } + +Examples: + +- a directory with contents + + /nix/store/<hash>-hello-2.10 + ├── bin + │ └── hello + └── share + ├── info + │ └── hello.info + └── man + └── man1 + └── hello.1.gz + +- a directory with relative symlink and other contents + + /nix/store/<hash>-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ + +- a directory with absolute symlink + + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. + +A bare file or symlink can be a root file system object. +Examples: + + /nix/store/<hash>-hello-2.10.tar.gz + + /nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc + +Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended. +Examples: + +- an arbitrarily symlinked file may change or not exist at all + + /nix/store/<hash>-foo + └── foo -> /home/foo + +- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted + + /nix/store/<hash>-bar + └── bar -> /nix/store/abc...-foo + +Nix file system objects do not support [hard links][hardlink]: +each file system object which is not the root has exactly one parent and one name. +However, as store objects are immutable, an underlying file system can use hard links for optimization. + +[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link +[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md new file mode 100644 index 000000000..663f04f46 --- /dev/null +++ b/doc/manual/src/architecture/store/path.md @@ -0,0 +1,105 @@ +# Store Path + +Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*. + +Store paths are pairs of + +- a 20-byte [digest](#digest) for identification +- a symbolic name for people to read. + +Example: + +- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +- name: `firefox-33.1` + +It is rendered to a file system path as the concatenation of + + - [store directory](#store-directory) + - path-separator (`/`) + - [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - hyphen (`-`) + - name + +Example: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 + |--------| |------------------------------| |----------| + store directory digest name + +## Store Directory + +Every [store](./store.md) has a store directory. + +If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path). + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. + +> **Note** +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- the source and target stores' directories match + + or + +- the store object in question has no references, that is, contains no store paths. + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. + +# Digest + +In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. + +Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). + +> **Historical Note** +> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. + +[digest]: https://en.m.wiktionary.org/wiki/digest#Noun +[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function +[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 +[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 + +### Reference scanning + +When a new store object is built, Nix scans its file contents for store paths to construct its set of references. + +The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data. +Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives. + +This way, scanning files captures run time dependencies without the user having to declare them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. + +> **Note** +> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +This depends on the specifics of the software to build and run. +> +> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. + +## Input Addressing + +Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. + +To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes. + +Nix has a custom serialisation format called Nix Archive (NAR) + +Store object references of this sort can *not* be validated from the content of the store object. +Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. + +## Content Addressing + +Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. +If one knows content addressing was used, one can recalculate the reference and thus verify the store object. + +Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance. +Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md). + diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md new file mode 100644 index 000000000..08b6701d5 --- /dev/null +++ b/doc/manual/src/architecture/store/store.md @@ -0,0 +1,151 @@ +# Store + +A Nix store is a collection of *store objects* with references between them. +It supports operations to manipulate that collection. + +The following concept map is a graphical outline of this chapter. +Arrows indicate suggested reading order. + +``` + ,--------------[ store ]----------------, + | | | + v v v + [ store object ] [ closure ]--, [ operations ] + | | | | | | + v | | v v | + [ files and processes ] | | [ garbage collection ] | + / \ | | | + v v | v v +[ file system object ] [ store path ] | [ derivation ]--->[ building ] + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ]<------------' + / \ + v v +[ input addressing ] [ content addressing ] +``` + +## Store Object + +A store object can hold + +- arbitrary *data* +- *references* to other store objects. + +Store objects can be build inputs, build results, or build tasks. + +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. + +## Reference + +A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +## Operations + +A Nix store can *add*, *retrieve*, and *delete* store objects. + + [ data ] + | + V + [ store ] ---> add ----> [ store' ] + | + V + [ reference ] + +<!-- --> + + [ reference ] + | + V + [ store ] ---> get + | + V + [ store object ] + +<!-- --> + + [ reference ] + | + V + [ store ] --> delete --> [ store' ] + + +It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. + + + [ reference ] + | + V + [ store ] --> build --(maybe)--> [ store' ] + | + V + [ reference ] + + +As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. + + + [ store ] --> collect garbage --> [ store' ] + +## Files and Processes + +Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. +That allows processes to resolve references contained in files and thus access the contents of store objects. + +Store objects are therefore implemented as the pair of + + - a [file system object](fso.md) for data + - a set of [store paths](path.md) for references. + +[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file +[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor + +The following diagram shows a radical simplification of how Nix interacts with the operating system: +It uses files as build inputs, and build outputs are files again. +On the operating system, files can be run as processes, which in turn operate on files. +A build function also amounts to an operating system process (not depicted). + +``` ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | ^ | | | +| +---------|----------------------------------------|----------+ | ++-----------|----------------------------------------|------------+ + | | + file system object store path + | | ++-----------|----------------------------------------|------------+ +| operating system +------------+ | | +| '------------ | | <-----------' | +| | file | | +| ,-- | | <-, | +| | +------------+ | | +| execute as | | read, write, execute | +| | +------------+ | | +| '-> | process | --' | +| +------------+ | ++-----------------------------------------------------------------+ +``` + +There exist different types of stores, which all follow this model. +Examples: +- store on the local file system +- remote store accessible via SSH +- binary cache store accessible via HTTP + +To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. + diff --git a/doc/manual/src/architecture/store/store/build-system-terminology.md b/doc/manual/src/architecture/store/store/build-system-terminology.md new file mode 100644 index 000000000..eefbaa630 --- /dev/null +++ b/doc/manual/src/architecture/store/store/build-system-terminology.md @@ -0,0 +1,32 @@ +# A [Rosetta stone][rosetta-stone] for build system terminology + +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. + +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | + +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf diff --git a/doc/manual/src/architecture/store/store/closure.md b/doc/manual/src/architecture/store/store/closure.md new file mode 100644 index 000000000..065b95ffc --- /dev/null +++ b/doc/manual/src/architecture/store/store/closure.md @@ -0,0 +1,29 @@ +# Closure + +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. + +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: + +- A newly added store object cannot have references, unless it is a build task. + +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to the build task. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + <!-- more details in section on garbage collection, link to it once it exists --> + +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + + diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index a372c5eae..a5df35d77 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -198,7 +198,7 @@ a number of possible ways: another. - If `--from-expression` is given, *args* are Nix - [functions](../expressions/language-constructs.md#functions) + [functions](../language/constructs.md#functions) that are called with the active Nix expression as their single argument. The derivations returned by those function calls are installed. This allows derivations to be specified in an diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 2e198daed..8f143729e 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -51,7 +51,7 @@ standard input. - `--strict`\ When used with `--eval`, recursively evaluate list elements and attributes. Normally, such sub-expressions are left unevaluated - (since the Nix expression language is lazy). + (since the Nix language is lazy). > **Warning** > @@ -66,7 +66,7 @@ standard input. When used with `--eval`, print the resulting value as an XML representation of the abstract syntax tree rather than as an ATerm. The schema is the same as that used by the [`toXML` - built-in](../expressions/builtins.md). + built-in](../language/builtins.md). - `--read-write-mode`\ When used with `--eval`, perform evaluation in read/write mode so diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index dc8faba68..ecd838e8d 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -121,7 +121,7 @@ Special exit codes: - `102`\ Hash mismatch, the build output was rejected because it does not match the [`outputHash` attribute of the - derivation](../expressions/advanced-attributes.md). + derivation](../language/advanced-attributes.md). - `104`\ Not deterministic, the build succeeded in check mode but the diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 51d7de18a..e612c416f 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -145,7 +145,7 @@ Most Nix commands accept the following command-line options: expression evaluator will automatically try to call functions that it encounters. It can automatically call functions for which every argument has a [default - value](../expressions/language-constructs.md#functions) (e.g., + value](../language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a @@ -164,7 +164,7 @@ Most Nix commands accept the following command-line options: So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](../expressions/builtins.md) for + value [`builtins.currentSystem`](../language/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md deleted file mode 100644 index 12198c879..000000000 --- a/doc/manual/src/expressions/arguments-variables.md +++ /dev/null @@ -1,80 +0,0 @@ -# Arguments and Variables - -The [Nix expression for GNU Hello](expression-syntax.md) is a -function; it is missing some arguments that have to be filled in -somewhere. In the Nix Packages collection this is done in the file -`pkgs/top-level/all-packages.nix`, where all Nix expressions for -packages are imported and called with the appropriate arguments. Here -are some fragments of `all-packages.nix`, with annotations of what -they mean: - -```nix -... - -rec { ① - - hello = import ../applications/misc/hello/ex-1 ② { ③ - inherit fetchurl stdenv perl; - }; - - perl = import ../development/interpreters/perl { ④ - inherit fetchurl stdenv; - }; - - fetchurl = import ../build-support/fetchurl { - inherit stdenv; ... - }; - - stdenv = ...; - -} -``` - -1. This file defines a set of attributes, all of which are concrete - derivations (i.e., not functions). In fact, we define a *mutually - recursive* set of attributes. That is, the attributes can refer to - each other. This is precisely what we want since we want to “plug” - the various packages into each other. - -2. Here we *import* the Nix expression for GNU Hello. The import - operation just loads and returns the specified Nix expression. In - fact, we could just have put the contents of the Nix expression - for GNU Hello in `all-packages.nix` at this point. That would be - completely equivalent, but it would make `all-packages.nix` rather - bulky. - - Note that we refer to `../applications/misc/hello/ex-1`, not - `../applications/misc/hello/ex-1/default.nix`. When you try to - import a directory, Nix automatically appends `/default.nix` to the - file name. - -3. This is where the actual composition takes place. Here we *call* the - function imported from `../applications/misc/hello/ex-1` with a set - containing the things that the function expects, namely `fetchurl`, - `stdenv`, and `perl`. We use inherit again to use the attributes - defined in the surrounding scope (we could also have written - `fetchurl = fetchurl;`, etc.). - - The result of this function call is an actual derivation that can be - built by Nix (since when we fill in the arguments of the function, - what we get is its body, which is the call to `stdenv.mkDerivation` - in the [Nix expression for GNU Hello](expression-syntax.md)). - - > **Note** - > - > Nixpkgs has a convenience function `callPackage` that imports and - > calls a function, filling in any missing arguments by passing the - > corresponding attribute from the Nixpkgs set, like this: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { }; - > ``` - > - > If necessary, you can set or override arguments: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; }; - > ``` - -4. Likewise, we have to instantiate Perl, `fetchurl`, and the standard - environment. diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md deleted file mode 100644 index b1eacae88..000000000 --- a/doc/manual/src/expressions/build-script.md +++ /dev/null @@ -1,70 +0,0 @@ -# Build Script - -Here is the builder referenced from Hello's Nix expression (stored in -`pkgs/applications/misc/hello/ex-1/builder.sh`): - -```bash -source $stdenv/setup ① - -PATH=$perl/bin:$PATH ② - -tar xvfz $src ③ -cd hello-* -./configure --prefix=$out ④ -make ⑤ -make install -``` - -The builder can actually be made a lot shorter by using the *generic -builder* functions provided by `stdenv`, but here we write out the build -steps to elucidate what a builder does. It performs the following steps: - -1. When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the derivation). - This is done to prevent undeclared inputs from being used in the - build process. If for example the `PATH` contained `/usr/bin`, then - you might accidentally use `/usr/bin/gcc`. - - So the first step is to set up the environment. This is done by - calling the `setup` script of the standard environment. The - environment variable `stdenv` points to the location of the - standard environment being used. (It wasn't specified explicitly - as an attribute in Hello's Nix expression, but `mkDerivation` adds - it automatically.) - -2. Since Hello needs Perl, we have to make sure that Perl is in the - `PATH`. The `perl` environment variable points to the location of - the Perl package (since it was passed in as an attribute to the - derivation), so `$perl/bin` is the directory containing the Perl - interpreter. - -3. Now we have to unpack the sources. The `src` attribute was bound to - the result of fetching the Hello source tarball from the network, so - the `src` environment variable points to the location in the Nix - store to which the tarball was downloaded. After unpacking, we `cd` - to the resulting source directory. - - The whole build is performed in a temporary directory created in - `/tmp`, by the way. This directory is removed after the builder - finishes, so there is no need to clean up the sources afterwards. - Also, the temporary directory is always newly created, so you don't - have to worry about files from previous builds interfering with the - current build. - -4. GNU Hello is a typical Autoconf-based package, so we first have to - run its `configure` script. In Nix every package is stored in a - separate location in the Nix store, for instance - `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix - computes this path by cryptographically hashing all attributes of - the derivation. The path is passed to the builder through the `out` - environment variable. So here we give `configure` the parameter - `--prefix=$out` to cause Hello to be installed in the expected - location. - -5. Finally we build Hello (`make`) and install it into the location - specified by `out` (`make install`). - -If you are wondering about the absence of error checking on the result -of various commands called in the builder: this is because the shell -script is evaluated with Bash's `-e` option, which causes the script to -be aborted if any command fails without an error check. diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md deleted file mode 100644 index 6b93e692c..000000000 --- a/doc/manual/src/expressions/expression-syntax.md +++ /dev/null @@ -1,93 +0,0 @@ -# Expression Syntax - -Here is a Nix expression for GNU Hello: - -```nix -{ stdenv, fetchurl, perl }: ① - -stdenv.mkDerivation { ② - name = "hello-2.1.1"; ③ - builder = ./builder.sh; ④ - src = fetchurl { ⑤ - url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - }; - inherit perl; ⑥ -} -``` - -This file is actually already in the Nix Packages collection in -`pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to -place each package in a separate directory and call the single Nix -expression in that directory `default.nix`. The file has the following -elements (referenced from the figure by number): - -1. This states that the expression is a *function* that expects to be - called with three arguments: `stdenv`, `fetchurl`, and `perl`. They - are needed to build Hello, but we don't know how to build them here; - that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages; it provides a - “standard” environment consisting of the things you would expect - in a basic Unix environment: a C/C++ compiler (GCC, to be precise), - the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, - etc. `fetchurl` is a function that downloads files. `perl` is the - Perl interpreter. - - Nix functions generally have the form `{ x, y, ..., z }: e` where - `x`, `y`, etc. are the names of the expected arguments, and where - *e* is the body of the function. So here, the entire remainder of - the file is the body of the function; when given the required - arguments, the body should describe how to build an instance of - the Hello package. - -2. So we have to build a package. Building something from other stuff - is called a *derivation* in Nix (as opposed to sources, which are - built by humans instead of computers). We perform a derivation by - calling `stdenv.mkDerivation`. `mkDerivation` is a function - provided by `stdenv` that builds a package from a set of - *attributes*. A set is just a list of key/value pairs where each - key is a string and each value is an arbitrary Nix - expression. They take the general form `{ name1 = expr1; ... - nameN = exprN; }`. - -3. The attribute `name` specifies the symbolic name and version of - the package. Nix doesn't really care about these things, but they - are used by for instance `nix-env -q` to show a “human-readable” - name for packages. This attribute is required by `mkDerivation`. - -4. The attribute `builder` specifies the builder. This attribute can - sometimes be omitted, in which case `mkDerivation` will fill in a - default builder (which does a `configure; make; make install`, in - essence). Hello is sufficiently simple that the default builder - would suffice, but in this case, we will show an actual builder - for educational purposes. The value `./builder.sh` refers to the - shell script shown in the [next section](build-script.md), - discussed below. - -5. The builder has to know what the sources of the package are. Here, - the attribute `src` is bound to the result of a call to the - `fetchurl` function. Given a URL and a SHA-256 hash of the expected - contents of the file at that URL, this function builds a derivation - that downloads the file and checks its hash. So the sources are a - dependency that like all other dependencies is built before Hello - itself is built. - - Instead of `src` any other name could have been used, and in fact - there can be any number of sources (bound to different attributes). - However, `src` is customary, and it's also expected by the default - builder (which we don't use in this example). - -6. Since the derivation requires Perl, we have to pass the value of the - `perl` function argument to the builder. All attributes in the set - are actually passed as environment variables to the builder, so - declaring an attribute - - ```nix - perl = perl; - ``` - - will do the trick: it binds an attribute `perl` to the function - argument which also happens to be called `perl`. However, it looks a - bit silly, so there is a shorter syntax. The `inherit` keyword - causes the specified attributes to be bound to whatever variables - with the same name happen to be in scope. diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md deleted file mode 100644 index cf26b5f82..000000000 --- a/doc/manual/src/expressions/generic-builder.md +++ /dev/null @@ -1,66 +0,0 @@ -# Generic Builder Syntax - -Recall that the [build script for GNU Hello](build-script.md) looked -something like this: - -```bash -PATH=$perl/bin:$PATH -tar xvfz $src -cd hello-* -./configure --prefix=$out -make -make install -``` - -The builders for almost all Unix packages look like this — set up some -environment variables, unpack the sources, configure, build, and -install. For this reason the standard environment provides some Bash -functions that automate the build process. Here is what a builder using -the generic build facilities looks like: - -```bash -buildInputs="$perl" ① - -source $stdenv/setup ② - -genericBuild ③ -``` - -Here is what each line means: - -1. The `buildInputs` variable tells `setup` to use the indicated - packages as “inputs”. This means that if a package provides a `bin` - subdirectory, it's added to `PATH`; if it has a `include` - subdirectory, it's added to GCC's header search path; and so on. - (This is implemented in a modular way: `setup` tries to source the - file `pkg/nix-support/setup-hook` of all dependencies. These “setup - hooks” can then set up whatever environment variables they want; for - instance, the setup hook for Perl sets the `PERL5LIB` environment - variable to contain the `lib/site_perl` directories of all inputs.) - -2. The function `genericBuild` is defined in the file `$stdenv/setup`. - -3. The final step calls the shell function `genericBuild`, which - performs the steps that were done explicitly in the previous build - script. The generic builder is smart enough to figure out whether - to unpack the sources using `gzip`, `bzip2`, etc. It can be - customised in many ways; see the Nixpkgs manual for details. - -Discerning readers will note that the `buildInputs` could just as well -have been set in the Nix expression, like this: - -```nix - buildInputs = [ perl ]; -``` - -The `perl` attribute can then be removed, and the builder becomes even -shorter: - -```bash -source $stdenv/setup -genericBuild -``` - -In fact, `mkDerivation` provides a default builder that looks exactly -like that, so it is actually possible to omit the builder for Hello -entirely. diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md deleted file mode 100644 index 75ae9f2eb..000000000 --- a/doc/manual/src/expressions/language-values.md +++ /dev/null @@ -1,251 +0,0 @@ -# Values - -## Simple Values - -Nix has the following basic data types: - - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the result of an expression into a string by - enclosing it in `${...}`, a feature known as *antiquotation*. The - enclosed expression must evaluate to something that can be coerced - into a string (meaning that it must be a string, a path, or a - derivation). For instance, rather than writing - - ```nix - "--with-freetype2-library=" + freetype + "/lib" - ``` - - (where `freetype` is a derivation), you can instead write the more - natural - - ```nix - "--with-freetype2-library=${freetype}/lib" - ``` - - The latter is automatically translated to the former. A more - complicated example (from the Nix expression for - [Qt](http://www.trolltech.com/products/qt)): - - ```nix - configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} - "; - ``` - - Note that Nix expressions and strings can be arbitrarily nested; in - this case the outer string contains various antiquotations that - themselves contain strings (e.g., `"-thread"`), some of which in - turn contain expressions (e.g., `${mesa}`). - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Antiquotation (`${expr}`) is supported in indented strings. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - Numbers are type-compatible: pure integer operations will always - return integers, whereas any operation involving at least one - floating point number will have a floating point number as a result. - - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - Paths can also be specified between angle brackets, e.g. - `<nixpkgs>`. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - - Antiquotation is supported in any paths except those in angle brackets. - `./${foo}-${bar}.nix` is a more convenient way of writing - `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At - least one slash must appear *before* any antiquotations for this to be - recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division - operation. `./a.${foo}/b.${bar}` is a path. - - - *Booleans* with values `true` and `false`. - - - The null value, denoted as `null`. - -## Lists - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -## Sets - -Sets are really the core of the language, since ultimately the Nix -language is all about creating derivations, which are really just sets -of attributes to be passed to build scripts. - -Sets are just a list of name/value pairs (called *attributes*) enclosed -in curly brackets, where each value is an arbitrary expression -terminated by a semicolon. For example: - -```nix -{ x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. The order of -the attributes is irrelevant. An attribute name may only occur once. - -Attributes can be selected from a set using the `.` operator. For -instance, - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -evaluates to `"Foo"`. It is possible to provide a default value in an -attribute selection using the `or` keyword. For example, - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -will evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}" -``` - -This will evaluate to `123` (Assuming `bar` is antiquotable). In the -case where an attribute name is just a single antiquotation, the quotes -can be dropped: - -```nix -{ foo = 123; }.${bar} or 456 -``` - -This will evaluate to `123` if `bar` evaluates to `"foo"` when coerced -to a string and `456` otherwise (again assuming `bar` is antiquotable). - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` is not -antiquotable), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md deleted file mode 100644 index 7f0d8f841..000000000 --- a/doc/manual/src/expressions/simple-building-testing.md +++ /dev/null @@ -1,61 +0,0 @@ -# Building and Testing - -You can now try to build Hello. Of course, you could do `nix-env -f . -iA -hello`, but you may not want to install a possibly broken package just -yet. The best way to test the package is by using the command -`nix-build`, which builds a Nix expression and creates a symlink named -`result` in the current directory: - -```console -$ nix-build -A hello -building path `/nix/store/632d2b22514d...-hello-2.1.1' -hello-2.1.1/ -hello-2.1.1/intl/ -hello-2.1.1/intl/ChangeLog -... - -$ ls -l result -lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1 - -$ ./result/bin/hello -Hello, world! -``` - -The `-A` option selects the `hello` attribute. This is faster than -using the symbolic package name specified by the `name` attribute -(which also happens to be `hello`) and is unambiguous (there can be -multiple packages with the symbolic name `hello`, but there can be -only one attribute in a set named `hello`). - -`nix-build` registers the `./result` symlink as a garbage collection -root, so unless and until you delete the `./result` symlink, the output -of the build will be safely kept on your system. You can use -`nix-build`’s `-o` switch to give the symlink another name. - -Nix has transactional semantics. Once a build finishes successfully, Nix -makes a note of this in its database: it registers that the path denoted -by `out` is now “valid”. If you try to build the derivation again, Nix -will see that the path is already valid and finish immediately. If a -build fails, either because it returns a non-zero exit code, because Nix -or the builder are killed, or because the machine crashes, then the -output paths will not be registered as valid. If you try to build the -derivation again, Nix will remove the output paths if they exist (e.g., -because the builder died half-way through `make -install`) and try again. Note that there is no “negative caching”: Nix -doesn't remember that a build failed, and so a failed build can always -be repeated. This is because Nix cannot distinguish between permanent -failures (e.g., a compiler error due to a syntax error in the source) -and transient failures (e.g., a disk full condition). - -Nix also performs locking. If you run multiple Nix builds -simultaneously, and they try to build the same derivation, the first Nix -instance that gets there will perform the build, while the others block -(or perform other derivations if available) until the build finishes: - -```console -$ nix-build -A hello -waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x' -``` - -So it is always safe to run multiple instances of Nix in parallel (which -isn’t the case with, say, `make`). diff --git a/doc/manual/src/expressions/simple-expression.md b/doc/manual/src/expressions/simple-expression.md deleted file mode 100644 index 857f71b9b..000000000 --- a/doc/manual/src/expressions/simple-expression.md +++ /dev/null @@ -1,23 +0,0 @@ -# A Simple Nix Expression - -This section shows how to add and test the [GNU Hello -package](http://www.gnu.org/software/hello/hello.html) to the Nix -Packages collection. Hello is a program that prints out the text “Hello, -world\!”. - -To add a package to the Nix Packages collection, you generally need to -do three things: - -1. Write a Nix expression for the package. This is a file that - describes all the inputs involved in building the package, such as - dependencies, sources, and so on. - -2. Write a *builder*. This is a shell script that builds the package - from the inputs. (In fact, it can be written in any language, but - typically it's a `bash` shell script.) - -3. Add the package to the file `pkgs/top-level/all-packages.nix`. The - Nix expression written in the first step is a *function*; it - requires other packages in order to build it. In this step you put - it all together, i.e., you call the function with the right - arguments to build the actual package. diff --git a/doc/manual/src/expressions/writing-nix-expressions.md b/doc/manual/src/expressions/writing-nix-expressions.md deleted file mode 100644 index 5664108e7..000000000 --- a/doc/manual/src/expressions/writing-nix-expressions.md +++ /dev/null @@ -1,12 +0,0 @@ -This chapter shows you how to write Nix expressions, which instruct Nix -how to build packages. It starts with a simple example (a Nix expression -for GNU Hello), and then moves on to a more in-depth look at the Nix -expression language. - -> **Note** -> -> This chapter is mostly about the Nix expression language. For more -> extensive information on adding packages to the Nix Packages -> collection (such as functions in the standard environment and coding -> conventions), please consult [its -> manual](http://nixos.org/nixpkgs/manual/). diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 3448b971b..aa0ac78cb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -3,7 +3,7 @@ - [derivation]{#gloss-derivation}\ A description of a build action. The result of a derivation is a store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](expressions/derivations.md). These are + using the [`derivation` primitive](language/derivations.md). These are translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 9fb9c80c3..2d007ca1b 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -13,7 +13,7 @@ for your platform: - multi-user on macOS > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > + > > - It took some time to support this cleanly. You may see posts, > examples, and tutorials using obsolete workarounds. > - Supporting it cleanly made macOS installs too complex to qualify @@ -31,8 +31,8 @@ $ sh <(curl -L https://nixos.org/nix/install) --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You should run this under your usual user -account, *not* as root. The script will invoke `sudo` to create `/nix` +is owned by the invoking user. You can run this under your usual user +account or root. The script will invoke `sudo` to create `/nix` if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: @@ -71,11 +71,11 @@ $ sh <(curl -L https://nixos.org/nix/install) --daemon The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. You -should run this under your usual user account, *not* as root. The script +can run this under your usual user account or root. The script will invoke `sudo` as needed. > **Note** -> +> > If you need Nix to use a different group ID or user ID set, you will > have to download the tarball manually and [edit the install > script](#installing-from-a-binary-tarball). @@ -148,7 +148,8 @@ and `/etc/zshrc` which you may remove. This will remove all the build users that no longer serve a purpose. 4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like this, + volume on `/nix`, which looks like + `UUID=<uuid> /nix apfs rw,noauto,nobrowse,suid,owners` or `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic mounting of the Nix Store volume. @@ -167,7 +168,7 @@ and `/etc/zshrc` which you may remove. removed next. 7. Remove the Nix Store volume: - + ```console sudo diskutil apfs deleteVolume /nix ``` @@ -175,8 +176,20 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. + If the output indicates that the command couldn't remove the volume, you should + make sure you don't have an _unmounted_ Nix Store volume. Look for a + "Nix Store" volume in the output of the following command: + + ```console + diskutil list + ``` + + If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil + deleteVolume command, but replace `/nix` with the store volume's `diskXsY` + identifier. + > **Note** -> +> > After you complete the steps here, you will still have an empty `/nix` > directory. This is an expected sign of a successful uninstall. The empty > `/nix` directory will disappear the next time you reboot. @@ -191,8 +204,7 @@ and `/etc/zshrc` which you may remove. <!-- Note: anchors above to catch permalinks to old explanations --> We believe we have ironed out how to cleanly support the read-only root -on modern macOS. New installs will do this automatically, and you can -also re-run a new installer to convert your existing setup. +on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, but it now only outlines what the installer does. You don't need to know diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2e7e80ed0..2e7e80ed0 100644 --- a/doc/manual/src/expressions/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md diff --git a/doc/manual/src/expressions/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index 78d066a82..78d066a82 100644 --- a/doc/manual/src/expressions/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index c631a8453..c631a8453 100644 --- a/doc/manual/src/expressions/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md diff --git a/doc/manual/src/expressions/builtins-suffix.md b/doc/manual/src/language/builtins-suffix.md index a74db2857..a74db2857 100644 --- a/doc/manual/src/expressions/builtins-suffix.md +++ b/doc/manual/src/language/builtins-suffix.md diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/language/constructs.md index 1c01f2cc7..1c01f2cc7 100644 --- a/doc/manual/src/expressions/language-constructs.md +++ b/doc/manual/src/language/constructs.md diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/language/derivations.md index 3391ec0d8..3391ec0d8 100644 --- a/doc/manual/src/expressions/derivations.md +++ b/doc/manual/src/language/derivations.md diff --git a/doc/manual/src/expressions/expression-language.md b/doc/manual/src/language/index.md index 267fcb983..c4b3abf75 100644 --- a/doc/manual/src/expressions/expression-language.md +++ b/doc/manual/src/language/index.md @@ -1,6 +1,6 @@ -# Nix Expression Language +# Nix Language -The Nix expression language is a pure, lazy, functional language. Purity +The Nix language is a pure, lazy, functional language. Purity means that operations in the language don't have side-effects (for instance, there is no variable assignment). Laziness means that arguments to functions are evaluated only when they are needed. @@ -10,3 +10,4 @@ full-featured, general purpose language. Its main job is to describe packages, compositions of packages, and the variability within packages. This section presents the various features of the language. + diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/language/operators.md index 268b44f4c..32398189d 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/language/operators.md @@ -1,6 +1,6 @@ # Operators -The table below lists the operators in the Nix expression language, in +The table below lists the operators in the Nix language, in order of precedence (from strongest to weakest binding). | Name | Syntax | Associativity | Description | Precedence | diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md new file mode 100644 index 000000000..f09400d02 --- /dev/null +++ b/doc/manual/src/language/values.md @@ -0,0 +1,261 @@ +# Data Types + +## Primitives + +- <a id="type-string" href="#type-string">String</a> + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, + e.g., `"foo bar"`. Strings can span multiple lines. The special + characters `"` and `\` and the character sequence `${` must be + escaped by prefixing them with a backslash (`\`). Newlines, carriage + returns and tabs can be written as `\n`, `\r` and `\t`, + respectively. + + You can include the result of an expression into a string by + enclosing it in `${...}`, a feature known as *antiquotation*. The + enclosed expression must evaluate to something that can be coerced + into a string (meaning that it must be a string, a path, or a + derivation). For instance, rather than writing + + ```nix + "--with-freetype2-library=" + freetype + "/lib" + ``` + + (where `freetype` is a derivation), you can instead write the more + natural + + ```nix + "--with-freetype2-library=${freetype}/lib" + ``` + + The latter is automatically translated to the former. A more + complicated example (from the Nix expression for + [Qt](http://www.trolltech.com/products/qt)): + + ```nix + configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} + "; + ``` + + Note that Nix expressions and strings can be arbitrarily nested; in + this case the outer string contains various antiquotations that + themselves contain strings (e.g., `"-thread"`), some of which in + turn contain expressions (e.g., `${mesa}`). + + The second way to write string literals is as an *indented string*, + which is enclosed between pairs of *double single-quotes*, like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + Note that the whitespace and newline following the opening `''` is + ignored if there is no non-whitespace text on the initial line. + + Antiquotation (`${expr}`) is supported in indented strings. + + Since `${` and `''` have special meaning in indented strings, you + need a way to quote them. `$` can be escaped by prefixing it with + `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped + by prefixing it with `'`, i.e., `'''`. `$` removes any special + meaning from the following `$`. Linefeed, carriage-return and tab + characters can be written as `''\n`, `''\r`, `''\t`, and `''\` + escapes any other character. + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +- <a id="type-number" href="#type-number">Number</a> + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + Numbers are type-compatible: pure integer operations will always + return integers, whereas any operation involving at least one + floating point number will have a floating point number as a result. + +- <a id="type-path" href="#type-path">Path</a> + + *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at + least one slash to be recognised as such. For instance, `builder.sh` + is not a path: it's parsed as an expression that selects the + attribute `sh` from the variable `builder`. If the file name is + relative, i.e., if it does not begin with a slash, it is made + absolute at parse time relative to the directory of the Nix + expression that contained it. For instance, if a Nix expression in + `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path + is `/foo/xyzzy/fnord.nix`. + + If the first component of a path is a `~`, it is interpreted as if + the rest of the path were relative to the user's home directory. + e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user + whose home directory is `/home/edolstra`. + + Paths can also be specified between angle brackets, e.g. + `<nixpkgs>`. This means that the directories listed in the + environment variable `NIX_PATH` will be searched for the given file + or directory name. + + Antiquotation is supported in any paths except those in angle brackets. + `./${foo}-${bar}.nix` is a more convenient way of writing + `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At + least one slash must appear *before* any antiquotations for this to be + recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division + operation. `./a.${foo}/b.${bar}` is a path. + +- <a id="type-boolean" href="#type-boolean">Boolean</a> + + *Booleans* with values `true` and `false`. + +- <a id="type-null" href="#type-null">Null</a> + + The null value, denoted as `null`. + +## List + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +## Attribute Set + +An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). + +Names and values are separated by an equal sign (`=`). +Each value is an arbitrary expression terminated by a semicolon (`;`). + +Attributes can appear in any order. +An attribute name may only occur once. + +Example: + +```nix +{ + x = 123; + text = "Hello"; + y = f { bla = 456; }; +} +``` + +This defines a set with attributes named `x`, `text`, `y`. + +Attributes can be selected from a set using the `.` operator. For +instance, + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +evaluates to `"Foo"`. It is possible to provide a default value in an +attribute selection using the `or` keyword. For example, + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +will evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}" +``` + +This will evaluate to `123` (Assuming `bar` is antiquotable). In the +case where an attribute name is just a single antiquotation, the quotes +can be dropped: + +```nix +{ foo = 123; }.${bar} or 456 +``` + +This will evaluate to `123` if `bar` evaluates to `"foo"` when coerced +to a string and `456` otherwise (again assuming `bar` is antiquotable). + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` is not +antiquotable), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/package-management.md index bd26a09ab..d528112e2 100644 --- a/doc/manual/src/package-management/package-management.md +++ b/doc/manual/src/package-management/package-management.md @@ -1,5 +1,4 @@ This chapter discusses how to do package management with Nix, i.e., how to obtain, install, upgrade, and erase packages. This is the “user’s” perspective of the Nix system — people who want to *create* -packages should consult the [chapter on writing Nix -expressions](../expressions/writing-nix-expressions.md). +packages should consult the chapter on the [Nix language](../language/index.md). diff --git a/docker.nix b/docker.nix index ddf6feff5..8e6aa227f 100644 --- a/docker.nix +++ b/docker.nix @@ -6,6 +6,7 @@ , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] , maxLayers ? 100 +, nixConf ? {} }: let defaultPkgs = with pkgs; [ @@ -123,12 +124,17 @@ let (lib.attrValues (lib.mapAttrs groupToGroup groups)) ); - nixConf = { + defaultNixConf = { sandbox = "false"; build-users-group = "nixbld"; - trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; }; - nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: + let + vStr = if builtins.isList v then lib.concatStringsSep " " v else v; + in + "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n"; baseSystem = let diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..01dbf0c0e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -59,6 +59,30 @@ headless() { fi } +is_root() { + if [ "$EUID" -eq 0 ]; then + return 0 + else + return 1 + fi +} + +is_os_linux() { + if [ "$(uname -s)" = "Linux" ]; then + return 0 + else + return 1 + fi +} + +is_os_darwin() { + if [ "$(uname -s)" = "Darwin" ]; then + return 0 + else + return 1 + fi +} + contact_us() { echo "You can open an issue at https://github.com/nixos/nix/issues" echo "" @@ -313,14 +337,23 @@ __sudo() { _sudo() { local expl="$1" shift - if ! headless; then + if ! headless || is_root; then __sudo "$expl" "$*" >&2 fi - sudo "$@" + + if is_root; then + env "$@" + else + sudo "$@" + fi } +# Ensure that $TMPDIR exists if defined. +if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then + mkdir -m 0700 -p "${TMPDIR:-}" +fi -readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" } @@ -423,7 +456,7 @@ EOF fi done - if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then + if is_os_linux && [ ! -e /run/systemd/system ]; then warning <<EOF We did not detect systemd on your system. With a multi-user install without systemd you will have to manually configure your init system to @@ -640,7 +673,7 @@ place_channel_configuration() { check_selinux() { if command -v getenforce > /dev/null 2>&1; then - if ! [ "$(getenforce)" = "Disabled" ]; then + if [ "$(getenforce)" = "Enforcing" ]; then failure <<EOF Nix does not work with selinux enabled yet! see https://github.com/NixOS/nix/issues/2374 @@ -865,24 +898,14 @@ EOF install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf } -main() { - # TODO: I've moved this out of validate_starting_assumptions so we - # can fail faster in this case. Sourcing install-darwin... now runs - # `touch /` to detect Read-only root, but it could update times on - # pre-Catalina macOS if run as root user. - if [ "$EUID" -eq 0 ]; then - failure <<EOF -Please do not run this script with root privileges. I will call sudo -when I need to. -EOF - fi +main() { check_selinux - if [ "$(uname -s)" = "Darwin" ]; then + if is_os_darwin; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ]; then + elif is_os_linux; then # shellcheck source=./install-systemd-multi-user.sh . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also else @@ -890,7 +913,10 @@ EOF fi welcome_to_nix - chat_about_sudo + + if ! is_root; then + chat_about_sudo + fi cure_artifacts # TODO: there's a tension between cure and validate. I moved the diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 59162c4df..e097f23b3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -616,6 +616,8 @@ InstallableFlake::InstallableFlake( std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() { + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + auto attr = getCursor(*state); auto attrPath = attr->getAttrPathStr(); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 23df40337..150bd42ac 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -35,6 +35,7 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" +#include "progress-bar.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -252,6 +253,10 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif + /* Stop the progress bar because it interferes with the display of + the repl. */ + stopProgressBar(); + std::string input; while (true) { @@ -1037,9 +1042,10 @@ void runRepl( struct CmdRepl : InstallablesCommand { - CmdRepl(){ + CmdRepl() { evalSettings.pureEval = false; } + void prepare() { if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { @@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand } installables = InstallablesCommand::load(); } + std::vector<std::string> files; + Strings getDefaultFlakeAttrPaths() override { return {""}; } - virtual bool useDefaultInstallables() override + + bool useDefaultInstallables() override { return file.has_value() or expr.has_value(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f485e2fed..e3716f217 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2501,18 +2501,18 @@ void EvalState::printStats() } { auto list = topObj.list("functions"); - for (auto & i : functionCalls) { + for (auto & [fun, count] : functionCalls) { auto obj = list.object(); - if (i.first->name) - obj.attr("name", (const std::string &) i.first->name); + if (fun->name) + obj.attr("name", (std::string_view) symbols[fun->name]); else obj.attr("name", nullptr); - if (auto pos = positions[i.first->pos]) { - obj.attr("file", (const std::string &) pos.file); + if (auto pos = positions[fun->pos]) { + obj.attr("file", (std::string_view) pos.file); obj.attr("line", pos.line); obj.attr("column", pos.column); } - obj.attr("count", i.second); + obj.attr("count", count); } } { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index cc9be1336..105e76bc6 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -341,7 +341,6 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - // FIXME: check whether all overrides are used. std::map<InputPath, FlakeInput> overrides; std::set<InputPath> overridesUsed, updatesUsed; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a9182f4bf..fe4f67193 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -28,7 +28,7 @@ typedef std::string FlakeId; * object that fetcher generates (usually via * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). * - * The actual fetch not have been performed yet (i.e. a FlakeRef may + * The actual fetch may not have been performed yet (i.e. a FlakeRef may * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d01aaa7a..c1a21e764 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f4306ab91..5183f212f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -8,6 +8,7 @@ #include <map> #include <thread> #include <iostream> +#include <chrono> namespace nix { @@ -48,6 +49,7 @@ private: bool visible = true; ActivityId parent; std::optional<std::string> name; + std::chrono::time_point<std::chrono::steady_clock> startTime; }; struct ActivitiesByType @@ -91,10 +93,11 @@ public: state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); + auto nextWakeup = std::chrono::milliseconds::max(); while (state->active) { if (!state->haveUpdate) - state.wait(updateCV); - draw(*state); + state.wait_for(updateCV, nextWakeup); + nextWakeup = draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } }); @@ -118,7 +121,8 @@ public: updateThread.join(); } - bool isVerbose() override { + bool isVerbose() override + { return printBuildLogs; } @@ -159,11 +163,13 @@ public: if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); - state->activities.emplace_back(ActInfo()); + state->activities.emplace_back(ActInfo { + .s = s, + .type = type, + .parent = parent, + .startTime = std::chrono::steady_clock::now() + }); auto i = std::prev(state->activities.end()); - i->s = s; - i->type = type; - i->parent = parent; state->its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i); @@ -327,10 +333,12 @@ public: updateCV.notify_one(); } - void draw(State & state) + std::chrono::milliseconds draw(State & state) { + auto nextWakeup = std::chrono::milliseconds::max(); + state.haveUpdate = false; - if (!state.active) return; + if (!state.active) return nextWakeup; std::string line; @@ -341,12 +349,25 @@ public: line += "]"; } + auto now = std::chrono::steady_clock::now(); + if (!state.activities.empty()) { if (!status.empty()) line += " "; auto i = state.activities.rbegin(); - while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + while (i != state.activities.rend()) { + if (i->visible && (!i->s.empty() || !i->lastLine.empty())) { + /* Don't show activities until some time has + passed, to avoid displaying very short + activities. */ + auto delay = std::chrono::milliseconds(10); + if (i->startTime + delay < now) + break; + else + nextWakeup = std::min(nextWakeup, std::chrono::duration_cast<std::chrono::milliseconds>(delay - (now - i->startTime))); + } ++i; + } if (i != state.activities.rend()) { line += i->s; @@ -366,6 +387,8 @@ public: if (width <= 0) width = std::numeric_limits<decltype(width)>::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + + return nextWakeup; } std::string getStatus(State & state) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..83da657f0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -914,12 +913,6 @@ void DerivationGoal::buildDone() outputPaths ); - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built, std::move(builtOutputs)); - return; - } - cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d1ec91ed5..18b682e13 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); + renameFile((chrootRootDir + p), p); } return diskFull; @@ -845,18 +844,43 @@ void LocalDerivationGoal::startBuilder() /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. + * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 */ usingUserNamespace = false; flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - + if (child == -1) { + switch(errno) { + case EPERM: + case EINVAL: { + int errno_ = errno; + if (!userNamespacesEnabled && errno==EPERM) + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + if (userNamespacesEnabled) { + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + } + } + Path procSelfNsUser = "/proc/self/ns/user"; + if (!pathExists(procSelfNsUser)) + notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); + /* Mention sandbox-fallback in the error message so the user + knows that having it disabled contributed to the + unrecoverability of this failure */ + throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); + } + default: + throw SysError("creating sandboxed builder process using clone()"); + } + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); @@ -2350,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() if (*scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - rsink2(sink.s); + dumpPath(actualPath, rsink2); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; @@ -2600,8 +2622,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); + renameFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 426d58a53..ba04bb16c 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) - throw SysError("renaming channel directory"); + renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8454ad7d2..252403cb5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d58ed78b1..4c1a82279 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError("cannot rename '%1%' to '%2%'", - tempLink , link); + renameFile(tempLink, link); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0f2ca4b15..d724897bb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -114,7 +114,13 @@ std::vector<Path> getUserConfigFiles() unsigned int Settings::getDefaultCores() { - return std::max(1U, std::thread::hardware_concurrency()); + const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int maxCPU = getMaxCPU(); + + if (maxCPU > 0) + return maxCPU; + else + return concurrency; } StringSet Settings::getDefaultSystemFeatures() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..e9d721e59 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -746,6 +746,13 @@ public: /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting<unsigned int> downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maximum transfer rate in kilobytes per second you want + Nix to use for downloads. + )"}; + Setting<std::string> netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ba4416f6d..f20b1fa02 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,8 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - if (rename(tmp.c_str(), path2.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path2); + renameFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eba3b0fa5..a272e4301 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - if (rename(tmpFile.c_str(), logPath.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); + renameFile(tmpFile, logPath); } std::optional<std::string> LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8af9b1dde..4d2781180 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { + try { + renameFile(tempLink, path); + } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { @@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); + throw; } stats.filesLinked++; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ad2e5c18a..96a29155c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -580,7 +580,6 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( try { conn->to.written = 0; - conn->to.warn = true; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); @@ -591,7 +590,6 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( dumpString(contents, conn->to); } } - conn->to.warn = false; conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a53e9802e..3d1479c54 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -204,13 +204,19 @@ public: int errNo; template<typename... Args> - SysError(const Args & ... args) + SysError(int errNo_, const Args & ... args) : Error("") { - errNo = errno; + errNo = errNo_; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } + + template<typename... Args> + SysError(const Args & ... args) + : SysError(errno, args ...) + { + } }; } diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc new file mode 100644 index 000000000..403389e60 --- /dev/null +++ b/src/libutil/filesystem.cc @@ -0,0 +1,172 @@ +#include <sys/time.h> +#include <filesystem> + +#include "finally.hh" +#include "util.hh" +#include "types.hh" + +namespace fs = std::filesystem; + +namespace nix { + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } +} + +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp, mtime); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +} diff --git a/src/libutil/json.cc b/src/libutil/json.cc index b0a5d7e75..2f9e97ff5 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -6,7 +6,8 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end) +template<> +void toJSON<std::string_view>(std::ostream & str, const std::string_view & s) { constexpr size_t BUF_SIZE = 4096; char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts @@ -21,7 +22,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) }; put('"'); - for (auto i = start; i != end; i++) { + for (auto i = s.begin(); i != s.end(); i++) { if (bufPos >= BUF_SIZE) flush(); if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } else if (*i == '\n') { put('\\'); put('n'); } @@ -44,7 +45,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) void toJSON(std::ostream & str, const char * s) { - if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); + if (!s) str << "null"; else toJSON(str, std::string_view(s)); } template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; } @@ -55,11 +56,7 @@ template<> void toJSON<long long>(std::ostream & str, const long long & n) { str template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; } template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; } - -template<> void toJSON<std::string>(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} +template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); } template<> void toJSON<bool>(std::ostream & str, const bool & b) { @@ -154,7 +151,7 @@ JSONObject::~JSONObject() } } -void JSONObject::attr(const std::string & s) +void JSONObject::attr(std::string_view s) { comma(); toJSON(state->str, s); @@ -162,19 +159,19 @@ void JSONObject::attr(const std::string & s) if (state->indent) state->str << ' '; } -JSONList JSONObject::list(const std::string & name) +JSONList JSONObject::list(std::string_view name) { attr(name); return JSONList(state); } -JSONObject JSONObject::object(const std::string & name) +JSONObject JSONObject::object(std::string_view name) { attr(name); return JSONObject(state); } -JSONPlaceholder JSONObject::placeholder(const std::string & name) +JSONPlaceholder JSONObject::placeholder(std::string_view name) { attr(name); return JSONPlaceholder(state); @@ -196,7 +193,11 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + if (first) { + assert(std::uncaught_exceptions()); + if (state->stack != 0) + write(nullptr); + } } } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 83213ca66..3790b1a2e 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -6,7 +6,6 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end); void toJSON(std::ostream & str, const char * s); template<typename T> @@ -107,7 +106,7 @@ private: open(); } - void attr(const std::string & s); + void attr(std::string_view s); public: @@ -128,18 +127,18 @@ public: ~JSONObject(); template<typename T> - JSONObject & attr(const std::string & name, const T & v) + JSONObject & attr(std::string_view name, const T & v) { attr(name); toJSON(state->str, v); return *this; } - JSONList list(const std::string & name); + JSONList list(std::string_view name); - JSONObject object(const std::string & name); + JSONObject object(std::string_view name); - JSONPlaceholder placeholder(const std::string & name); + JSONPlaceholder placeholder(std::string_view name); }; class JSONPlaceholder : JSONWriter diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8ff904583..2c3597775 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -48,24 +48,9 @@ FdSink::~FdSink() } -size_t threshold = 256 * 1024 * 1024; - -static void warnLargeDump() -{ - warn("dumping very large path (> 256 MiB); this may run out of memory"); -} - - void FdSink::write(std::string_view data) { written += data.size(); - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } try { writeFull(fd, data); } catch (SysError & e) { @@ -448,11 +433,6 @@ Error readError(Source & source) void StringSink::operator () (std::string_view data) { - static bool warned = false; - if (!warned && s.size() > threshold) { - warnLargeDump(); - warned = true; - } s.append(data); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 13da26c6a..84847835a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -97,19 +97,17 @@ protected: struct FdSink : BufferedSink { int fd; - bool warn = false; size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink && s) + FdSink & operator=(FdSink && s) { flush(); fd = s.fd; s.fd = -1; - warn = s.warn; written = s.written; return *this; } diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc index dea73f53a..156286999 100644 --- a/src/libutil/tests/json.cc +++ b/src/libutil/tests/json.cc @@ -102,8 +102,8 @@ namespace nix { TEST(toJSON, substringEscape) { std::stringstream out; - const char *s = "foo\t"; - toJSON(out, s+3, s + strlen(s)); + std::string_view s = "foo\t"; + toJSON(out, s.substr(3)); ASSERT_EQ(out.str(), "\"\\t\""); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 28df30fef..96ac11ea2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -35,6 +35,9 @@ #ifdef __linux__ #include <sys/prctl.h> #include <sys/resource.h> + +#include <mntent.h> +#include <cmath> #endif @@ -505,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed) } -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -574,6 +522,7 @@ Path getHome() { static Path homeDir = []() { + std::optional<std::string> unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); if (homeDir) { // Only use $HOME if doesn't exist or is owned by the current user. @@ -585,8 +534,7 @@ Path getHome() homeDir.reset(); } } else if (st.st_uid != geteuid()) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); - homeDir.reset(); + unownedUserHomeDir.swap(homeDir); } } if (!homeDir) { @@ -597,6 +545,9 @@ Path getHome() || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } } return *homeDir; }(); @@ -678,44 +629,6 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - - -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); - - break; - } -} - - void readFull(int fd, char * buf, size_t count) { while (count) { @@ -788,7 +701,55 @@ void drainFD(int fd, Sink & sink, bool block) } } +////////////////////////////////////////////////////////////////////// + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return 0; + + Strings cgPathParts; + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString<Strings>(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector<std::string> cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } + } + } catch (Error &) { ignoreException(); } + #endif + + return 0; +} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d3ed15b0b..cd83f250f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,6 +168,17 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional<time_t> mtime = {}); +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ @@ -182,6 +193,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); +/* If cgroups are active, attempt to calculate the number of CPUs available. + If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +unsigned int getMaxCPU(); /* Automatic cleanup of resources. */ diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..fdd66220a 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -940,12 +940,12 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin JSONObject metaObj = pkgObj.object("meta"); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); + metaObj.attr(j, nullptr); } else { + auto placeholder = metaObj.placeholder(j); PathSet context; printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 2bb70711f..a18161a3c 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -44,7 +44,7 @@ flake output attributes: * `bundlers.<system>.default` -If an attribute *name* is given, `nix run` tries the following flake +If an attribute *name* is given, `nix bundle` tries the following flake output attributes: * `bundlers.<system>.<name>` diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 03b50e38e..2ee8a707d 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -6,7 +6,7 @@ R""( lock file: ```console - # nix flake update + # nix flake update --commit-lock-file * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' … diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e01bc6d10..3967f1102 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -212,7 +212,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + if (!lockedFlake.lockFile.root->inputs.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set<std::shared_ptr<Node>> visited; diff --git a/src/nix/profile.md b/src/nix/profile.md index 8dade051d..be3c5ba1a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`/nix/var/nix/profiles/default` if Nix is invoked by the `root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. You can specify another profile location using `--profile` *path*. diff --git a/tests/check.sh b/tests/check.sh index ab48ff865..495202781 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -40,6 +40,14 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix deterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log +if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi +# Repeat is set to 1, ie. nix should build deterministic twice. +if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log @@ -50,6 +58,12 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix nondeterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log diff --git a/tests/common.sh.in b/tests/common.sh.in index 79da10199..73c2d2309 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -193,7 +193,7 @@ fi onError() { set +x echo "$0: test failed at:" >&2 - for ((i = 1; i < 16; i++)); do + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 done |