diff options
175 files changed, 18510 insertions, 0 deletions
diff --git a/ascii-table.nix b/ascii-table.nix new file mode 100644 index 000000000..74989936e --- /dev/null +++ b/ascii-table.nix @@ -0,0 +1,99 @@ +{ "\t" = 9; + "\n" = 10; + "\r" = 13; + " " = 32; + "!" = 33; + "\"" = 34; + "#" = 35; + "$" = 36; + "%" = 37; + "&" = 38; + "'" = 39; + "(" = 40; + ")" = 41; + "*" = 42; + "+" = 43; + "," = 44; + "-" = 45; + "." = 46; + "/" = 47; + "0" = 48; + "1" = 49; + "2" = 50; + "3" = 51; + "4" = 52; + "5" = 53; + "6" = 54; + "7" = 55; + "8" = 56; + "9" = 57; + ":" = 58; + ";" = 59; + "<" = 60; + "=" = 61; + ">" = 62; + "?" = 63; + "@" = 64; + "A" = 65; + "B" = 66; + "C" = 67; + "D" = 68; + "E" = 69; + "F" = 70; + "G" = 71; + "H" = 72; + "I" = 73; + "J" = 74; + "K" = 75; + "L" = 76; + "M" = 77; + "N" = 78; + "O" = 79; + "P" = 80; + "Q" = 81; + "R" = 82; + "S" = 83; + "T" = 84; + "U" = 85; + "V" = 86; + "W" = 87; + "X" = 88; + "Y" = 89; + "Z" = 90; + "[" = 91; + "\\" = 92; + "]" = 93; + "^" = 94; + "_" = 95; + "`" = 96; + "a" = 97; + "b" = 98; + "c" = 99; + "d" = 100; + "e" = 101; + "f" = 102; + "g" = 103; + "h" = 104; + "i" = 105; + "j" = 106; + "k" = 107; + "l" = 108; + "m" = 109; + "n" = 110; + "o" = 111; + "p" = 112; + "q" = 113; + "r" = 114; + "s" = 115; + "t" = 116; + "u" = 117; + "v" = 118; + "w" = 119; + "x" = 120; + "y" = 121; + "z" = 122; + "{" = 123; + "|" = 124; + "}" = 125; + "~" = 126; +} diff --git a/asserts.nix b/asserts.nix new file mode 100644 index 000000000..98e0b490a --- /dev/null +++ b/asserts.nix @@ -0,0 +1,53 @@ +{ lib }: + +rec { + + /* Throw if pred is false, else return pred. + Intended to be used to augment asserts with helpful error messages. + + Example: + assertMsg false "nope" + stderr> error: nope + + assert assertMsg ("foo" == "bar") "foo is not bar, silly"; "" + stderr> error: foo is not bar, silly + + Type: + assertMsg :: Bool -> String -> Bool + */ + # TODO(Profpatsch): add tests that check stderr + assertMsg = + # Predicate that needs to succeed, otherwise `msg` is thrown + pred: + # Message to throw in case `pred` fails + msg: + pred || builtins.throw msg; + + /* Specialized `assertMsg` for checking if `val` is one of the elements + of the list `xs`. Useful for checking enums. + + Example: + let sslLibrary = "libressl"; + in assertOneOf "sslLibrary" sslLibrary [ "openssl" "bearssl" ] + stderr> error: sslLibrary must be one of [ + stderr> "openssl" + stderr> "bearssl" + stderr> ], but is: "libressl" + + Type: + assertOneOf :: String -> ComparableVal -> List ComparableVal -> Bool + */ + assertOneOf = + # The name of the variable the user entered `val` into, for inclusion in the error message + name: + # The value of what the user provided, to be compared against the values in `xs` + val: + # The list of valid values + xs: + assertMsg + (lib.elem val xs) + "${name} must be one of ${ + lib.generators.toPretty {} xs}, but is: ${ + lib.generators.toPretty {} val}"; + +} diff --git a/attrsets.nix b/attrsets.nix new file mode 100644 index 000000000..73b71f68d --- /dev/null +++ b/attrsets.nix @@ -0,0 +1,1018 @@ +{ lib }: +# Operations on attribute sets. + +let + inherit (builtins) head tail length; + inherit (lib.trivial) flip id mergeAttrs pipe; + inherit (lib.strings) concatStringsSep concatMapStringsSep escapeNixIdentifier sanitizeDerivationName; + inherit (lib.lists) foldr foldl' concatMap concatLists elemAt all partition groupBy take foldl; +in + +rec { + inherit (builtins) attrNames listToAttrs hasAttr isAttrs getAttr removeAttrs; + + + /* Return an attribute from nested attribute sets. + + Example: + x = { a = { b = 3; }; } + # ["a" "b"] is equivalent to x.a.b + # 6 is a default value to return if the path does not exist in attrset + attrByPath ["a" "b"] 6 x + => 3 + attrByPath ["z" "z"] 6 x + => 6 + + Type: + attrByPath :: [String] -> Any -> AttrSet -> Any + + */ + attrByPath = + # A list of strings representing the attribute path to return from `set` + attrPath: + # Default value if `attrPath` does not resolve to an existing value + default: + # The nested attribute set to select values from + set: + let attr = head attrPath; + in + if attrPath == [] then set + else if set ? ${attr} + then attrByPath (tail attrPath) default set.${attr} + else default; + + /* Return if an attribute from nested attribute set exists. + + Example: + x = { a = { b = 3; }; } + hasAttrByPath ["a" "b"] x + => true + hasAttrByPath ["z" "z"] x + => false + + Type: + hasAttrByPath :: [String] -> AttrSet -> Bool + */ + hasAttrByPath = + # A list of strings representing the attribute path to check from `set` + attrPath: + # The nested attribute set to check + e: + let attr = head attrPath; + in + if attrPath == [] then true + else if e ? ${attr} + then hasAttrByPath (tail attrPath) e.${attr} + else false; + + + /* Create a new attribute set with `value` set at the nested attribute location specified in `attrPath`. + + Example: + setAttrByPath ["a" "b"] 3 + => { a = { b = 3; }; } + + Type: + setAttrByPath :: [String] -> Any -> AttrSet + */ + setAttrByPath = + # A list of strings representing the attribute path to set + attrPath: + # The value to set at the location described by `attrPath` + value: + let + len = length attrPath; + atDepth = n: + if n == len + then value + else { ${elemAt attrPath n} = atDepth (n + 1); }; + in atDepth 0; + + /* Like `attrByPath`, but without a default value. If it doesn't find the + path it will throw an error. + + Example: + x = { a = { b = 3; }; } + getAttrFromPath ["a" "b"] x + => 3 + getAttrFromPath ["z" "z"] x + => error: cannot find attribute `z.z' + + Type: + getAttrFromPath :: [String] -> AttrSet -> Any + */ + getAttrFromPath = + # A list of strings representing the attribute path to get from `set` + attrPath: + # The nested attribute set to find the value in. + set: + let errorMsg = "cannot find attribute `" + concatStringsSep "." attrPath + "'"; + in attrByPath attrPath (abort errorMsg) set; + + /* Map each attribute in the given set and merge them into a new attribute set. + + Type: + concatMapAttrs :: (String -> a -> AttrSet) -> AttrSet -> AttrSet + + Example: + concatMapAttrs + (name: value: { + ${name} = value; + ${name + value} = value; + }) + { x = "a"; y = "b"; } + => { x = "a"; xa = "a"; y = "b"; yb = "b"; } + */ + concatMapAttrs = f: v: + foldl' mergeAttrs { } + (attrValues + (mapAttrs f v) + ); + + + /* Update or set specific paths of an attribute set. + + Takes a list of updates to apply and an attribute set to apply them to, + and returns the attribute set with the updates applied. Updates are + represented as `{ path = ...; update = ...; }` values, where `path` is a + list of strings representing the attribute path that should be updated, + and `update` is a function that takes the old value at that attribute path + as an argument and returns the new + value it should be. + + Properties: + + - Updates to deeper attribute paths are applied before updates to more + shallow attribute paths + + - Multiple updates to the same attribute path are applied in the order + they appear in the update list + + - If any but the last `path` element leads into a value that is not an + attribute set, an error is thrown + + - If there is an update for an attribute path that doesn't exist, + accessing the argument in the update function causes an error, but + intermediate attribute sets are implicitly created as needed + + Example: + updateManyAttrsByPath [ + { + path = [ "a" "b" ]; + update = old: { d = old.c; }; + } + { + path = [ "a" "b" "c" ]; + update = old: old + 1; + } + { + path = [ "x" "y" ]; + update = old: "xy"; + } + ] { a.b.c = 0; } + => { a = { b = { d = 1; }; }; x = { y = "xy"; }; } + + Type: updateManyAttrsByPath :: [{ path :: [String]; update :: (Any -> Any); }] -> AttrSet -> AttrSet + */ + updateManyAttrsByPath = let + # When recursing into attributes, instead of updating the `path` of each + # update using `tail`, which needs to allocate an entirely new list, + # we just pass a prefix length to use and make sure to only look at the + # path without the prefix length, so that we can reuse the original list + # entries. + go = prefixLength: hasValue: value: updates: + let + # Splits updates into ones on this level (split.right) + # And ones on levels further down (split.wrong) + split = partition (el: length el.path == prefixLength) updates; + + # Groups updates on further down levels into the attributes they modify + nested = groupBy (el: elemAt el.path prefixLength) split.wrong; + + # Applies only nested modification to the input value + withNestedMods = + # Return the value directly if we don't have any nested modifications + if split.wrong == [] then + if hasValue then value + else + # Throw an error if there is no value. This `head` call here is + # safe, but only in this branch since `go` could only be called + # with `hasValue == false` for nested updates, in which case + # it's also always called with at least one update + let updatePath = (head split.right).path; in + throw + ( "updateManyAttrsByPath: Path '${showAttrPath updatePath}' does " + + "not exist in the given value, but the first update to this " + + "path tries to access the existing value.") + else + # If there are nested modifications, try to apply them to the value + if ! hasValue then + # But if we don't have a value, just use an empty attribute set + # as the value, but simplify the code a bit + mapAttrs (name: go (prefixLength + 1) false null) nested + else if isAttrs value then + # If we do have a value and it's an attribute set, override it + # with the nested modifications + value // + mapAttrs (name: go (prefixLength + 1) (value ? ${name}) value.${name}) nested + else + # However if it's not an attribute set, we can't apply the nested + # modifications, throw an error + let updatePath = (head split.wrong).path; in + throw + ( "updateManyAttrsByPath: Path '${showAttrPath updatePath}' needs to " + + "be updated, but path '${showAttrPath (take prefixLength updatePath)}' " + + "of the given value is not an attribute set, so we can't " + + "update an attribute inside of it."); + + # We get the final result by applying all the updates on this level + # after having applied all the nested updates + # We use foldl instead of foldl' so that in case of multiple updates, + # intermediate values aren't evaluated if not needed + in foldl (acc: el: el.update acc) withNestedMods split.right; + + in updates: value: go 0 true value updates; + + /* Return the specified attributes from a set. + + Example: + attrVals ["a" "b" "c"] as + => [as.a as.b as.c] + + Type: + attrVals :: [String] -> AttrSet -> [Any] + */ + attrVals = + # The list of attributes to fetch from `set`. Each attribute name must exist on the attrbitue set + nameList: + # The set to get attribute values from + set: map (x: set.${x}) nameList; + + + /* Return the values of all attributes in the given set, sorted by + attribute name. + + Example: + attrValues {c = 3; a = 1; b = 2;} + => [1 2 3] + + Type: + attrValues :: AttrSet -> [Any] + */ + attrValues = builtins.attrValues or (attrs: attrVals (attrNames attrs) attrs); + + + /* Given a set of attribute names, return the set of the corresponding + attributes from the given set. + + Example: + getAttrs [ "a" "b" ] { a = 1; b = 2; c = 3; } + => { a = 1; b = 2; } + + Type: + getAttrs :: [String] -> AttrSet -> AttrSet + */ + getAttrs = + # A list of attribute names to get out of `set` + names: + # The set to get the named attributes from + attrs: genAttrs names (name: attrs.${name}); + + /* Collect each attribute named `attr` from a list of attribute + sets. Sets that don't contain the named attribute are ignored. + + Example: + catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + => [1 2] + + Type: + catAttrs :: String -> [AttrSet] -> [Any] + */ + catAttrs = builtins.catAttrs or + (attr: l: concatLists (map (s: if s ? ${attr} then [s.${attr}] else []) l)); + + + /* Filter an attribute set by removing all attributes for which the + given predicate return false. + + Example: + filterAttrs (n: v: n == "foo") { foo = 1; bar = 2; } + => { foo = 1; } + + Type: + filterAttrs :: (String -> Any -> Bool) -> AttrSet -> AttrSet + */ + filterAttrs = + # Predicate taking an attribute name and an attribute value, which returns `true` to include the attribute, or `false` to exclude the attribute. + pred: + # The attribute set to filter + set: + listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + + + /* Filter an attribute set recursively by removing all attributes for + which the given predicate return false. + + Example: + filterAttrsRecursive (n: v: v != null) { foo = { bar = null; }; } + => { foo = {}; } + + Type: + filterAttrsRecursive :: (String -> Any -> Bool) -> AttrSet -> AttrSet + */ + filterAttrsRecursive = + # Predicate taking an attribute name and an attribute value, which returns `true` to include the attribute, or `false` to exclude the attribute. + pred: + # The attribute set to filter + set: + listToAttrs ( + concatMap (name: + let v = set.${name}; in + if pred name v then [ + (nameValuePair name ( + if isAttrs v then filterAttrsRecursive pred v + else v + )) + ] else [] + ) (attrNames set) + ); + + /* + Like builtins.foldl' but for attribute sets. + Iterates over every name-value pair in the given attribute set. + The result of the callback function is often called `acc` for accumulator. It is passed between callbacks from left to right and the final `acc` is the return value of `foldlAttrs`. + + Attention: + There is a completely different function + `lib.foldAttrs` + which has nothing to do with this function, despite the similar name. + + Example: + foldlAttrs + (acc: name: value: { + sum = acc.sum + value; + names = acc.names ++ [name]; + }) + { sum = 0; names = []; } + { + foo = 1; + bar = 10; + } + -> + { + sum = 11; + names = ["bar" "foo"]; + } + + foldlAttrs + (throw "function not needed") + 123 + {}; + -> + 123 + + foldlAttrs + (_: _: v: v) + (throw "initial accumulator not needed") + { z = 3; a = 2; }; + -> + 3 + + The accumulator doesn't have to be an attrset. + It can be as simple as a number or string. + + foldlAttrs + (acc: _: v: acc * 10 + v) + 1 + { z = 1; a = 2; }; + -> + 121 + + Type: + foldlAttrs :: ( a -> String -> b -> a ) -> a -> { ... :: b } -> a + */ + foldlAttrs = f: init: set: + foldl' + (acc: name: f acc name set.${name}) + init + (attrNames set); + + /* Apply fold functions to values grouped by key. + + Example: + foldAttrs (item: acc: [item] ++ acc) [] [{ a = 2; } { a = 3; }] + => { a = [ 2 3 ]; } + + Type: + foldAttrs :: (Any -> Any -> Any) -> Any -> [AttrSets] -> Any + + */ + foldAttrs = + # A function, given a value and a collector combines the two. + op: + # The starting value. + nul: + # A list of attribute sets to fold together by key. + list_of_attrs: + foldr (n: a: + foldr (name: o: + o // { ${name} = op n.${name} (a.${name} or nul); } + ) a (attrNames n) + ) {} list_of_attrs; + + + /* Recursively collect sets that verify a given predicate named `pred` + from the set `attrs`. The recursion is stopped when the predicate is + verified. + + Example: + collect isList { a = { b = ["b"]; }; c = [1]; } + => [["b"] [1]] + + collect (x: x ? outPath) + { a = { outPath = "a/"; }; b = { outPath = "b/"; }; } + => [{ outPath = "a/"; } { outPath = "b/"; }] + + Type: + collect :: (AttrSet -> Bool) -> AttrSet -> [x] + */ + collect = + # Given an attribute's value, determine if recursion should stop. + pred: + # The attribute set to recursively collect. + attrs: + if pred attrs then + [ attrs ] + else if isAttrs attrs then + concatMap (collect pred) (attrValues attrs) + else + []; + + /* Return the cartesian product of attribute set value combinations. + + Example: + cartesianProductOfSets { a = [ 1 2 ]; b = [ 10 20 ]; } + => [ + { a = 1; b = 10; } + { a = 1; b = 20; } + { a = 2; b = 10; } + { a = 2; b = 20; } + ] + Type: + cartesianProductOfSets :: AttrSet -> [AttrSet] + */ + cartesianProductOfSets = + # Attribute set with attributes that are lists of values + attrsOfLists: + foldl' (listOfAttrs: attrName: + concatMap (attrs: + map (listValue: attrs // { ${attrName} = listValue; }) attrsOfLists.${attrName} + ) listOfAttrs + ) [{}] (attrNames attrsOfLists); + + + /* Utility function that creates a `{name, value}` pair as expected by `builtins.listToAttrs`. + + Example: + nameValuePair "some" 6 + => { name = "some"; value = 6; } + + Type: + nameValuePair :: String -> Any -> { name :: String; value :: Any; } + */ + nameValuePair = + # Attribute name + name: + # Attribute value + value: + { inherit name value; }; + + + /* Apply a function to each element in an attribute set, creating a new attribute set. + + Example: + mapAttrs (name: value: name + "-" + value) + { x = "foo"; y = "bar"; } + => { x = "x-foo"; y = "y-bar"; } + + Type: + mapAttrs :: (String -> Any -> Any) -> AttrSet -> AttrSet + */ + mapAttrs = builtins.mapAttrs or + (f: set: + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))); + + + /* Like `mapAttrs`, but allows the name of each attribute to be + changed in addition to the value. The applied function should + return both the new name and value as a `nameValuePair`. + + Example: + mapAttrs' (name: value: nameValuePair ("foo_" + name) ("bar-" + value)) + { x = "a"; y = "b"; } + => { foo_x = "bar-a"; foo_y = "bar-b"; } + + Type: + mapAttrs' :: (String -> Any -> { name :: String; value :: Any; }) -> AttrSet -> AttrSet + */ + mapAttrs' = + # A function, given an attribute's name and value, returns a new `nameValuePair`. + f: + # Attribute set to map over. + set: + listToAttrs (map (attr: f attr set.${attr}) (attrNames set)); + + + /* Call a function for each attribute in the given set and return + the result in a list. + + Example: + mapAttrsToList (name: value: name + value) + { x = "a"; y = "b"; } + => [ "xa" "yb" ] + + Type: + mapAttrsToList :: (String -> a -> b) -> AttrSet -> [b] + + */ + mapAttrsToList = + # A function, given an attribute's name and value, returns a new value. + f: + # Attribute set to map over. + attrs: + map (name: f name attrs.${name}) (attrNames attrs); + + + /* Like `mapAttrs`, except that it recursively applies itself to + the *leaf* attributes of a potentially-nested attribute set: + the second argument of the function will never be an attrset. + Also, the first argument of the argument function is a *list* + of the attribute names that form the path to the leaf attribute. + + For a function that gives you control over what counts as a leaf, + see `mapAttrsRecursiveCond`. + + Example: + mapAttrsRecursive (path: value: concatStringsSep "-" (path ++ [value])) + { n = { a = "A"; m = { b = "B"; c = "C"; }; }; d = "D"; } + => { n = { a = "n-a-A"; m = { b = "n-m-b-B"; c = "n-m-c-C"; }; }; d = "d-D"; } + + Type: + mapAttrsRecursive :: ([String] -> a -> b) -> AttrSet -> AttrSet + */ + mapAttrsRecursive = + # A function, given a list of attribute names and a value, returns a new value. + f: + # Set to recursively map over. + set: + mapAttrsRecursiveCond (as: true) f set; + + + /* Like `mapAttrsRecursive`, but it takes an additional predicate + function that tells it whether to recurse into an attribute + set. If it returns false, `mapAttrsRecursiveCond` does not + recurse, but does apply the map function. If it returns true, it + does recurse, and does not apply the map function. + + Example: + # To prevent recursing into derivations (which are attribute + # sets with the attribute "type" equal to "derivation"): + mapAttrsRecursiveCond + (as: !(as ? "type" && as.type == "derivation")) + (x: ... do something ...) + attrs + + Type: + mapAttrsRecursiveCond :: (AttrSet -> Bool) -> ([String] -> a -> b) -> AttrSet -> AttrSet + */ + mapAttrsRecursiveCond = + # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set. + cond: + # A function, given a list of attribute names and a value, returns a new value. + f: + # Attribute set to recursively map over. + set: + let + recurse = path: + let + g = + name: value: + if isAttrs value && cond value + then recurse (path ++ [name]) value + else f (path ++ [name]) value; + in mapAttrs g; + in recurse [] set; + + + /* Generate an attribute set by mapping a function over a list of + attribute names. + + Example: + genAttrs [ "foo" "bar" ] (name: "x_" + name) + => { foo = "x_foo"; bar = "x_bar"; } + + Type: + genAttrs :: [ String ] -> (String -> Any) -> AttrSet + */ + genAttrs = + # Names of values in the resulting attribute set. + names: + # A function, given the name of the attribute, returns the attribute's value. + f: + listToAttrs (map (n: nameValuePair n (f n)) names); + + + /* Check whether the argument is a derivation. Any set with + `{ type = "derivation"; }` counts as a derivation. + + Example: + nixpkgs = import <nixpkgs> {} + isDerivation nixpkgs.ruby + => true + isDerivation "foobar" + => false + + Type: + isDerivation :: Any -> Bool + */ + isDerivation = + # Value to check. + value: value.type or null == "derivation"; + + /* Converts a store path to a fake derivation. + + Type: + toDerivation :: Path -> Derivation + */ + toDerivation = + # A store path to convert to a derivation. + path: + let + path' = builtins.storePath path; + res = + { type = "derivation"; + name = sanitizeDerivationName (builtins.substring 33 (-1) (baseNameOf path')); + outPath = path'; + outputs = [ "out" ]; + out = res; + outputName = "out"; + }; + in res; + + + /* If `cond` is true, return the attribute set `as`, + otherwise an empty attribute set. + + Example: + optionalAttrs (true) { my = "set"; } + => { my = "set"; } + optionalAttrs (false) { my = "set"; } + => { } + + Type: + optionalAttrs :: Bool -> AttrSet -> AttrSet + */ + optionalAttrs = + # Condition under which the `as` attribute set is returned. + cond: + # The attribute set to return if `cond` is `true`. + as: + if cond then as else {}; + + + /* Merge sets of attributes and use the function `f` to merge attributes + values. + + Example: + zipAttrsWithNames ["a"] (name: vs: vs) [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; } + + Type: + zipAttrsWithNames :: [ String ] -> (String -> [ Any ] -> Any) -> [ AttrSet ] -> AttrSet + */ + zipAttrsWithNames = + # List of attribute names to zip. + names: + # A function, accepts an attribute name, all the values, and returns a combined value. + f: + # List of values from the list of attribute sets. + sets: + listToAttrs (map (name: { + inherit name; + value = f name (catAttrs name sets); + }) names); + + + /* Merge sets of attributes and use the function f to merge attribute values. + Like `lib.attrsets.zipAttrsWithNames` with all key names are passed for `names`. + + Implementation note: Common names appear multiple times in the list of + names, hopefully this does not affect the system because the maximal + laziness avoid computing twice the same expression and `listToAttrs` does + not care about duplicated attribute names. + + Example: + zipAttrsWith (name: values: values) [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; b = ["z"]; } + + Type: + zipAttrsWith :: (String -> [ Any ] -> Any) -> [ AttrSet ] -> AttrSet + */ + zipAttrsWith = + builtins.zipAttrsWith or (f: sets: zipAttrsWithNames (concatMap attrNames sets) f sets); + + + /* Merge sets of attributes and combine each attribute value in to a list. + + Like `lib.attrsets.zipAttrsWith` with `(name: values: values)` as the function. + + Example: + zipAttrs [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; b = ["z"]; } + + Type: + zipAttrs :: [ AttrSet ] -> AttrSet + */ + zipAttrs = + # List of attribute sets to zip together. + sets: + zipAttrsWith (name: values: values) sets; + + + /* Does the same as the update operator '//' except that attributes are + merged until the given predicate is verified. The predicate should + accept 3 arguments which are the path to reach the attribute, a part of + the first attribute set and a part of the second attribute set. When + the predicate is satisfied, the value of the first attribute set is + replaced by the value of the second attribute set. + + Example: + recursiveUpdateUntil (path: l: r: path == ["foo"]) { + # first attribute set + foo.bar = 1; + foo.baz = 2; + bar = 3; + } { + #second attribute set + foo.bar = 1; + foo.quz = 2; + baz = 4; + } + + => { + foo.bar = 1; # 'foo.*' from the second set + foo.quz = 2; # + bar = 3; # 'bar' from the first set + baz = 4; # 'baz' from the second set + } + + Type: + recursiveUpdateUntil :: ( [ String ] -> AttrSet -> AttrSet -> Bool ) -> AttrSet -> AttrSet -> AttrSet + */ + recursiveUpdateUntil = + # Predicate, taking the path to the current attribute as a list of strings for attribute names, and the two values at that path from the original arguments. + pred: + # Left attribute set of the merge. + lhs: + # Right attribute set of the merge. + rhs: + let f = attrPath: + zipAttrsWith (n: values: + let here = attrPath ++ [n]; in + if length values == 1 + || pred here (elemAt values 1) (head values) then + head values + else + f here values + ); + in f [] [rhs lhs]; + + + /* A recursive variant of the update operator ‘//’. The recursion + stops when one of the attribute values is not an attribute set, + in which case the right hand side value takes precedence over the + left hand side value. + + Example: + recursiveUpdate { + boot.loader.grub.enable = true; + boot.loader.grub.device = "/dev/hda"; + } { + boot.loader.grub.device = ""; + } + + returns: { + boot.loader.grub.enable = true; + boot.loader.grub.device = ""; + } + + Type: + recursiveUpdate :: AttrSet -> AttrSet -> AttrSet + */ + recursiveUpdate = + # Left attribute set of the merge. + lhs: + # Right attribute set of the merge. + rhs: + recursiveUpdateUntil (path: lhs: rhs: !(isAttrs lhs && isAttrs rhs)) lhs rhs; + + + /* Returns true if the pattern is contained in the set. False otherwise. + + Example: + matchAttrs { cpu = {}; } { cpu = { bits = 64; }; } + => true + + Type: + matchAttrs :: AttrSet -> AttrSet -> Bool + */ + matchAttrs = + # Attribute set structure to match + pattern: + # Attribute set to find patterns in + attrs: + assert isAttrs pattern; + all id (attrValues (zipAttrsWithNames (attrNames pattern) (n: values: + let pat = head values; val = elemAt values 1; in + if length values == 1 then false + else if isAttrs pat then isAttrs val && matchAttrs pat val + else pat == val + ) [pattern attrs])); + + + /* Override only the attributes that are already present in the old set + useful for deep-overriding. + + Example: + overrideExisting {} { a = 1; } + => {} + overrideExisting { b = 2; } { a = 1; } + => { b = 2; } + overrideExisting { a = 3; b = 2; } { a = 1; } + => { a = 1; b = 2; } + + Type: + overrideExisting :: AttrSet -> AttrSet -> AttrSet + */ + overrideExisting = + # Original attribute set + old: + # Attribute set with attributes to override in `old`. + new: + mapAttrs (name: value: new.${name} or value) old; + + + /* Turns a list of strings into a human-readable description of those + strings represented as an attribute path. The result of this function is + not intended to be machine-readable. + Create a new attribute set with `value` set at the nested attribute location specified in `attrPath`. + + Example: + showAttrPath [ "foo" "10" "bar" ] + => "foo.\"10\".bar" + showAttrPath [] + => "<root attribute path>" + + Type: + showAttrPath :: [String] -> String + */ + showAttrPath = + # Attribute path to render to a string + path: + if path == [] then "<root attribute path>" + else concatMapStringsSep "." escapeNixIdentifier path; + + + /* Get a package output. + If no output is found, fallback to `.out` and then to the default. + + Example: + getOutput "dev" pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev" + + Type: + getOutput :: String -> Derivation -> String + */ + getOutput = output: pkg: + if ! pkg ? outputSpecified || ! pkg.outputSpecified + then pkg.${output} or pkg.out or pkg + else pkg; + + /* Get a package's `bin` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getBin pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r" + + Type: + getBin :: Derivation -> String + */ + getBin = getOutput "bin"; + + + /* Get a package's `lib` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getLib pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-lib" + + Type: + getLib :: Derivation -> String + */ + getLib = getOutput "lib"; + + + /* Get a package's `dev` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getDev pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev" + + Type: + getDev :: Derivation -> String + */ + getDev = getOutput "dev"; + + + /* Get a package's `man` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getMan pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-man" + + Type: + getMan :: Derivation -> String + */ + getMan = getOutput "man"; + + /* Pick the outputs of packages to place in `buildInputs` + + Type: chooseDevOutputs :: [Derivation] -> [String] + + */ + chooseDevOutputs = + # List of packages to pick `dev` outputs from + drvs: + builtins.map getDev drvs; + + /* Make various Nix tools consider the contents of the resulting + attribute set when looking for what to build, find, etc. + + This function only affects a single attribute set; it does not + apply itself recursively for nested attribute sets. + + Example: + { pkgs ? import <nixpkgs> {} }: + { + myTools = pkgs.lib.recurseIntoAttrs { + inherit (pkgs) hello figlet; + }; + } + + Type: + recurseIntoAttrs :: AttrSet -> AttrSet + + */ + recurseIntoAttrs = + # An attribute set to scan for derivations. + attrs: + attrs // { recurseForDerivations = true; }; + + /* Undo the effect of recurseIntoAttrs. + + Type: + dontRecurseIntoAttrs :: AttrSet -> AttrSet + */ + dontRecurseIntoAttrs = + # An attribute set to not scan for derivations. + attrs: + attrs // { recurseForDerivations = false; }; + + /* `unionOfDisjoint x y` is equal to `x // y // z` where the + attrnames in `z` are the intersection of the attrnames in `x` and + `y`, and all values `assert` with an error message. This + operator is commutative, unlike (//). + + Type: unionOfDisjoint :: AttrSet -> AttrSet -> AttrSet + */ + unionOfDisjoint = x: y: + let + intersection = builtins.intersectAttrs x y; + collisions = lib.concatStringsSep " " (builtins.attrNames intersection); + mask = builtins.mapAttrs (name: value: builtins.throw + "unionOfDisjoint: collision on ${name}; complete list: ${collisions}") + intersection; + in + (x // y) // mask; + + # DEPRECATED + zipWithNames = zipAttrsWithNames; + + # DEPRECATED + zip = builtins.trace + "lib.zip is deprecated, use lib.zipAttrsWith instead" zipAttrsWith; +} diff --git a/cli.nix b/cli.nix new file mode 100644 index 000000000..c96d4dbb0 --- /dev/null +++ b/cli.nix @@ -0,0 +1,83 @@ +{ lib }: + +rec { + /* Automatically convert an attribute set to command-line options. + + This helps protect against malformed command lines and also to reduce + boilerplate related to command-line construction for simple use cases. + + `toGNUCommandLine` returns a list of nix strings. + `toGNUCommandLineShell` returns an escaped shell string. + + Example: + cli.toGNUCommandLine {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + } + => [ + "-X" "PUT" + "--data" "{\"id\":0}" + "--retry" "3" + "--url" "https://example.com/foo" + "--url" "https://example.com/bar" + "--verbose" + ] + + cli.toGNUCommandLineShell {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + } + => "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'"; + */ + toGNUCommandLineShell = + options: attrs: lib.escapeShellArgs (toGNUCommandLine options attrs); + + toGNUCommandLine = { + # how to string-format the option name; + # by default one character is a short option (`-`), + # more than one characters a long option (`--`). + mkOptionName ? + k: if builtins.stringLength k == 1 + then "-${k}" + else "--${k}", + + # how to format a boolean value to a command list; + # by default it’s a flag option + # (only the option name if true, left out completely if false). + mkBool ? k: v: lib.optional v (mkOptionName k), + + # how to format a list value to a command list; + # by default the option name is repeated for each value + # and `mkOption` is applied to the values themselves. + mkList ? k: v: lib.concatMap (mkOption k) v, + + # how to format any remaining value to a command list; + # on the toplevel, booleans and lists are handled by `mkBool` and `mkList`, + # though they can still appear as values of a list. + # By default, everything is printed verbatim and complex types + # are forbidden (lists, attrsets, functions). `null` values are omitted. + mkOption ? + k: v: if v == null + then [] + else [ (mkOptionName k) (lib.generators.mkValueStringDefault {} v) ] + }: + options: + let + render = k: v: + if builtins.isBool v then mkBool k v + else if builtins.isList v then mkList k v + else mkOption k v; + + in + builtins.concatLists (lib.mapAttrsToList render options); +} diff --git a/customisation.nix b/customisation.nix new file mode 100644 index 000000000..fe32e890f --- /dev/null +++ b/customisation.nix @@ -0,0 +1,315 @@ +{ lib }: + +rec { + + + /* `overrideDerivation drv f` takes a derivation (i.e., the result + of a call to the builtin function `derivation`) and returns a new + derivation in which the attributes of the original are overridden + according to the function `f`. The function `f` is called with + the original derivation attributes. + + `overrideDerivation` allows certain "ad-hoc" customisation + scenarios (e.g. in ~/.config/nixpkgs/config.nix). For instance, + if you want to "patch" the derivation returned by a package + function in Nixpkgs to build another version than what the + function itself provides, you can do something like this: + + mySed = overrideDerivation pkgs.gnused (oldAttrs: { + name = "sed-4.2.2-pre"; + src = fetchurl { + url = ftp://alpha.gnu.org/gnu/sed/sed-4.2.2-pre.tar.bz2; + sha256 = "11nq06d131y4wmf3drm0yk502d2xc6n5qy82cg88rb9nqd2lj41k"; + }; + patches = []; + }); + + For another application, see build-support/vm, where this + function is used to build arbitrary derivations inside a QEMU + virtual machine. + + Note that in order to preserve evaluation errors, the new derivation's + outPath depends on the old one's, which means that this function cannot + be used in circular situations when the old derivation also depends on the + new one. + + You should in general prefer `drv.overrideAttrs` over this function; + see the nixpkgs manual for more information on overriding. + */ + overrideDerivation = drv: f: + let + newDrv = derivation (drv.drvAttrs // (f drv)); + in lib.flip (extendDerivation (builtins.seq drv.drvPath true)) newDrv ( + { meta = drv.meta or {}; + passthru = if drv ? passthru then drv.passthru else {}; + } + // + (drv.passthru or {}) + // + # TODO(@Artturin): remove before release 23.05 and only have __spliced. + (lib.optionalAttrs (drv ? crossDrv && drv ? nativeDrv) { + crossDrv = overrideDerivation drv.crossDrv f; + nativeDrv = overrideDerivation drv.nativeDrv f; + }) + // + lib.optionalAttrs (drv ? __spliced) { + __spliced = {} // (lib.mapAttrs (_: sDrv: overrideDerivation sDrv f) drv.__spliced); + }); + + + /* `makeOverridable` takes a function from attribute set to attribute set and + injects `override` attribute which can be used to override arguments of + the function. + + nix-repl> x = {a, b}: { result = a + b; } + + nix-repl> y = lib.makeOverridable x { a = 1; b = 2; } + + nix-repl> y + { override = «lambda»; overrideDerivation = «lambda»; result = 3; } + + nix-repl> y.override { a = 10; } + { override = «lambda»; overrideDerivation = «lambda»; result = 12; } + + Please refer to "Nixpkgs Contributors Guide" section + "<pkg>.overrideDerivation" to learn about `overrideDerivation` and caveats + related to its use. + */ + makeOverridable = f: origArgs: + let + result = f origArgs; + + # Creates a functor with the same arguments as f + copyArgs = g: lib.setFunctionArgs g (lib.functionArgs f); + # Changes the original arguments with (potentially a function that returns) a set of new attributes + overrideWith = newArgs: origArgs // (if lib.isFunction newArgs then newArgs origArgs else newArgs); + + # Re-call the function but with different arguments + overrideArgs = copyArgs (newArgs: makeOverridable f (overrideWith newArgs)); + # Change the result of the function call by applying g to it + overrideResult = g: makeOverridable (copyArgs (args: g (f args))) origArgs; + in + if builtins.isAttrs result then + result // { + override = overrideArgs; + overrideDerivation = fdrv: overrideResult (x: overrideDerivation x fdrv); + ${if result ? overrideAttrs then "overrideAttrs" else null} = fdrv: + overrideResult (x: x.overrideAttrs fdrv); + } + else if lib.isFunction result then + # Transform the result into a functor while propagating its arguments + lib.setFunctionArgs result (lib.functionArgs result) // { + override = overrideArgs; + } + else result; + + + /* Call the package function in the file `fn` with the required + arguments automatically. The function is called with the + arguments `args`, but any missing arguments are obtained from + `autoArgs`. This function is intended to be partially + parameterised, e.g., + + callPackage = callPackageWith pkgs; + pkgs = { + libfoo = callPackage ./foo.nix { }; + libbar = callPackage ./bar.nix { }; + }; + + If the `libbar` function expects an argument named `libfoo`, it is + automatically passed as an argument. Overrides or missing + arguments can be supplied in `args`, e.g. + + libbar = callPackage ./bar.nix { + libfoo = null; + enableX11 = true; + }; + */ + callPackageWith = autoArgs: fn: args: + let + f = if lib.isFunction fn then fn else import fn; + fargs = lib.functionArgs f; + + # All arguments that will be passed to the function + # This includes automatic ones and ones passed explicitly + allArgs = builtins.intersectAttrs fargs autoArgs // args; + + # A list of argument names that the function requires, but + # wouldn't be passed to it + missingArgs = lib.attrNames + # Filter out arguments that have a default value + (lib.filterAttrs (name: value: ! value) + # Filter out arguments that would be passed + (removeAttrs fargs (lib.attrNames allArgs))); + + # Get a list of suggested argument names for a given missing one + getSuggestions = arg: lib.pipe (autoArgs // args) [ + lib.attrNames + # Only use ones that are at most 2 edits away. While mork would work, + # levenshteinAtMost is only fast for 2 or less. + (lib.filter (lib.strings.levenshteinAtMost 2 arg)) + # Put strings with shorter distance first + (lib.sort (x: y: lib.strings.levenshtein x arg < lib.strings.levenshtein y arg)) + # Only take the first couple results + (lib.take 3) + # Quote all entries + (map (x: "\"" + x + "\"")) + ]; + + prettySuggestions = suggestions: + if suggestions == [] then "" + else if lib.length suggestions == 1 then ", did you mean ${lib.elemAt suggestions 0}?" + else ", did you mean ${lib.concatStringsSep ", " (lib.init suggestions)} or ${lib.last suggestions}?"; + + errorForArg = arg: + let + loc = builtins.unsafeGetAttrPos arg fargs; + # loc' can be removed once lib/minver.nix is >2.3.4, since that includes + # https://github.com/NixOS/nix/pull/3468 which makes loc be non-null + loc' = if loc != null then loc.file + ":" + toString loc.line + else if ! lib.isFunction fn then + toString fn + lib.optionalString (lib.sources.pathIsDirectory fn) "/default.nix" + else "<unknown location>"; + in "Function called without required argument \"${arg}\" at " + + "${loc'}${prettySuggestions (getSuggestions arg)}"; + + # Only show the error for the first missing argument + error = errorForArg (lib.head missingArgs); + + in if missingArgs == [] then makeOverridable f allArgs else abort error; + + + /* Like callPackage, but for a function that returns an attribute + set of derivations. The override function is added to the + individual attributes. */ + callPackagesWith = autoArgs: fn: args: + let + f = if lib.isFunction fn then fn else import fn; + auto = builtins.intersectAttrs (lib.functionArgs f) autoArgs; + origArgs = auto // args; + pkgs = f origArgs; + mkAttrOverridable = name: _: makeOverridable (newArgs: (f newArgs).${name}) origArgs; + in + if lib.isDerivation pkgs then throw + ("function `callPackages` was called on a *single* derivation " + + ''"${pkgs.name or "<unknown-name>"}";'' + + " did you mean to use `callPackage` instead?") + else lib.mapAttrs mkAttrOverridable pkgs; + + + /* Add attributes to each output of a derivation without changing + the derivation itself and check a given condition when evaluating. */ + extendDerivation = condition: passthru: drv: + let + outputs = drv.outputs or [ "out" ]; + + commonAttrs = drv // (builtins.listToAttrs outputsList) // + ({ all = map (x: x.value) outputsList; }) // passthru; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + inherit (drv.${outputName}) type outputName; + outputSpecified = true; + drvPath = assert condition; drv.${outputName}.drvPath; + outPath = assert condition; drv.${outputName}.outPath; + } // + # TODO: give the derivation control over the outputs. + # `overrideAttrs` may not be the only attribute that needs + # updating when switching outputs. + lib.optionalAttrs (passthru?overrideAttrs) { + # TODO: also add overrideAttrs when overrideAttrs is not custom, e.g. when not splicing. + overrideAttrs = f: (passthru.overrideAttrs f).${outputName}; + }; + }; + + outputsList = map outputToAttrListElement outputs; + in commonAttrs // { + drvPath = assert condition; drv.drvPath; + outPath = assert condition; drv.outPath; + }; + + /* Strip a derivation of all non-essential attributes, returning + only those needed by hydra-eval-jobs. Also strictly evaluate the + result to ensure that there are no thunks kept alive to prevent + garbage collection. */ + hydraJob = drv: + let + outputs = drv.outputs or ["out"]; + + commonAttrs = + { inherit (drv) name system meta; inherit outputs; } + // lib.optionalAttrs (drv._hydraAggregate or false) { + _hydraAggregate = true; + constituents = map hydraJob (lib.flatten drv.constituents); + } + // (lib.listToAttrs outputsList); + + makeOutput = outputName: + let output = drv.${outputName}; in + { name = outputName; + value = commonAttrs // { + outPath = output.outPath; + drvPath = output.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map makeOutput outputs; + + drv' = (lib.head outputsList).value; + in if drv == null then null else + lib.deepSeq drv' drv'; + + /* Make a set of packages with a common scope. All packages called + with the provided `callPackage` will be evaluated with the same + arguments. Any package in the set may depend on any other. The + `overrideScope'` function allows subsequent modification of the package + set in a consistent way, i.e. all packages in the set will be + called with the overridden packages. The package sets may be + hierarchical: the packages in the set are called with the scope + provided by `newScope` and the set provides a `newScope` attribute + which can form the parent scope for later package sets. */ + makeScope = newScope: f: + let self = f self // { + newScope = scope: newScope (self // scope); + callPackage = self.newScope {}; + overrideScope = g: lib.warn + "`overrideScope` (from `lib.makeScope`) is deprecated. Do `overrideScope' (self: super: { … })` instead of `overrideScope (super: self: { … })`. All other overrides have the parameters in that order, including other definitions of `overrideScope`. This was the only definition violating the pattern." + (makeScope newScope (lib.fixedPoints.extends (lib.flip g) f)); + overrideScope' = g: makeScope newScope (lib.fixedPoints.extends g f); + packages = f; + }; + in self; + + /* Like the above, but aims to support cross compilation. It's still ugly, but + hopefully it helps a little bit. */ + makeScopeWithSplicing = splicePackages: newScope: otherSplices: keep: extra: f: + let + spliced0 = splicePackages { + pkgsBuildBuild = otherSplices.selfBuildBuild; + pkgsBuildHost = otherSplices.selfBuildHost; + pkgsBuildTarget = otherSplices.selfBuildTarget; + pkgsHostHost = otherSplices.selfHostHost; + pkgsHostTarget = self; # Not `otherSplices.selfHostTarget`; + pkgsTargetTarget = otherSplices.selfTargetTarget; + }; + spliced = extra spliced0 // spliced0 // keep self; + self = f self // { + newScope = scope: newScope (spliced // scope); + callPackage = newScope spliced; # == self.newScope {}; + # N.B. the other stages of the package set spliced in are *not* + # overridden. + overrideScope = g: makeScopeWithSplicing + splicePackages + newScope + otherSplices + keep + extra + (lib.fixedPoints.extends g f); + packages = f; + }; + in self; + +} diff --git a/debug.nix b/debug.nix new file mode 100644 index 000000000..a851cd747 --- /dev/null +++ b/debug.nix @@ -0,0 +1,253 @@ +/* Collection of functions useful for debugging + broken nix expressions. + + * `trace`-like functions take two values, print + the first to stderr and return the second. + * `traceVal`-like functions take one argument + which both printed and returned. + * `traceSeq`-like functions fully evaluate their + traced value before printing (not just to “weak + head normal form” like trace does by default). + * Functions that end in `-Fn` take an additional + function as their first argument, which is applied + to the traced value before it is printed. +*/ +{ lib }: +let + inherit (lib) + isInt + attrNames + isList + isAttrs + substring + addErrorContext + attrValues + concatLists + concatStringsSep + const + elem + generators + head + id + isDerivation + isFunction + mapAttrs + trace; +in + +rec { + + # -- TRACING -- + + /* Conditionally trace the supplied message, based on a predicate. + + Type: traceIf :: bool -> string -> a -> a + + Example: + traceIf true "hello" 3 + trace: hello + => 3 + */ + traceIf = + # Predicate to check + pred: + # Message that should be traced + msg: + # Value to return + x: if pred then trace msg x else x; + + /* Trace the supplied value after applying a function to it, and + return the original value. + + Type: traceValFn :: (a -> b) -> a -> a + + Example: + traceValFn (v: "mystring ${v}") "foo" + trace: mystring foo + => "foo" + */ + traceValFn = + # Function to apply + f: + # Value to trace and return + x: trace (f x) x; + + /* Trace the supplied value and return it. + + Type: traceVal :: a -> a + + Example: + traceVal 42 + # trace: 42 + => 42 + */ + traceVal = traceValFn id; + + /* `builtins.trace`, but the value is `builtins.deepSeq`ed first. + + Type: traceSeq :: a -> b -> b + + Example: + trace { a.b.c = 3; } null + trace: { a = <CODE>; } + => null + traceSeq { a.b.c = 3; } null + trace: { a = { b = { c = 3; }; }; } + => null + */ + traceSeq = + # The value to trace + x: + # The value to return + y: trace (builtins.deepSeq x x) y; + + /* Like `traceSeq`, but only evaluate down to depth n. + This is very useful because lots of `traceSeq` usages + lead to an infinite recursion. + + Example: + traceSeqN 2 { a.b.c = 3; } null + trace: { a = { b = {…}; }; } + => null + + Type: traceSeqN :: Int -> a -> b -> b + */ + traceSeqN = depth: x: y: + let snip = v: if isList v then noQuotes "[…]" v + else if isAttrs v then noQuotes "{…}" v + else v; + noQuotes = str: v: { __pretty = const str; val = v; }; + modify = n: fn: v: if (n == 0) then fn v + else if isList v then map (modify (n - 1) fn) v + else if isAttrs v then mapAttrs + (const (modify (n - 1) fn)) v + else v; + in trace (generators.toPretty { allowPrettyValues = true; } + (modify depth snip x)) y; + + /* A combination of `traceVal` and `traceSeq` that applies a + provided function to the value to be traced after `deepSeq`ing + it. + */ + traceValSeqFn = + # Function to apply + f: + # Value to trace + v: traceValFn f (builtins.deepSeq v v); + + /* A combination of `traceVal` and `traceSeq`. */ + traceValSeq = traceValSeqFn id; + + /* A combination of `traceVal` and `traceSeqN` that applies a + provided function to the value to be traced. */ + traceValSeqNFn = + # Function to apply + f: + depth: + # Value to trace + v: traceSeqN depth (f v) v; + + /* A combination of `traceVal` and `traceSeqN`. */ + traceValSeqN = traceValSeqNFn id; + + /* Trace the input and output of a function `f` named `name`, + both down to `depth`. + + This is useful for adding around a function call, + to see the before/after of values as they are transformed. + + Example: + traceFnSeqN 2 "id" (x: x) { a.b.c = 3; } + trace: { fn = "id"; from = { a.b = {…}; }; to = { a.b = {…}; }; } + => { a.b.c = 3; } + */ + traceFnSeqN = depth: name: f: v: + let res = f v; + in lib.traceSeqN + (depth + 1) + { + fn = name; + from = v; + to = res; + } + res; + + + # -- TESTING -- + + /* Evaluates a set of tests. + + A test is an attribute set `{expr, expected}`, + denoting an expression and its expected result. + + The result is a `list` of __failed tests__, each represented as + `{name, expected, result}`, + + - expected + - What was passed as `expected` + - result + - The actual `result` of the test + + Used for regression testing of the functions in lib; see + tests.nix for more examples. + + Important: Only attributes that start with `test` are executed. + + - If you want to run only a subset of the tests add the attribute `tests = ["testName"];` + + Example: + + runTests { + testAndOk = { + expr = lib.and true false; + expected = false; + }; + testAndFail = { + expr = lib.and true false; + expected = true; + }; + } + -> + [ + { + name = "testAndFail"; + expected = true; + result = false; + } + ] + + Type: + runTests :: { + tests = [ String ]; + ${testName} :: { + expr :: a; + expected :: a; + }; + } + -> + [ + { + name :: String; + expected :: a; + result :: a; + } + ] + */ + runTests = + # Tests to run + tests: concatLists (attrValues (mapAttrs (name: test: + let testsToRun = if tests ? tests then tests.tests else []; + in if (substring 0 4 name == "test" || elem name testsToRun) + && ((testsToRun == []) || elem name tests.tests) + && (test.expr != test.expected) + + then [ { inherit name; expected = test.expected; result = test.expr; } ] + else [] ) tests)); + + /* Create a test assuming that list elements are `true`. + + Example: + { testX = allTrue [ true ]; } + */ + testAllTrue = expr: { inherit expr; expected = map (x: true) expr; }; +} diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..7f7f73168 --- /dev/null +++ b/default.nix @@ -0,0 +1,167 @@ +/* Library of low-level helper functions for nix expressions. + * + * Please implement (mostly) exhaustive unit tests + * for new functions in `./tests.nix`. + */ +let + + inherit (import ./fixed-points.nix { inherit lib; }) makeExtensible; + + lib = makeExtensible (self: let + callLibs = file: import file { lib = self; }; + in { + + # often used, or depending on very little + trivial = callLibs ./trivial.nix; + fixedPoints = callLibs ./fixed-points.nix; + + # datatypes + attrsets = callLibs ./attrsets.nix; + lists = callLibs ./lists.nix; + strings = callLibs ./strings.nix; + stringsWithDeps = callLibs ./strings-with-deps.nix; + + # packaging + customisation = callLibs ./customisation.nix; + derivations = callLibs ./derivations.nix; + maintainers = import ../maintainers/maintainer-list.nix; + teams = callLibs ../maintainers/team-list.nix; + meta = callLibs ./meta.nix; + versions = callLibs ./versions.nix; + + # module system + modules = callLibs ./modules.nix; + options = callLibs ./options.nix; + types = callLibs ./types.nix; + + # constants + licenses = callLibs ./licenses.nix; + sourceTypes = callLibs ./source-types.nix; + systems = callLibs ./systems; + + # serialization + cli = callLibs ./cli.nix; + generators = callLibs ./generators.nix; + + # misc + asserts = callLibs ./asserts.nix; + debug = callLibs ./debug.nix; + misc = callLibs ./deprecated.nix; + + # domain-specific + fetchers = callLibs ./fetchers.nix; + + # Eval-time filesystem handling + path = callLibs ./path; + filesystem = callLibs ./filesystem.nix; + fileset = callLibs ./fileset.nix; + sources = callLibs ./sources.nix; + + # back-compat aliases + platforms = self.systems.doubles; + + # linux kernel configuration + kernel = callLibs ./kernel.nix; + + inherit (builtins) add addErrorContext attrNames concatLists + deepSeq elem elemAt filter genericClosure genList getAttr + hasAttr head isAttrs isBool isInt isList isPath isString length + lessThan listToAttrs pathExists readFile replaceStrings seq + stringLength sub substring tail trace; + inherit (self.trivial) id const pipe concat or and bitAnd bitOr bitXor + bitNot boolToString mergeAttrs flip mapNullable inNixShell isFloat min max + importJSON importTOML warn warnIf warnIfNot throwIf throwIfNot checkListOfEnum + info showWarnings nixpkgsVersion version isInOldestRelease + mod compare splitByAndCompare + functionArgs setFunctionArgs isFunction toFunction + toHexString toBaseDigits inPureEvalMode; + inherit (self.fixedPoints) fix fix' converge extends composeExtensions + composeManyExtensions makeExtensible makeExtensibleWithCustomName; + inherit (self.attrsets) attrByPath hasAttrByPath setAttrByPath + getAttrFromPath attrVals attrValues getAttrs catAttrs filterAttrs + filterAttrsRecursive foldlAttrs foldAttrs collect nameValuePair mapAttrs + mapAttrs' mapAttrsToList concatMapAttrs mapAttrsRecursive mapAttrsRecursiveCond + genAttrs isDerivation toDerivation optionalAttrs + zipAttrsWithNames zipAttrsWith zipAttrs recursiveUpdateUntil + recursiveUpdate matchAttrs overrideExisting showAttrPath getOutput getBin + getLib getDev getMan chooseDevOutputs zipWithNames zip + recurseIntoAttrs dontRecurseIntoAttrs cartesianProductOfSets + updateManyAttrsByPath; + inherit (self.lists) singleton forEach foldr fold foldl foldl' imap0 imap1 + concatMap flatten remove findSingle findFirst any all count + optional optionals toList range replicate partition zipListsWith zipLists + reverseList listDfs toposort sort naturalSort compareLists take + drop sublist last init crossLists unique intersectLists + subtractLists mutuallyExclusive groupBy groupBy'; + inherit (self.strings) concatStrings concatMapStrings concatImapStrings + intersperse concatStringsSep concatMapStringsSep + concatImapStringsSep concatLines makeSearchPath makeSearchPathOutput + makeLibraryPath makeBinPath optionalString + hasInfix hasPrefix hasSuffix stringToCharacters stringAsChars escape + escapeShellArg escapeShellArgs + isStorePath isStringLike + isValidPosixName toShellVar toShellVars + escapeRegex escapeURL escapeXML replaceChars lowerChars + upperChars toLower toUpper addContextFrom splitString + removePrefix removeSuffix versionOlder versionAtLeast + getName getVersion + mesonOption mesonBool mesonEnable + nameFromURL enableFeature enableFeatureAs withFeature + withFeatureAs fixedWidthString fixedWidthNumber + toInt toIntBase10 readPathsFromFile fileContents; + inherit (self.stringsWithDeps) textClosureList textClosureMap + noDepEntry fullDepEntry packEntry stringAfter; + inherit (self.customisation) overrideDerivation makeOverridable + callPackageWith callPackagesWith extendDerivation hydraJob + makeScope makeScopeWithSplicing; + inherit (self.derivations) lazyDerivation; + inherit (self.meta) addMetaAttrs dontDistribute setName updateName + appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio + hiPrioSet getLicenseFromSpdxId getExe; + inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile; + inherit (self.sources) cleanSourceFilter + cleanSource sourceByRegex sourceFilesBySuffices + commitIdFromGitRepo cleanSourceWith pathHasContext + canCleanSource pathIsGitRepo; + inherit (self.modules) evalModules setDefaultModuleLocation + unifyModuleSyntax applyModuleArgsIfFunction mergeModules + mergeModules' mergeOptionDecls evalOptionValue mergeDefinitions + pushDownProperties dischargeProperties filterOverrides + sortProperties fixupOptionType mkIf mkAssert mkMerge mkOverride + mkOptionDefault mkDefault mkImageMediaOverride mkForce mkVMOverride + mkFixStrictness mkOrder mkBefore mkAfter mkAliasDefinitions + mkAliasAndWrapDefinitions fixMergeModules mkRemovedOptionModule + mkRenamedOptionModule mkRenamedOptionModuleWith + mkMergedOptionModule mkChangedOptionModule + mkAliasOptionModule mkDerivedConfig doRename + mkAliasOptionModuleMD; + inherit (self.options) isOption mkEnableOption mkSinkUndeclaredOptions + mergeDefaultOption mergeOneOption mergeEqualOption mergeUniqueOption + getValues getFiles + optionAttrSetToDocList optionAttrSetToDocList' + scrubOptionValue literalExpression literalExample literalDocBook + showOption showOptionWithDefLocs showFiles + unknownModule mkOption mkPackageOption mkPackageOptionMD + mdDoc literalMD; + inherit (self.types) isType setType defaultTypeMerge defaultFunctor + isOptionType mkOptionType; + inherit (self.asserts) + assertMsg assertOneOf; + inherit (self.debug) traceIf traceVal traceValFn + traceSeq traceSeqN traceValSeq + traceValSeqFn traceValSeqN traceValSeqNFn traceFnSeqN + runTests testAllTrue; + inherit (self.misc) maybeEnv defaultMergeArg defaultMerge foldArgs + maybeAttrNullable maybeAttr ifEnable checkFlag getValue + checkReqs uniqList uniqListExt condConcat lazyGenericClosure + innerModifySumArgs modifySumArgs innerClosePropagation + closePropagation mapAttrsFlatten nvs setAttr setAttrMerge + mergeAttrsWithFunc mergeAttrsConcatenateValues + mergeAttrsNoOverride mergeAttrByFunc mergeAttrsByFuncDefaults + mergeAttrsByFuncDefaultsClean mergeAttrBy + fakeHash fakeSha256 fakeSha512 + nixType imap; + inherit (self.versions) + splitVersion; + }); +in lib diff --git a/deprecated.nix b/deprecated.nix new file mode 100644 index 000000000..ed14e04bb --- /dev/null +++ b/deprecated.nix @@ -0,0 +1,307 @@ +{ lib }: +let + inherit (builtins) head tail isList isAttrs isInt attrNames; + +in + +with lib.lists; +with lib.attrsets; +with lib.strings; + +rec { + + # returns default if env var is not set + maybeEnv = name: default: + let value = builtins.getEnv name; in + if value == "" then default else value; + + defaultMergeArg = x : y: if builtins.isAttrs y then + y + else + (y x); + defaultMerge = x: y: x // (defaultMergeArg x y); + foldArgs = merger: f: init: x: + let arg = (merger init (defaultMergeArg init x)); + # now add the function with composed args already applied to the final attrs + base = (setAttrMerge "passthru" {} (f arg) + ( z: z // { + function = foldArgs merger f arg; + args = (lib.attrByPath ["passthru" "args"] {} z) // x; + } )); + withStdOverrides = base // { + override = base.passthru.function; + }; + in + withStdOverrides; + + + # shortcut for attrByPath ["name"] default attrs + maybeAttrNullable = maybeAttr; + + # shortcut for attrByPath ["name"] default attrs + maybeAttr = name: default: attrs: attrs.${name} or default; + + + # Return the second argument if the first one is true or the empty version + # of the second argument. + ifEnable = cond: val: + if cond then val + else if builtins.isList val then [] + else if builtins.isAttrs val then {} + # else if builtins.isString val then "" + else if val == true || val == false then false + else null; + + + # Return true only if there is an attribute and it is true. + checkFlag = attrSet: name: + if name == "true" then true else + if name == "false" then false else + if (elem name (attrByPath ["flags"] [] attrSet)) then true else + attrByPath [name] false attrSet ; + + + # Input : attrSet, [ [name default] ... ], name + # Output : its value or default. + getValue = attrSet: argList: name: + ( attrByPath [name] (if checkFlag attrSet name then true else + if argList == [] then null else + let x = builtins.head argList; in + if (head x) == name then + (head (tail x)) + else (getValue attrSet + (tail argList) name)) attrSet ); + + + # Input : attrSet, [[name default] ...], [ [flagname reqs..] ... ] + # Output : are reqs satisfied? It's asserted. + checkReqs = attrSet: argList: condList: + ( + foldr lib.and true + (map (x: let name = (head x); in + + ((checkFlag attrSet name) -> + (foldr lib.and true + (map (y: let val=(getValue attrSet argList y); in + (val!=null) && (val!=false)) + (tail x))))) condList)); + + + # This function has O(n^2) performance. + uniqList = { inputList, acc ? [] }: + let go = xs: acc: + if xs == [] + then [] + else let x = head xs; + y = if elem x acc then [] else [x]; + in y ++ go (tail xs) (y ++ acc); + in go inputList acc; + + uniqListExt = { inputList, + outputList ? [], + getter ? (x: x), + compare ? (x: y: x==y) }: + if inputList == [] then outputList else + let x = head inputList; + isX = y: (compare (getter y) (getter x)); + newOutputList = outputList ++ + (if any isX outputList then [] else [x]); + in uniqListExt { outputList = newOutputList; + inputList = (tail inputList); + inherit getter compare; + }; + + condConcat = name: list: checker: + if list == [] then name else + if checker (head list) then + condConcat + (name + (head (tail list))) + (tail (tail list)) + checker + else condConcat + name (tail (tail list)) checker; + + lazyGenericClosure = {startSet, operator}: + let + work = list: doneKeys: result: + if list == [] then + result + else + let x = head list; key = x.key; in + if elem key doneKeys then + work (tail list) doneKeys result + else + work (tail list ++ operator x) ([key] ++ doneKeys) ([x] ++ result); + in + work startSet [] []; + + innerModifySumArgs = f: x: a: b: if b == null then (f a b) // x else + innerModifySumArgs f x (a // b); + modifySumArgs = f: x: innerModifySumArgs f x {}; + + + innerClosePropagation = acc: xs: + if xs == [] + then acc + else let y = head xs; + ys = tail xs; + in if ! isAttrs y + then innerClosePropagation acc ys + else let acc' = [y] ++ acc; + in innerClosePropagation + acc' + (uniqList { inputList = (maybeAttrNullable "propagatedBuildInputs" [] y) + ++ (maybeAttrNullable "propagatedNativeBuildInputs" [] y) + ++ ys; + acc = acc'; + } + ); + + closePropagationSlow = list: (uniqList {inputList = (innerClosePropagation [] list);}); + + # This is an optimisation of lib.closePropagation which avoids the O(n^2) behavior + # Using a list of derivations, it generates the full closure of the propagatedXXXBuildInputs + # The ordering / sorting / comparison is done based on the `outPath` + # attribute of each derivation. + # On some benchmarks, it performs up to 15 times faster than lib.closePropagation. + # See https://github.com/NixOS/nixpkgs/pull/194391 for details. + closePropagationFast = list: + builtins.map (x: x.val) (builtins.genericClosure { + startSet = builtins.map (x: { + key = x.outPath; + val = x; + }) (builtins.filter (x: x != null) list); + operator = item: + if !builtins.isAttrs item.val then + [ ] + else + builtins.concatMap (x: + if x != null then [{ + key = x.outPath; + val = x; + }] else + [ ]) ((item.val.propagatedBuildInputs or [ ]) + ++ (item.val.propagatedNativeBuildInputs or [ ])); + }); + + closePropagation = if builtins ? genericClosure + then closePropagationFast + else closePropagationSlow; + + # calls a function (f attr value ) for each record item. returns a list + mapAttrsFlatten = f: r: map (attr: f attr r.${attr}) (attrNames r); + + # attribute set containing one attribute + nvs = name: value: listToAttrs [ (nameValuePair name value) ]; + # adds / replaces an attribute of an attribute set + setAttr = set: name: v: set // (nvs name v); + + # setAttrMerge (similar to mergeAttrsWithFunc but only merges the values of a particular name) + # setAttrMerge "a" [] { a = [2];} (x: x ++ [3]) -> { a = [2 3]; } + # setAttrMerge "a" [] { } (x: x ++ [3]) -> { a = [ 3]; } + setAttrMerge = name: default: attrs: f: + setAttr attrs name (f (maybeAttr name default attrs)); + + # Using f = a: b = b the result is similar to // + # merge attributes with custom function handling the case that the attribute + # exists in both sets + mergeAttrsWithFunc = f: set1: set2: + foldr (n: set: if set ? ${n} + then setAttr set n (f set.${n} set2.${n}) + else set ) + (set2 // set1) (attrNames set2); + + # merging two attribute set concatenating the values of same attribute names + # eg { a = 7; } { a = [ 2 3 ]; } becomes { a = [ 7 2 3 ]; } + mergeAttrsConcatenateValues = mergeAttrsWithFunc ( a: b: (toList a) ++ (toList b) ); + + # merges attributes using //, if a name exists in both attributes + # an error will be triggered unless its listed in mergeLists + # so you can mergeAttrsNoOverride { buildInputs = [a]; } { buildInputs = [a]; } {} to get + # { buildInputs = [a b]; } + # merging buildPhase doesn't really make sense. The cases will be rare where appending /prefixing will fit your needs? + # in these cases the first buildPhase will override the second one + # ! deprecated, use mergeAttrByFunc instead + mergeAttrsNoOverride = { mergeLists ? ["buildInputs" "propagatedBuildInputs"], + overrideSnd ? [ "buildPhase" ] + }: attrs1: attrs2: + foldr (n: set: + setAttr set n ( if set ? ${n} + then # merge + if elem n mergeLists # attribute contains list, merge them by concatenating + then attrs2.${n} ++ attrs1.${n} + else if elem n overrideSnd + then attrs1.${n} + else throw "error mergeAttrsNoOverride, attribute ${n} given in both attributes - no merge func defined" + else attrs2.${n} # add attribute not existing in attr1 + )) attrs1 (attrNames attrs2); + + + # example usage: + # mergeAttrByFunc { + # inherit mergeAttrBy; # defined below + # buildInputs = [ a b ]; + # } { + # buildInputs = [ c d ]; + # }; + # will result in + # { mergeAttrsBy = [...]; buildInputs = [ a b c d ]; } + # is used by defaultOverridableDelayableArgs and can be used when composing using + # foldArgs, composedArgsAndFun or applyAndFun. Example: composableDerivation in all-packages.nix + mergeAttrByFunc = x: y: + let + mergeAttrBy2 = { mergeAttrBy = lib.mergeAttrs; } + // (maybeAttr "mergeAttrBy" {} x) + // (maybeAttr "mergeAttrBy" {} y); in + foldr lib.mergeAttrs {} [ + x y + (mapAttrs ( a: v: # merge special names using given functions + if x ? ${a} + then if y ? ${a} + then v x.${a} y.${a} # both have attr, use merge func + else x.${a} # only x has attr + else y.${a} # only y has attr) + ) (removeAttrs mergeAttrBy2 + # don't merge attrs which are neither in x nor y + (filter (a: ! x ? ${a} && ! y ? ${a}) + (attrNames mergeAttrBy2)) + ) + ) + ]; + mergeAttrsByFuncDefaults = foldl mergeAttrByFunc { inherit mergeAttrBy; }; + mergeAttrsByFuncDefaultsClean = list: removeAttrs (mergeAttrsByFuncDefaults list) ["mergeAttrBy"]; + + # sane defaults (same name as attr name so that inherit can be used) + mergeAttrBy = # { buildInputs = concatList; [...]; passthru = mergeAttr; [..]; } + listToAttrs (map (n: nameValuePair n lib.concat) + [ "nativeBuildInputs" "buildInputs" "propagatedBuildInputs" "configureFlags" "prePhases" "postAll" "patches" ]) + // listToAttrs (map (n: nameValuePair n lib.mergeAttrs) [ "passthru" "meta" "cfg" "flags" ]) + // listToAttrs (map (n: nameValuePair n (a: b: "${a}\n${b}") ) [ "preConfigure" "postInstall" ]) + ; + + nixType = x: + if isAttrs x then + if x ? outPath then "derivation" + else "attrs" + else if lib.isFunction x then "function" + else if isList x then "list" + else if x == true then "bool" + else if x == false then "bool" + else if x == null then "null" + else if isInt x then "int" + else "string"; + + /* deprecated: + + For historical reasons, imap has an index starting at 1. + + But for consistency with the rest of the library we want an index + starting at zero. + */ + imap = imap1; + + # Fake hashes. Can be used as hash placeholders, when computing hash ahead isn't trivial + fakeHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + fakeSha256 = "0000000000000000000000000000000000000000000000000000000000000000"; + fakeSha512 = "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; +} diff --git a/derivations.nix b/derivations.nix new file mode 100644 index 000000000..5b7ed1868 --- /dev/null +++ b/derivations.nix @@ -0,0 +1,101 @@ +{ lib }: + +let + inherit (lib) throwIfNot; +in +{ + /* + Restrict a derivation to a predictable set of attribute names, so + that the returned attrset is not strict in the actual derivation, + saving a lot of computation when the derivation is non-trivial. + + This is useful in situations where a derivation might only be used for its + passthru attributes, improving evaluation performance. + + The returned attribute set is lazy in `derivation`. Specifically, this + means that the derivation will not be evaluated in at least the + situations below. + + For illustration and/or testing, we define derivation such that its + evaluation is very noticeable. + + let derivation = throw "This won't be evaluated."; + + In the following expressions, `derivation` will _not_ be evaluated: + + (lazyDerivation { inherit derivation; }).type + + attrNames (lazyDerivation { inherit derivation; }) + + (lazyDerivation { inherit derivation; } // { foo = true; }).foo + + (lazyDerivation { inherit derivation; meta.foo = true; }).meta + + In these expressions, `derivation` _will_ be evaluated: + + "${lazyDerivation { inherit derivation }}" + + (lazyDerivation { inherit derivation }).outPath + + (lazyDerivation { inherit derivation }).meta + + And the following expressions are not valid, because the refer to + implementation details and/or attributes that may not be present on + some derivations: + + (lazyDerivation { inherit derivation }).buildInputs + + (lazyDerivation { inherit derivation }).passthru + + (lazyDerivation { inherit derivation }).pythonPath + + */ + lazyDerivation = + args@{ + # The derivation to be wrapped. + derivation + , # Optional meta attribute. + # + # While this function is primarily about derivations, it can improve + # the `meta` package attribute, which is usually specified through + # `mkDerivation`. + meta ? null + , # Optional extra values to add to the returned attrset. + # + # This can be used for adding package attributes, such as `tests`. + passthru ? { } + }: + let + # These checks are strict in `drv` and some `drv` attributes, but the + # attrset spine returned by lazyDerivation does not depend on it. + # Instead, the individual derivation attributes do depend on it. + checked = + throwIfNot (derivation.type or null == "derivation") + "lazySimpleDerivation: input must be a derivation." + throwIfNot + (derivation.outputs == [ "out" ]) + # Supporting multiple outputs should be a matter of inheriting more attrs. + "The derivation ${derivation.name or "<unknown>"} has multiple outputs. This is not supported by lazySimpleDerivation yet. Support could be added, and be useful as long as the set of outputs is known in advance, without evaluating the actual derivation." + derivation; + in + { + # Hardcoded `type` + # + # `lazyDerivation` requires its `derivation` argument to be a derivation, + # so if it is not, that is a programming error by the caller and not + # something that `lazyDerivation` consumers should be able to correct + # for after the fact. + # So, to improve laziness, we assume correctness here and check it only + # when actual derivation values are accessed later. + type = "derivation"; + + # A fixed set of derivation values, so that `lazyDerivation` can return + # its attrset before evaluating `derivation`. + # This must only list attributes that are available on _all_ derivations. + inherit (checked) outputs out outPath outputName drvPath name system; + + # The meta attribute can either be taken from the derivation, or if the + # `lazyDerivation` caller knew a shortcut, be taken from there. + meta = args.meta or checked.meta; + } // passthru; +} diff --git a/fetchers.nix b/fetchers.nix new file mode 100644 index 000000000..1107353b5 --- /dev/null +++ b/fetchers.nix @@ -0,0 +1,13 @@ +# snippets that can be shared by multiple fetchers (pkgs/build-support) +{ lib }: +{ + + proxyImpureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + +} diff --git a/fileset.nix b/fileset.nix new file mode 100644 index 000000000..3eb909162 --- /dev/null +++ b/fileset.nix @@ -0,0 +1,993 @@ +# Add this to docs somewhere: If you pass the same arguments to the same functions, you will get the same result +{ lib }: +let + # TODO: Add builtins.traceVerbose or so to all the operations for debugging + # TODO: Document limitation that empty directories won't be included + # TODO: Point out that equality comparison using `==` doesn't quite work because there's multiple representations for all files in a directory: "directory" and `readDir path`. + # TODO: subset and superset check functions. Can easily be implemented with `difference` and `isEmpty` + # TODO: Write down complexity of each operation, most should be O(1)! + # TODO: Implement an operation for optionally including a file if it exists. + # TODO: Derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets + + inherit (builtins) + isAttrs + isString + isPath + isList + typeOf + readDir + match + pathExists + seq + ; + + inherit (lib.trivial) + mapNullable + ; + + inherit (lib.lists) + head + tail + foldl' + range + length + elemAt + all + imap0 + drop + commonPrefix + ; + + inherit (lib.strings) + isCoercibleToString + ; + + inherit (lib.attrsets) + mapAttrs + attrNames + attrValues + ; + + inherit (lib.filesystem) + pathType + ; + + inherit (lib.sources) + cleanSourceWith + ; + + inherit (lib.path) + append + deconstruct + construct + hasPrefix + removePrefix + ; + + inherit (lib.path.components) + fromSubpath + ; + + # Internal file set structure: + # + # # A set of files + # <fileset> = { + # _type = "fileset"; + # + # # The base path, only files under this path can be represented + # # Always a directory + # _base = <path>; + # + # # A tree representation of all included files + # _tree = <tree>; + # }; + # + # # A directory entry value + # <tree> = + # # A nested directory + # <directory> + # + # # A nested file + # | <file> + # + # # A removed file or directory + # # This is represented like this instead of removing the entry from the attribute set because: + # # - It improves laziness + # # - It allows keeping the attribute set as a `builtins.readDir` cache + # | null + # + # # A directory + # <directory> = + # # The inclusion state for every directory entry + # { <name> = <tree>; } + # + # # All files in a directory, recursively. + # # Semantically this is equivalent to `builtins.readDir path`, but lazier, because + # # operations that don't require the entry listing can avoid it. + # # This string is chosen to be compatible with `builtins.readDir` for a simpler implementation + # "directory"; + # + # # A file + # <file> = + # # A file with this filetype + # # These strings match `builtins.readDir` for a simpler implementation + # "regular" | "symlink" | "unknown" + + # Create a fileset structure + # Type: Path -> <tree> -> <fileset> + _create = base: tree: { + _type = "fileset"; + # All attributes are internal + _base = base; + _tree = tree; + # Double __ to make it be evaluated and ordered first + __noEval = throw '' + File sets are not intended to be directly inspected or evaluated. Instead prefer: + - If you want to print a file set, use the `lib.fileset.trace` or `lib.fileset.pretty` function. + - If you want to check file sets for equality, use the `lib.fileset.equals` function. + ''; + }; + + # Create a file set from a path + # Type: Path -> <fileset> + _singleton = path: + let + type = pathType path; + in + if type == "directory" then + _create path type + else + # Always coerce to a directory + # If we don't do this we run into problems like: + # - What should `toSource { base = ./default.nix; fileset = difference ./default.nix ./default.nix; }` do? + # - Importing an empty directory wouldn't make much sense because our `base` is a file + # - Neither can we create a store path containing nothing at all + # - The only option is to throw an error that `base` should be a directory + # - Should `fileFilter (file: file.name == "default.nix") ./default.nix` run the predicate on the ./default.nix file? + # - If no, should the result include or exclude ./default.nix? In any case, it would be confusing and inconsistent + # - If yes, it needs to consider ./. to have influence the filesystem result, because file names are part of the parent directory, so filter would change the necessary base + _create (dirOf path) + (_nestTree + (dirOf path) + [ (baseNameOf path) ] + type + ); + + # Turn a builtins.filterSource-based source filter on a root path into a file set containing only files included by the filter + # Type: Path -> (String -> String -> Bool) -> <fileset> + _fromSource = root: filter: + let + recurse = focusPath: type: + # FIXME: Generally we shouldn't use toString on paths, though it might be correct + # here since we're trying to mimic the impure behavior of `builtins.filterPath` + if ! filter (toString focusPath) type then + null + else if type == "directory" then + mapAttrs + (name: recurse (append focusPath name)) + (readDir focusPath) + else + type; + + + rootPathType = pathType root; + tree = + if rootPathType == "directory" then + recurse root rootPathType + else + rootPathType; + in + _create root tree; + + # Coerce a value to a fileset + # Type: String -> String -> Any -> <fileset> + _coerce = function: context: value: + if value._type or "" == "fileset" then + value + else if ! isPath value then + if value._isLibCleanSourceWith or false then + throw '' + lib.fileset.${function}: Expected ${context} to be a path, but it's a value produced by `lib.sources` instead. + Such a value is only supported when converted to a file set using `lib.fileset.fromSource`.'' + else if isCoercibleToString value then + throw '' + lib.fileset.${function}: Expected ${context} to be a path, but it's a string-coercible value instead, possibly a Nix store path. + Such a value is not supported, `lib.fileset` only supports local file filtering.'' + else + throw "lib.fileset.${function}: Expected ${context} to be a path, but got a ${typeOf value}." + else if ! pathExists value then + throw "lib.fileset.${function}: Expected ${context} \"${toString value}\" to be a path that exists, but it doesn't." + else + _singleton value; + + # Nest a tree under some further components + # Type: Path -> [ String ] -> <tree> -> <tree> + _nestTree = targetBase: extraComponents: tree: + let + recurse = index: focusPath: + if index == length extraComponents then + tree + else + let + focusedName = elemAt extraComponents index; + in + mapAttrs + (name: _: + if name == focusedName then + recurse (index + 1) (append focusPath name) + else + null + ) + (readDir focusPath); + in + recurse 0 targetBase; + + # Expand "directory" to { <name> = <tree>; } + # Type: Path -> <directory> -> { <name> = <tree>; } + _directoryEntries = path: value: + if isAttrs value then + value + else + readDir path; + + # The following tables are a bit complicated, but they nicely explain the + # corresponding implementations, here's the legend: + # + # lhs\rhs: The values for the left hand side and right hand side arguments + # null: null, an excluded file/directory + # attrs: satisfies `isAttrs value`, an explicitly listed directory containing nested trees + # dir: "directory", a recursively included directory + # str: "regular", "symlink" or "unknown", a filetype string + # rec: A result computed by recursing + # -: Can't occur because one argument is a directory while the other is a file + # <number>: Indicates that the result is computed by the branch with that number + + # The union of two <tree>'s + # Type: <tree> -> <tree> -> <tree> + # + # lhs\rhs | null | attrs | dir | str | + # ------- | ------- | ------- | ----- | ----- | + # null | 2 null | 2 attrs | 2 dir | 2 str | + # attrs | 3 attrs | 1 rec | 2 dir | - | + # dir | 3 dir | 3 dir | 2 dir | - | + # str | 3 str | - | - | 2 str | + _unionTree = lhs: rhs: + # Branch 1 + if isAttrs lhs && isAttrs rhs then + mapAttrs (name: _unionTree lhs.${name}) rhs + # Branch 2 + else if lhs == null || isString rhs then + rhs + # Branch 3 + else + lhs; + + # The intersection of two <tree>'s + # Type: <tree> -> <tree> -> <tree> + # + # lhs\rhs | null | attrs | dir | str | + # ------- | ------- | ------- | ------- | ------ | + # null | 2 null | 2 null | 2 null | 2 null | + # attrs | 3 null | 1 rec | 2 attrs | - | + # dir | 3 null | 3 attrs | 2 dir | - | + # str | 3 null | - | - | 2 str | + _intersectTree = lhs: rhs: + # Branch 1 + if isAttrs lhs && isAttrs rhs then + mapAttrs (name: _intersectTree lhs.${name}) rhs + # Branch 2 + else if lhs == null || isString rhs then + lhs + # Branch 3 + else + rhs; + + # The difference between two <tree>'s + # Type: Path -> <tree> -> <tree> -> <tree> + # + # lhs\rhs | null | attrs | dir | str | + # ------- | ------- | ------- | ------ | ------ | + # null | 1 null | 1 null | 1 null | 1 null | + # attrs | 2 attrs | 3 rec | 1 null | - | + # dir | 2 dir | 3 rec | 1 null | - | + # str | 2 str | - | - | 1 null | + _differenceTree = path: lhs: rhs: + # Branch 1 + if isString rhs || lhs == null then + null + # Branch 2 + else if rhs == null then + lhs + # Branch 3 + else + mapAttrs (name: lhsValue: + _differenceTree (append path name) lhsValue rhs.${name} + ) (_directoryEntries path lhs); + + # Whether two <tree>'s are equal + # Type: Path -> <tree> -> <tree> -> <tree> + # + # | lhs\rhs | null | attrs | dir | str | + # | ------- | ------- | ------- | ------ | ------- | + # | null | 1 true | 1 rec | 1 rec | 1 false | + # | attrs | 2 rec | 3 rec | 3 rec | - | + # | dir | 2 rec | 3 rec | 4 true | - | + # | str | 2 false | - | - | 4 true | + _equalsTree = path: lhs: rhs: + # Branch 1 + if lhs == null then + _isEmptyTree path rhs + # Branch 2 + else if rhs == null then + _isEmptyTree path lhs + # Branch 3 + else if isAttrs lhs || isAttrs rhs then + let + lhs' = _directoryEntries path lhs; + rhs' = _directoryEntries path rhs; + in + all (name: + _equalsTree (append path name) lhs'.${name} rhs'.${name} + ) (attrNames lhs') + # Branch 4 + else + true; + + # Whether a tree is empty, containing no files + # Type: Path -> <tree> -> Bool + _isEmptyTree = path: tree: + if isAttrs tree || tree == "directory" then + let + entries = _directoryEntries path tree; + in + all (name: _isEmptyTree (append path name) entries.${name}) (attrNames entries) + else + tree == null; + + # Simplifies a tree, optionally expanding all "directory"'s into complete listings + # Type: Bool -> Path -> <tree> -> <tree> + _simplifyTree = expand: base: tree: + let + recurse = focusPath: tree: + if tree == "directory" && expand || isAttrs tree then + let + expanded = _directoryEntries focusPath tree; + transformedSubtrees = mapAttrs (name: recurse (append focusPath name)) expanded; + values = attrValues transformedSubtrees; + in + if all (value: value == "emptyDir") values then + "emptyDir" + else if all (value: isNull value || value == "emptyDir") values then + null + else if !expand && all (value: isString value || value == "emptyDir") values then + "directory" + else + mapAttrs (name: value: if value == "emptyDir" then null else value) transformedSubtrees + else + tree; + result = recurse base tree; + in + if result == "emptyDir" then + null + else + result; + + _prettyTreeSuffix = tree: + if isAttrs tree then + "" + else if tree == "directory" then + " (recursive directory)" + else + " (${tree})"; + + # Pretty-print all files included in the file set. + # Type: (b -> String -> b) -> b -> Path -> FileSet -> b + _prettyFoldl' = f: start: base: tree: + let + traceTreeAttrs = start: indent: tree: + # Nix should really be evaluating foldl''s second argument before starting the iteration + # See the same problem in Haskell: + # - https://stackoverflow.com/a/14282642 + # - https://gitlab.haskell.org/ghc/ghc/-/issues/12173 + # - https://well-typed.com/blog/90/#a-postscript-which-foldl + # - https://old.reddit.com/r/haskell/comments/21wvk7/foldl_is_broken/ + seq start + (foldl' (prev: name: + let + subtree = tree.${name}; + + intermediate = + f prev "${indent}- ${name}${_prettyTreeSuffix subtree}"; + in + if subtree == null then + # Don't print anything at all if this subtree is empty + prev + else if isAttrs subtree then + # A directory with explicit entries + # Do print this node, but also recurse + traceTreeAttrs intermediate "${indent} " subtree + else + # Either a file, or a recursively included directory + # Do print this node but no further recursion needed + intermediate + ) start (attrNames tree)); + + intermediate = + if tree == null then + f start "${toString base} (empty)" + else + f start "${toString base}${_prettyTreeSuffix tree}"; + in + if isAttrs tree then + traceTreeAttrs intermediate "" tree + else + intermediate; + + # Coerce and normalise the bases of multiple file set values passed to user-facing functions + # Type: String -> [ { context :: String, value :: Any } ] -> { commonBase :: Path, trees :: [ <tree> ] } + _normaliseBase = function: list: + let + processed = map ({ context, value }: + let + fileset = _coerce function context value; + in { + inherit fileset context; + baseParts = deconstruct fileset._base; + } + ) list; + + first = head processed; + + commonComponents = foldl' (components: el: + if first.baseParts.root != el.baseParts.root then + throw "lib.fileset.${function}: Expected file sets to have the same filesystem root, but ${first.context} has root \"${toString first.baseParts.root}\" while ${el.context} has root \"${toString el.baseParts.root}\"." + else + commonPrefix components el.baseParts.components + ) first.baseParts.components (tail processed); + + commonBase = construct { + root = first.baseParts.root; + components = commonComponents; + }; + + commonComponentsLength = length commonComponents; + + trees = map (value: + _nestTree + commonBase + (drop commonComponentsLength value.baseParts.components) + value.fileset._tree + ) processed; + in + { + inherit commonBase trees; + }; + +in { + + /* + Import a file set into the Nix store, making it usable inside derivations. + Return a source-like value that can be coerced to a Nix store path. + + This function takes an attribute set with these attributes as an argument: + + - `root` (required): The local path that should be the root of the result. + `fileset` must not be influenceable by paths outside `root`, meaning `lib.fileset.getInfluenceBase fileset` must be under `root`. + + Warning: Setting `root` to `lib.fileset.getInfluenceBase fileset` directly would make the resulting Nix store path file structure dependent on how `fileset` is declared. + This makes it non-trivial to predict where specific paths are located in the result. + + - `fileset` (required): The set of files to import into the Nix store. + Use the other `lib.fileset` functions to define `fileset`. + Only directories containing at least one file are included in the result, unless `extraExistingDirs` is used to ensure the existence of specific directories even without any files. + + - `extraExistingDirs` (optional, default `[]`): Additionally ensure the existence of these directory paths in the result, even they don't contain any files in `fileset`. + + Type: + toSource :: { + root :: Path, + fileset :: FileSet, + extraExistingDirs :: [ Path ] ? [ ], + } -> SourceLike + */ + toSource = { root, fileset, extraExistingDirs ? [ ] }: + let + maybeFileset = fileset; + in + let + fileset = _coerce "toSource" "`fileset` attribute" maybeFileset; + + # Directories that recursively have no files in them will always be `null` + sparseTree = + let + recurse = focusPath: tree: + if tree == "directory" || isAttrs tree then + let + entries = _directoryEntries focusPath tree; + sparseSubtrees = mapAttrs (name: recurse (append focusPath name)) entries; + values = attrValues sparseSubtrees; + in + if all isNull values then + null + else if all isString values then + "directory" + else + sparseSubtrees + else + tree; + resultingTree = recurse fileset._base fileset._tree; + # The fileset's _base might be below the root of the `toSource`, so we need to lift the tree up to `root` + extraRootNesting = removePrefix root fileset._base; + in _nestTree root extraRootNesting resultingTree; + + sparseExtendedTree = + if ! isList extraExistingDirs then + throw "lib.fileset.toSource: Expected the `extraExistingDirs` attribute to be a list, but it's a ${typeOf extraExistingDirs} instead." + else + lib.foldl' (tree: i: + let + dir = elemAt extraExistingDirs i; + + # We're slightly abusing the internal functions and structure to ensure that the extra directory is represented in the sparse tree. + value = mapAttrs (name: value: null) (readDir dir); + extraTree = _nestTree root (removePrefix root dir) value; + result = _unionTree tree extraTree; + in + if ! isPath dir then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths, but element at index ${toString i} is a ${typeOf dir} instead." + else if ! pathExists dir then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths that exist, but the path at index ${toString i} \"${toString dir}\" does not." + else if pathType dir != "directory" then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths pointing to directories, but the path at index ${toString i} \"${toString dir}\" points to a file instead." + else if ! hasPrefix root dir then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths under the `root` attribute \"${toString root}\", but the path at index ${toString i} \"${toString dir}\" is not." + else + result + ) sparseTree (range 0 (length extraExistingDirs - 1)); + + rootComponentsLength = length (deconstruct root).components; + + # This function is called often for the filter, so it should be fast + inSet = components: + let + recurse = index: localTree: + if index == length components then + localTree != null + else if localTree ? ${elemAt components index} then + recurse (index + 1) localTree.${elemAt components index} + else + localTree == "directory"; + in recurse rootComponentsLength sparseExtendedTree; + + in + if ! isPath root then + if root._isLibCleanSourceWith or false then + throw '' + lib.fileset.toSource: Expected attribute `root` to be a path, but it's a value produced by `lib.sources` instead. + Such a value is only supported when converted to a file set using `lib.fileset.fromSource` and passed to the `fileset` attribute, where it may also be combined using other functions from `lib.fileset`.'' + else if isCoercibleToString root then + throw '' + lib.fileset.toSource: Expected attribute `root` to be a path, but it's a string-like value instead, possibly a Nix store path. + Such a value is not supported, `lib.fileset` only supports local file filtering.'' + else + throw "lib.fileset.toSource: Expected attribute `root` to be a path, but it's a ${typeOf root} instead." + else if ! pathExists root then + throw "lib.fileset.toSource: Expected attribute `root` \"${toString root}\" to be a path that exists, but it doesn't." + else if pathType root != "directory" then + throw "lib.fileset.toSource: Expected attribute `root` \"${toString root}\" to be a path pointing to a directory, but it's pointing to a file instead." + else if ! hasPrefix root fileset._base then + throw "lib.fileset.toSource: Expected attribute `fileset` to not be influenceable by any paths outside `root`, but `lib.fileset.getInfluenceBase fileset` \"${toString fileset._base}\" is outside `root`." + else + cleanSourceWith { + name = "source"; + src = root; + filter = pathString: _: inSet (fromSubpath "./${pathString}"); + }; + + /* + Create a file set from a filtered local source as produced by the `lib.sources` functions. + This does not import anything into the store. + + Type: + fromSource :: SourceLike -> FileSet + + Example: + fromSource (lib.sources.cleanSource ./.) + */ + fromSource = source: + if ! source._isLibCleanSourceWith or false || ! source ? origSrc || ! source ? filter then + throw "lib.fileset.fromSource: Expected the argument to be a value produced from `lib.sources`, but got a ${typeOf source} instead." + else if ! isPath source.origSrc then + throw "lib.fileset.fromSource: Expected the argument to be source-like value of a local path." + else + _fromSource source.origSrc source.filter; + + /* + Coerce a value to a file set: + + - If the value is a file set already, return it directly + + - If the value is a path pointing to a file, return a file set with that single file + + - If the value is a path pointing to a directory, return a file set with all files contained in that directory + + This function is mostly not needed because all functions in `lib.fileset` will implicitly apply it for arguments that are expected to be a file set. + + Type: + coerce :: Any -> FileSet + */ + coerce = value: _coerce "coerce" "argument" value; + + + /* + Create a file set containing all files contained in a path (see `coerce`), or no files if the path doesn't exist. + + This is useful when you want to include a file only if it actually exists. + + Type: + optional :: Path -> FileSet + */ + optional = path: + if ! isPath path then + throw "lib.fileset.optional: Expected argument to be a path, but got a ${typeOf path}." + else if pathExists path then + _singleton path + else + _create path null; + + /* + Return the common ancestor directory of all file set operations used to construct this file set, meaning that nothing outside the this directory can influence the set of files included. + + Type: + getInfluenceBase :: FileSet -> Path + + Example: + getInfluenceBase ./Makefile + => ./. + + getInfluenceBase ./src + => ./src + + getInfluenceBase (fileFilter (file: false) ./.) + => ./. + + getInfluenceBase (union ./Makefile ../default.nix) + => ../. + */ + getInfluenceBase = maybeFileset: + let + fileset = _coerce "getInfluenceBase" "argument" maybeFileset; + in + fileset._base; + + /* + Incrementally evaluate and trace a file set in a pretty way. + Functionally this is the same as splitting the result from `lib.fileset.pretty` into lines and tracing those. + However this function can do the same thing incrementally, so it can already start printing the result as the first lines are known. + + The `expand` argument (false by default) controls whether all files should be printed individually. + + Type: + trace :: { expand :: Bool ? false } -> FileSet -> Any -> Any + + Example: + trace {} (unions [ ./foo.nix ./bar/baz.c ./qux ]) + => + trace: /home/user/src/myProject + trace: - bar + trace: - baz.c (regular) + trace: - foo.nix (regular) + trace: - qux (recursive directory) + null + + trace { expand = true; } (unions [ ./foo.nix ./bar/baz.c ./qux ]) + => + trace: /home/user/src/myProject + trace: - bar + trace: - baz.c (regular) + trace: - foo.nix (regular) + trace: - qux + trace: - florp.c (regular) + trace: - florp.h (regular) + null + */ + trace = { expand ? false }: maybeFileset: + let + fileset = _coerce "trace" "second argument" maybeFileset; + simpleTree = _simplifyTree expand fileset._base fileset._tree; + in + _prettyFoldl' (acc: el: builtins.trace el acc) (x: x) + fileset._base + simpleTree; + + /* + The same as `lib.fileset.trace`, but instead of taking an argument for the value to return, the given file set is returned instead. + + Type: + traceVal :: { expand :: Bool ? false } -> FileSet -> FileSet + */ + traceVal = { expand ? false }: maybeFileset: + let + fileset = _coerce "traceVal" "second argument" maybeFileset; + simpleTree = _simplifyTree expand fileset._base fileset._tree; + in + _prettyFoldl' (acc: el: builtins.trace el acc) fileset + fileset._base + simpleTree; + + /* + The same as `lib.fileset.trace`, but instead of tracing each line, the result is returned as a string. + + Type: + pretty :: { expand :: Bool ? false } -> FileSet -> String + */ + pretty = { expand ? false }: maybeFileset: + let + fileset = _coerce "pretty" "second argument" maybeFileset; + simpleTree = _simplifyTree expand fileset._base fileset._tree; + in + _prettyFoldl' (acc: el: "${acc}\n${el}") "" + fileset._base + simpleTree; + + /* + The file set containing all files that are in either of two given file sets. + Recursively, the first argument is evaluated first, only evaluating the second argument if necessary. + + union a b = a ⋃ b + + Type: + union :: FileSet -> FileSet -> FileSet + */ + union = lhs: rhs: + let + normalised = _normaliseBase "union" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _create normalised.commonBase + (_unionTree + (elemAt normalised.trees 0) + (elemAt normalised.trees 1) + ); + + /* + The file containing all files from that are in any of the given file sets. + Recursively, the elements are evaluated from left to right, only evaluating arguments on the right if necessary. + + Type: + unions :: [FileSet] -> FileSet + */ + unions = list: + let + annotated = imap0 (i: el: { + context = "element ${toString i} of the argument"; + value = el; + }) list; + + normalised = _normaliseBase "unions" annotated; + + tree = foldl' _unionTree (head normalised.trees) (tail normalised.trees); + in + if ! isList list then + throw "lib.fileset.unions: Expected argument to be a list, but got a ${typeOf list}." + else if length list == 0 then + throw "lib.fileset.unions: Expected argument to be a list with at least one element, but it contains no elements." + else + _create normalised.commonBase tree; + + /* + The file set containing all files that are in both given sets. + Recursively, the first argument is evaluated first, only evaluating the second argument if necessary. + + intersect a b == a ⋂ b + + Type: + intersect :: FileSet -> FileSet -> FileSet + */ + intersect = lhs: rhs: + let + normalised = _normaliseBase "intersect" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _create normalised.commonBase + (_intersectTree + (elemAt normalised.trees 0) + (elemAt normalised.trees 1) + ); + + /* + The file set containing all files that are in all the given sets. + Recursively, the elements are evaluated from left to right, only evaluating arguments on the right if necessary. + + Type: + intersects :: [FileSet] -> FileSet + */ + intersects = list: + let + annotated = imap0 (i: el: { + context = "element ${toString i} of the argument"; + value = el; + }) list; + + normalised = _normaliseBase "intersects" annotated; + + tree = foldl' _intersectTree (head normalised.trees) (tail normalised.trees); + in + if ! isList list then + throw "lib.fileset.intersects: Expected argument to be a list, but got a ${typeOf list}." + else if length list == 0 then + throw "lib.fileset.intersects: Expected argument to be a list with at least one element, but it contains no elements." + else + _create normalised.commonBase tree; + + /* + The file set containing all files that are in the first file set but not in the second. + Recursively, the second argument is evaluated first, only evaluating the first argument if necessary. + + difference a b == a ∖ b + + Type: + difference :: FileSet -> FileSet -> FileSet + */ + difference = lhs: rhs: + let + normalised = _normaliseBase "difference" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _create normalised.commonBase + (_differenceTree normalised.commonBase + (elemAt normalised.trees 0) + (elemAt normalised.trees 1) + ); + + /* + Filter a file set to only contain files matching some predicate. + + The predicate is called with an attribute set containing these attributes: + + - `name`: The filename + + - `type`: The type of the file, either "regular", "symlink" or "unknown" + + - `ext`: The file extension or `null` if the file has none. + + More formally: + - `ext` contains no `.` + - `.${ext}` is a suffix of the `name` + + - Potentially other attributes in the future + + Type: + fileFilter :: + ({ + name :: String, + type :: String, + ext :: String | Null, + ... + } -> Bool) + -> FileSet + -> FileSet + */ + fileFilter = predicate: maybeFileset: + let + fileset = _coerce "fileFilter" "second argument" maybeFileset; + recurse = focusPath: tree: + mapAttrs (name: subtree: + if isAttrs subtree || subtree == "directory" then + recurse (append focusPath name) subtree + else if + predicate { + inherit name; + type = subtree; + ext = mapNullable head (match ".*\\.(.*)" name); + # To ensure forwards compatibility with more arguments being added in the future, + # adding an attribute which can't be deconstructed :) + "This attribute is passed to prevent `lib.fileset.fileFilter` predicate functions from breaking when more attributes are added in the future. Please add `...` to the function to handle this and future additional arguments." = null; + } + then + subtree + else + null + ) (_directoryEntries focusPath tree); + in + _create fileset._base (recurse fileset._base fileset._tree); + + /* + A file set containing all files that are contained in a directory whose name satisfies the given predicate. + Only directories under the given path are checked, this is to ensure that components outside of the given path cannot influence the result. + Consequently this function does not accept a file set as an argument. + If you need to filter files in a file set based on components, use `intersect myFileSet (directoryFilter myPredicate myPath)` instead. + + Type: + directoryFilter :: (String -> Bool) -> Path -> FileSet + + Example: + # Select all files in hidden directories within ./. + directoryFilter (hasPrefix ".") ./. + + # Select all files in directories named `build` within ./src + directoryFilter (name: name == "build") ./src + */ + directoryFilter = predicate: path: + let + recurse = focusPath: + mapAttrs (name: type: + if type == "directory" then + if predicate name then + type + else + recurse (append focusPath name) + else + null + ) (readDir focusPath); + in + if path._type or null == "fileset" then + throw '' + lib.fileset.directoryFilter: Expected second argument to be a path, but it's a file set. + If you need to filter files in a file set, use `intersect myFileSet (directoryFilter myPredicate myPath)` instead.'' + else if ! isPath path then + throw "lib.fileset.directoryFilter: Expected second argument to be a path, but got a ${typeOf path}." + else if pathType path != "directory" then + throw "lib.fileset.directoryFilter: Expected second argument \"${toString path}\" to be a directory, but it's not." + else + _create path (recurse path); + + /* + Check whether two file sets contain the same files. + + Type: + equals :: FileSet -> FileSet -> Bool + */ + equals = lhs: rhs: + let + normalised = _normaliseBase "equals" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _equalsTree normalised.commonBase + (elemAt normalised.trees 0) + (elemAt normalised.trees 1); + + /* + Check whether a file set contains no files. + + Type: + isEmpty :: FileSet -> Bool + */ + isEmpty = maybeFileset: + let + fileset = _coerce "isEmpty" "argument" maybeFileset; + in + _isEmptyTree fileset._base fileset._tree; +} diff --git a/filesystem.nix b/filesystem.nix new file mode 100644 index 000000000..5d048c0a1 --- /dev/null +++ b/filesystem.nix @@ -0,0 +1,194 @@ +# Functions for querying information about the filesystem +# without copying any files to the Nix store. +{ lib }: + +# Tested in lib/tests/filesystem.sh +let + inherit (builtins) + readDir + pathExists + storeDir + ; + + inherit (lib.trivial) + inPureEvalMode + ; + + inherit (lib.path) + deconstruct + ; + + inherit (lib.path.components) + fromSubpath + ; + + inherit (lib.lists) + take + length + ; + + inherit (lib.filesystem) + pathType + ; +in + +{ + + /* + The type of a path. The path needs to exist and be accessible. + The result is either "directory" for a directory, "regular" for a regular file, "symlink" for a symlink, or "unknown" for anything else. + + Type: + pathType :: Path -> String + + Example: + pathType /. + => "directory" + + pathType /some/file.nix + => "regular" + */ + pathType = + let + storeDirComponents = fromSubpath "./${storeDir}"; + + legacy = path: + if ! pathExists path + # Fail irrecoverably to mimic the historic behavior of this function and + # the new builtins.readFileType + then abort "lib.filesystem.pathType: Path ${toString path} does not exist." + # The filesystem root is the only path where `dirOf / == /` and + # `baseNameOf /` is not valid. We can detect this and directly return + # "directory", since we know the filesystem root can't be anything else. + else if dirOf path == path + then "directory" + else (readDir (dirOf path)).${baseNameOf path}; + + legacyPureEval = path: + let + # This is a workaround for the fact that `lib.filesystem.pathType` doesn't work correctly on /nix/store/...-name paths in pure evaluation mode. For file set combinators it's safe to assume that + pathParts = deconstruct path; + assumeDirectory = + storeDirComponents == take (length pathParts.components - 1) pathParts.components; + in + if assumeDirectory then + "directory" + else + legacy path; + + in + if builtins ? readFileType then + builtins.readFileType + # Nix <2.14 compatibility shim + else if inPureEvalMode then + legacyPureEval + else + legacy; + + /* + Whether a path exists and is a directory. + + Type: + pathIsDirectory :: Path -> Bool + + Example: + pathIsDirectory /. + => true + + pathIsDirectory /this/does/not/exist + => false + + pathIsDirectory /some/file.nix + => false + */ + pathIsDirectory = path: + pathExists path && pathType path == "directory"; + + /* + Whether a path exists and is a regular file, meaning not a symlink or any other special file type. + + Type: + pathIsRegularFile :: Path -> Bool + + Example: + pathIsRegularFile /. + => false + + pathIsRegularFile /this/does/not/exist + => false + + pathIsRegularFile /some/file.nix + => true + */ + pathIsRegularFile = path: + pathExists path && pathType path == "regular"; + + /* + A map of all haskell packages defined in the given path, + identified by having a cabal file with the same name as the + directory itself. + + Type: Path -> Map String Path + */ + haskellPathsInDir = + # The directory within to search + root: + let # Files in the root + root-files = builtins.attrNames (builtins.readDir root); + # Files with their full paths + root-files-with-paths = + map (file: + { name = file; value = root + "/${file}"; } + ) root-files; + # Subdirectories of the root with a cabal file. + cabal-subdirs = + builtins.filter ({ name, value }: + builtins.pathExists (value + "/${name}.cabal") + ) root-files-with-paths; + in builtins.listToAttrs cabal-subdirs; + /* + Find the first directory containing a file matching 'pattern' + upward from a given 'file'. + Returns 'null' if no directories contain a file matching 'pattern'. + + Type: RegExp -> Path -> Nullable { path : Path; matches : [ MatchResults ]; } + */ + locateDominatingFile = + # The pattern to search for + pattern: + # The file to start searching upward from + file: + let go = path: + let files = builtins.attrNames (builtins.readDir path); + matches = builtins.filter (match: match != null) + (map (builtins.match pattern) files); + in + if builtins.length matches != 0 + then { inherit path matches; } + else if path == /. + then null + else go (dirOf path); + parent = dirOf file; + isDir = + let base = baseNameOf file; + type = (builtins.readDir parent).${base} or null; + in file == /. || type == "directory"; + in go (if isDir then file else parent); + + + /* + Given a directory, return a flattened list of all files within it recursively. + + Type: Path -> [ Path ] + */ + listFilesRecursive = + # The path to recursively list + dir: + lib.flatten (lib.mapAttrsToList (name: type: + if type == "directory" then + lib.filesystem.listFilesRecursive (dir + "/${name}") + else + dir + "/${name}" + ) (builtins.readDir dir)); + +} diff --git a/fixed-points.nix b/fixed-points.nix new file mode 100644 index 000000000..926428293 --- /dev/null +++ b/fixed-points.nix @@ -0,0 +1,113 @@ +{ lib, ... }: +rec { + # Compute the fixed point of the given function `f`, which is usually an + # attribute set that expects its final, non-recursive representation as an + # argument: + # + # f = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } + # + # Nix evaluates this recursion until all references to `self` have been + # resolved. At that point, the final result is returned and `f x = x` holds: + # + # nix-repl> fix f + # { bar = "bar"; foo = "foo"; foobar = "foobar"; } + # + # Type: fix :: (a -> a) -> a + # + # See https://en.wikipedia.org/wiki/Fixed-point_combinator for further + # details. + fix = f: let x = f x; in x; + + # A variant of `fix` that records the original recursive attribute set in the + # result. This is useful in combination with the `extends` function to + # implement deep overriding. See pkgs/development/haskell-modules/default.nix + # for a concrete example. + fix' = f: let x = f x // { __unfix__ = f; }; in x; + + # Return the fixpoint that `f` converges to when called recursively, starting + # with the input `x`. + # + # nix-repl> converge (x: x / 2) 16 + # 0 + converge = f: x: + let + x' = f x; + in + if x' == x + then x + else converge f x'; + + # Modify the contents of an explicitly recursive attribute set in a way that + # honors `self`-references. This is accomplished with a function + # + # g = self: super: { foo = super.foo + " + "; } + # + # that has access to the unmodified input (`super`) as well as the final + # non-recursive representation of the attribute set (`self`). `extends` + # differs from the native `//` operator insofar as that it's applied *before* + # references to `self` are resolved: + # + # nix-repl> fix (extends g f) + # { bar = "bar"; foo = "foo + "; foobar = "foo + bar"; } + # + # The name of the function is inspired by object-oriented inheritance, i.e. + # think of it as an infix operator `g extends f` that mimics the syntax from + # Java. It may seem counter-intuitive to have the "base class" as the second + # argument, but it's nice this way if several uses of `extends` are cascaded. + # + # To get a better understanding how `extends` turns a function with a fix + # point (the package set we start with) into a new function with a different fix + # point (the desired packages set) lets just see, how `extends g f` + # unfolds with `g` and `f` defined above: + # + # extends g f = self: let super = f self; in super // g self super; + # = self: let super = { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }; in super // g self super + # = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // g self { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } + # = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // { foo = "foo" + " + "; } + # = self: { foo = "foo + "; bar = "bar"; foobar = self.foo + self.bar; } + # + extends = f: rattrs: self: let super = rattrs self; in super // f self super; + + # Compose two extending functions of the type expected by 'extends' + # into one where changes made in the first are available in the + # 'super' of the second + composeExtensions = + f: g: final: prev: + let fApplied = f final prev; + prev' = prev // fApplied; + in fApplied // g final prev'; + + # Compose several extending functions of the type expected by 'extends' into + # one where changes made in preceding functions are made available to + # subsequent ones. + # + # composeManyExtensions : [packageSet -> packageSet -> packageSet] -> packageSet -> packageSet -> packageSet + # ^final ^prev ^overrides ^final ^prev ^overrides + composeManyExtensions = + lib.foldr (x: y: composeExtensions x y) (final: prev: {}); + + # Create an overridable, recursive attribute set. For example: + # + # nix-repl> obj = makeExtensible (self: { }) + # + # nix-repl> obj + # { __unfix__ = «lambda»; extend = «lambda»; } + # + # nix-repl> obj = obj.extend (self: super: { foo = "foo"; }) + # + # nix-repl> obj + # { __unfix__ = «lambda»; extend = «lambda»; foo = "foo"; } + # + # nix-repl> obj = obj.extend (self: super: { foo = super.foo + " + "; bar = "bar"; foobar = self.foo + self.bar; }) + # + # nix-repl> obj + # { __unfix__ = «lambda»; bar = "bar"; extend = «lambda»; foo = "foo + "; foobar = "foo + bar"; } + makeExtensible = makeExtensibleWithCustomName "extend"; + + # Same as `makeExtensible` but the name of the extending attribute is + # customized. + makeExtensibleWithCustomName = extenderName: rattrs: + fix' (self: (rattrs self) // { + ${extenderName} = f: makeExtensibleWithCustomName extenderName (extends f rattrs); + }); +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..0b5e54d54 --- /dev/null +++ b/flake.nix @@ -0,0 +1,5 @@ +{ + description = "Library of low-level helper functions for nix expressions."; + + outputs = { self }: { lib = import ./.; }; +} diff --git a/generators.nix b/generators.nix new file mode 100644 index 000000000..496845fc9 --- /dev/null +++ b/generators.nix @@ -0,0 +1,526 @@ +/* Functions that generate widespread file + * formats from nix data structures. + * + * They all follow a similar interface: + * generator { config-attrs } data + * + * `config-attrs` are “holes” in the generators + * with sensible default implementations that + * can be overwritten. The default implementations + * are mostly generators themselves, called with + * their respective default values; they can be reused. + * + * Tests can be found in ./tests/misc.nix + * Documentation in the manual, #sec-generators + */ +{ lib }: +with (lib).trivial; +let + libStr = lib.strings; + libAttr = lib.attrsets; + + inherit (lib) isFunction; +in + +rec { + + ## -- HELPER FUNCTIONS & DEFAULTS -- + + /* Convert a value to a sensible default string representation. + * The builtin `toString` function has some strange defaults, + * suitable for bash scripts but not much else. + */ + mkValueStringDefault = {}: v: with builtins; + let err = t: v: abort + ("generators.mkValueStringDefault: " + + "${t} not supported: ${toPretty {} v}"); + in if isInt v then toString v + # convert derivations to store paths + else if lib.isDerivation v then toString v + # we default to not quoting strings + else if isString v then v + # isString returns "1", which is not a good default + else if true == v then "true" + # here it returns to "", which is even less of a good default + else if false == v then "false" + else if null == v then "null" + # if you have lists you probably want to replace this + else if isList v then err "lists" v + # same as for lists, might want to replace + else if isAttrs v then err "attrsets" v + # functions can’t be printed of course + else if isFunction v then err "functions" v + # Floats currently can't be converted to precise strings, + # condition warning on nix version once this isn't a problem anymore + # See https://github.com/NixOS/nix/pull/3480 + else if isFloat v then libStr.floatToString v + else err "this value is" (toString v); + + + /* Generate a line of key k and value v, separated by + * character sep. If sep appears in k, it is escaped. + * Helper for synaxes with different separators. + * + * mkValueString specifies how values should be formatted. + * + * mkKeyValueDefault {} ":" "f:oo" "bar" + * > "f\:oo:bar" + */ + mkKeyValueDefault = { + mkValueString ? mkValueStringDefault {} + }: sep: k: v: + "${libStr.escape [sep] k}${sep}${mkValueString v}"; + + + ## -- FILE FORMAT GENERATORS -- + + + /* Generate a key-value-style config file from an attrset. + * + * mkKeyValue is the same as in toINI. + */ + toKeyValue = { + mkKeyValue ? mkKeyValueDefault {} "=", + listsAsDuplicateKeys ? false + }: + let mkLine = k: v: mkKeyValue k v + "\n"; + mkLines = if listsAsDuplicateKeys + then k: v: map (mkLine k) (if lib.isList v then v else [v]) + else k: v: [ (mkLine k v) ]; + in attrs: libStr.concatStrings (lib.concatLists (libAttr.mapAttrsToList mkLines attrs)); + + + /* Generate an INI-style config file from an + * attrset of sections to an attrset of key-value pairs. + * + * generators.toINI {} { + * foo = { hi = "${pkgs.hello}"; ciao = "bar"; }; + * baz = { "also, integers" = 42; }; + * } + * + *> [baz] + *> also, integers=42 + *> + *> [foo] + *> ciao=bar + *> hi=/nix/store/y93qql1p5ggfnaqjjqhxcw0vqw95rlz0-hello-2.10 + * + * The mk* configuration attributes can generically change + * the way sections and key-value strings are generated. + * + * For more examples see the test cases in ./tests/misc.nix. + */ + toINI = { + # apply transformations (e.g. escapes) to section names + mkSectionName ? (name: libStr.escape [ "[" "]" ] name), + # format a setting line from key and value + mkKeyValue ? mkKeyValueDefault {} "=", + # allow lists as values for duplicate keys + listsAsDuplicateKeys ? false + }: attrsOfAttrs: + let + # map function to string for each key val + mapAttrsToStringsSep = sep: mapFn: attrs: + libStr.concatStringsSep sep + (libAttr.mapAttrsToList mapFn attrs); + mkSection = sectName: sectValues: '' + [${mkSectionName sectName}] + '' + toKeyValue { inherit mkKeyValue listsAsDuplicateKeys; } sectValues; + in + # map input to ini sections + mapAttrsToStringsSep "\n" mkSection attrsOfAttrs; + + /* Generate an INI-style config file from an attrset + * specifying the global section (no header), and an + * attrset of sections to an attrset of key-value pairs. + * + * generators.toINIWithGlobalSection {} { + * globalSection = { + * someGlobalKey = "hi"; + * }; + * sections = { + * foo = { hi = "${pkgs.hello}"; ciao = "bar"; }; + * baz = { "also, integers" = 42; }; + * } + * + *> someGlobalKey=hi + *> + *> [baz] + *> also, integers=42 + *> + *> [foo] + *> ciao=bar + *> hi=/nix/store/y93qql1p5ggfnaqjjqhxcw0vqw95rlz0-hello-2.10 + * + * The mk* configuration attributes can generically change + * the way sections and key-value strings are generated. + * + * For more examples see the test cases in ./tests/misc.nix. + * + * If you don’t need a global section, you can also use + * `generators.toINI` directly, which only takes + * the part in `sections`. + */ + toINIWithGlobalSection = { + # apply transformations (e.g. escapes) to section names + mkSectionName ? (name: libStr.escape [ "[" "]" ] name), + # format a setting line from key and value + mkKeyValue ? mkKeyValueDefault {} "=", + # allow lists as values for duplicate keys + listsAsDuplicateKeys ? false + }: { globalSection, sections }: + ( if globalSection == {} + then "" + else (toKeyValue { inherit mkKeyValue listsAsDuplicateKeys; } globalSection) + + "\n") + + (toINI { inherit mkSectionName mkKeyValue listsAsDuplicateKeys; } sections); + + /* Generate a git-config file from an attrset. + * + * It has two major differences from the regular INI format: + * + * 1. values are indented with tabs + * 2. sections can have sub-sections + * + * generators.toGitINI { + * url."ssh://git@github.com/".insteadOf = "https://github.com"; + * user.name = "edolstra"; + * } + * + *> [url "ssh://git@github.com/"] + *> insteadOf = https://github.com/ + *> + *> [user] + *> name = edolstra + */ + toGitINI = attrs: + with builtins; + let + mkSectionName = name: + let + containsQuote = libStr.hasInfix ''"'' name; + sections = libStr.splitString "." name; + section = head sections; + subsections = tail sections; + subsection = concatStringsSep "." subsections; + in if containsQuote || subsections == [ ] then + name + else + ''${section} "${subsection}"''; + + # generation for multiple ini values + mkKeyValue = k: v: + let mkKeyValue = mkKeyValueDefault { } " = " k; + in concatStringsSep "\n" (map (kv: "\t" + mkKeyValue kv) (lib.toList v)); + + # converts { a.b.c = 5; } to { "a.b".c = 5; } for toINI + gitFlattenAttrs = let + recurse = path: value: + if isAttrs value && !lib.isDerivation value then + lib.mapAttrsToList (name: value: recurse ([ name ] ++ path) value) value + else if length path > 1 then { + ${concatStringsSep "." (lib.reverseList (tail path))}.${head path} = value; + } else { + ${head path} = value; + }; + in attrs: lib.foldl lib.recursiveUpdate { } (lib.flatten (recurse [ ] attrs)); + + toINI_ = toINI { inherit mkKeyValue mkSectionName; }; + in + toINI_ (gitFlattenAttrs attrs); + + /* Generates JSON from an arbitrary (non-function) value. + * For more information see the documentation of the builtin. + */ + toJSON = {}: builtins.toJSON; + + + /* YAML has been a strict superset of JSON since 1.2, so we + * use toJSON. Before it only had a few differences referring + * to implicit typing rules, so it should work with older + * parsers as well. + */ + toYAML = toJSON; + + withRecursion = + { + /* If this option is not null, the given value will stop evaluating at a certain depth */ + depthLimit + /* If this option is true, an error will be thrown, if a certain given depth is exceeded */ + , throwOnDepthLimit ? true + }: + assert builtins.isInt depthLimit; + let + specialAttrs = [ + "__functor" + "__functionArgs" + "__toString" + "__pretty" + ]; + stepIntoAttr = evalNext: name: + if builtins.elem name specialAttrs + then id + else evalNext; + transform = depth: + if depthLimit != null && depth > depthLimit then + if throwOnDepthLimit + then throw "Exceeded maximum eval-depth limit of ${toString depthLimit} while trying to evaluate with `generators.withRecursion'!" + else const "<unevaluated>" + else id; + mapAny = with builtins; depth: v: + let + evalNext = x: mapAny (depth + 1) (transform (depth + 1) x); + in + if isAttrs v then mapAttrs (stepIntoAttr evalNext) v + else if isList v then map evalNext v + else transform (depth + 1) v; + in + mapAny 0; + + /* Pretty print a value, akin to `builtins.trace`. + * Should probably be a builtin as well. + * The pretty-printed string should be suitable for rendering default values + * in the NixOS manual. In particular, it should be as close to a valid Nix expression + * as possible. + */ + toPretty = { + /* If this option is true, attrsets like { __pretty = fn; val = …; } + will use fn to convert val to a pretty printed representation. + (This means fn is type Val -> String.) */ + allowPrettyValues ? false, + /* If this option is true, the output is indented with newlines for attribute sets and lists */ + multiline ? true, + /* Initial indentation level */ + indent ? "" + }: + let + go = indent: v: with builtins; + let isPath = v: typeOf v == "path"; + introSpace = if multiline then "\n${indent} " else " "; + outroSpace = if multiline then "\n${indent}" else " "; + in if isInt v then toString v + # toString loses precision on floats, so we use toJSON instead. This isn't perfect + # as the resulting string may not parse back as a float (e.g. 42, 1e-06), but for + # pretty-printing purposes this is acceptable. + else if isFloat v then builtins.toJSON v + else if isString v then + let + lines = filter (v: ! isList v) (builtins.split "\n" v); + escapeSingleline = libStr.escape [ "\\" "\"" "\${" ]; + escapeMultiline = libStr.replaceStrings [ "\${" "''" ] [ "''\${" "'''" ]; + singlelineResult = "\"" + concatStringsSep "\\n" (map escapeSingleline lines) + "\""; + multilineResult = let + escapedLines = map escapeMultiline lines; + # The last line gets a special treatment: if it's empty, '' is on its own line at the "outer" + # indentation level. Otherwise, '' is appended to the last line. + lastLine = lib.last escapedLines; + in "''" + introSpace + concatStringsSep introSpace (lib.init escapedLines) + + (if lastLine == "" then outroSpace else introSpace + lastLine) + "''"; + in + if multiline && length lines > 1 then multilineResult else singlelineResult + else if true == v then "true" + else if false == v then "false" + else if null == v then "null" + else if isPath v then toString v + else if isList v then + if v == [] then "[ ]" + else "[" + introSpace + + libStr.concatMapStringsSep introSpace (go (indent + " ")) v + + outroSpace + "]" + else if isFunction v then + let fna = lib.functionArgs v; + showFnas = concatStringsSep ", " (libAttr.mapAttrsToList + (name: hasDefVal: if hasDefVal then name + "?" else name) + fna); + in if fna == {} then "<function>" + else "<function, args: {${showFnas}}>" + else if isAttrs v then + # apply pretty values if allowed + if allowPrettyValues && v ? __pretty && v ? val + then v.__pretty v.val + else if v == {} then "{ }" + else if v ? type && v.type == "derivation" then + "<derivation ${v.name or "???"}>" + else "{" + introSpace + + libStr.concatStringsSep introSpace (libAttr.mapAttrsToList + (name: value: + "${libStr.escapeNixIdentifier name} = ${ + builtins.addErrorContext "while evaluating an attribute `${name}`" + (go (indent + " ") value) + };") v) + + outroSpace + "}" + else abort "generators.toPretty: should never happen (v = ${v})"; + in go indent; + + # PLIST handling + toPlist = {}: v: let + isFloat = builtins.isFloat or (x: false); + isPath = x: builtins.typeOf x == "path"; + expr = ind: x: with builtins; + if x == null then "" else + if isBool x then bool ind x else + if isInt x then int ind x else + if isString x then str ind x else + if isList x then list ind x else + if isAttrs x then attrs ind x else + if isPath x then str ind (toString x) else + if isFloat x then float ind x else + abort "generators.toPlist: should never happen (v = ${v})"; + + literal = ind: x: ind + x; + + bool = ind: x: literal ind (if x then "<true/>" else "<false/>"); + int = ind: x: literal ind "<integer>${toString x}</integer>"; + str = ind: x: literal ind "<string>${x}</string>"; + key = ind: x: literal ind "<key>${x}</key>"; + float = ind: x: literal ind "<real>${toString x}</real>"; + + indent = ind: expr "\t${ind}"; + + item = ind: libStr.concatMapStringsSep "\n" (indent ind); + + list = ind: x: libStr.concatStringsSep "\n" [ + (literal ind "<array>") + (item ind x) + (literal ind "</array>") + ]; + + attrs = ind: x: libStr.concatStringsSep "\n" [ + (literal ind "<dict>") + (attr ind x) + (literal ind "</dict>") + ]; + + attr = let attrFilter = name: value: name != "_module" && value != null; + in ind: x: libStr.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList + (name: value: lib.optionals (attrFilter name value) [ + (key "\t${ind}" name) + (expr "\t${ind}" value) + ]) x)); + + in ''<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +${expr "" v} +</plist>''; + + /* Translate a simple Nix expression to Dhall notation. + * Note that integers are translated to Integer and never + * the Natural type. + */ + toDhall = { }@args: v: + with builtins; + let concatItems = lib.strings.concatStringsSep ", "; + in if isAttrs v then + "{ ${ + concatItems (lib.attrsets.mapAttrsToList + (key: value: "${key} = ${toDhall args value}") v) + } }" + else if isList v then + "[ ${concatItems (map (toDhall args) v)} ]" + else if isInt v then + "${if v < 0 then "" else "+"}${toString v}" + else if isBool v then + (if v then "True" else "False") + else if isFunction v then + abort "generators.toDhall: cannot convert a function to Dhall" + else if v == null then + abort "generators.toDhall: cannot convert a null to Dhall" + else + builtins.toJSON v; + + /* + Translate a simple Nix expression to Lua representation with occasional + Lua-inlines that can be constructed by mkLuaInline function. + + Configuration: + * multiline - by default is true which results in indented block-like view. + * indent - initial indent. + * asBindings - by default generate single value, but with this use attrset to set global vars. + + Attention: + Regardless of multiline parameter there is no trailing newline. + + Example: + generators.toLua {} + { + cmd = [ "typescript-language-server" "--stdio" ]; + settings.workspace.library = mkLuaInline ''vim.api.nvim_get_runtime_file("", true)''; + } + -> + { + ["cmd"] = { + "typescript-language-server", + "--stdio" + }, + ["settings"] = { + ["workspace"] = { + ["library"] = (vim.api.nvim_get_runtime_file("", true)) + } + } + } + + Type: + toLua :: AttrSet -> Any -> String + */ + toLua = { + /* If this option is true, the output is indented with newlines for attribute sets and lists */ + multiline ? true, + /* Initial indentation level */ + indent ? "", + /* Interpret as variable bindings */ + asBindings ? false, + }@args: v: + with builtins; + let + innerIndent = "${indent} "; + introSpace = if multiline then "\n${innerIndent}" else " "; + outroSpace = if multiline then "\n${indent}" else " "; + innerArgs = args // { + indent = if asBindings then indent else innerIndent; + asBindings = false; + }; + concatItems = concatStringsSep ",${introSpace}"; + isLuaInline = { _type ? null, ... }: _type == "lua-inline"; + + generatedBindings = + assert lib.assertMsg (badVarNames == []) "Bad Lua var names: ${toPretty {} badVarNames}"; + libStr.concatStrings ( + lib.attrsets.mapAttrsToList (key: value: "${indent}${key} = ${toLua innerArgs value}\n") v + ); + + # https://en.wikibooks.org/wiki/Lua_Programming/variable#Variable_names + matchVarName = match "[[:alpha:]_][[:alnum:]_]*(\\.[[:alpha:]_][[:alnum:]_]*)*"; + badVarNames = filter (name: matchVarName name == null) (attrNames v); + in + if asBindings then + generatedBindings + else if v == null then + "nil" + else if isInt v || isFloat v || isString v || isBool v then + builtins.toJSON v + else if isList v then + (if v == [ ] then "{}" else + "{${introSpace}${concatItems (map (value: "${toLua innerArgs value}") v)}${outroSpace}}") + else if isAttrs v then + ( + if isLuaInline v then + "(${v.expr})" + else if v == { } then + "{}" + else + "{${introSpace}${concatItems ( + lib.attrsets.mapAttrsToList (key: value: "[${builtins.toJSON key}] = ${toLua innerArgs value}") v + )}${outroSpace}}" + ) + else + abort "generators.toLua: type ${typeOf v} is unsupported"; + + /* + Mark string as Lua expression to be inlined when processed by toLua. + + Type: + mkLuaInline :: String -> AttrSet + */ + mkLuaInline = expr: { _type = "lua-inline"; inherit expr; }; +} diff --git a/kernel.nix b/kernel.nix new file mode 100644 index 000000000..33da9663a --- /dev/null +++ b/kernel.nix @@ -0,0 +1,27 @@ +{ lib }: + +with lib; +{ + + + # Keeping these around in case we decide to change this horrible implementation :) + option = x: + x // { optional = true; }; + + yes = { tristate = "y"; optional = false; }; + no = { tristate = "n"; optional = false; }; + module = { tristate = "m"; optional = false; }; + unset = { tristate = null; optional = false; }; + freeform = x: { freeform = x; optional = false; }; + + /* + Common patterns/legacy used in common-config/hardened/config.nix + */ + whenHelpers = version: { + whenAtLeast = ver: mkIf (versionAtLeast version ver); + whenOlder = ver: mkIf (versionOlder version ver); + # range is (inclusive, exclusive) + whenBetween = verLow: verHigh: mkIf (versionAtLeast version verLow && versionOlder version verHigh); + }; + +} diff --git a/licenses.nix b/licenses.nix new file mode 100644 index 000000000..1d2565fba --- /dev/null +++ b/licenses.nix @@ -0,0 +1,1104 @@ +{ lib }: + +lib.mapAttrs (lname: lset: let + defaultLicense = rec { + shortName = lname; + free = true; # Most of our licenses are Free, explicitly declare unfree additions as such! + deprecated = false; + }; + + mkLicense = licenseDeclaration: let + applyDefaults = license: defaultLicense // license; + applySpdx = license: + if license ? spdxId + then license // { url = "https://spdx.org/licenses/${license.spdxId}.html"; } + else license; + applyRedistributable = license: { redistributable = license.free; } // license; + in lib.pipe licenseDeclaration [ + applyDefaults + applySpdx + applyRedistributable + ]; +in mkLicense lset) ({ + /* License identifiers from spdx.org where possible. + * If you cannot find your license here, then look for a similar license or + * add it to this list. The URL mentioned above is a good source for inspiration. + */ + + abstyles = { + spdxId = "Abstyles"; + fullName = "Abstyles License"; + }; + + afl20 = { + spdxId = "AFL-2.0"; + fullName = "Academic Free License v2.0"; + }; + + afl21 = { + spdxId = "AFL-2.1"; + fullName = "Academic Free License v2.1"; + }; + + afl3 = { + spdxId = "AFL-3.0"; + fullName = "Academic Free License v3.0"; + }; + + agpl3Only = { + spdxId = "AGPL-3.0-only"; + fullName = "GNU Affero General Public License v3.0 only"; + }; + + agpl3Plus = { + spdxId = "AGPL-3.0-or-later"; + fullName = "GNU Affero General Public License v3.0 or later"; + }; + + aladdin = { + spdxId = "Aladdin"; + fullName = "Aladdin Free Public License"; + free = false; + }; + + amazonsl = { + fullName = "Amazon Software License"; + url = "https://aws.amazon.com/asl/"; + free = false; + }; + + amd = { + fullName = "AMD License Agreement"; + url = "https://developer.amd.com/amd-license-agreement/"; + free = false; + }; + + aom = { + fullName = "Alliance for Open Media Patent License 1.0"; + url = "https://aomedia.org/license/patent-license/"; + }; + + apsl10 = { + spdxId = "APSL-1.0"; + fullName = "Apple Public Source License 1.0"; + }; + + apsl20 = { + spdxId = "APSL-2.0"; + fullName = "Apple Public Source License 2.0"; + }; + + arphicpl = { + fullName = "Arphic Public License"; + url = "https://www.freedesktop.org/wiki/Arphic_Public_License/"; + }; + + artistic1 = { + spdxId = "Artistic-1.0"; + fullName = "Artistic License 1.0"; + }; + + artistic2 = { + spdxId = "Artistic-2.0"; + fullName = "Artistic License 2.0"; + }; + + asl20 = { + spdxId = "Apache-2.0"; + fullName = "Apache License 2.0"; + }; + + asl20-llvm = { + spdxId = "Apache-2.0 WITH LLVM-exception"; + fullName = "Apache License 2.0 with LLVM Exceptions"; + }; + + bitstreamVera = { + spdxId = "Bitstream-Vera"; + fullName = "Bitstream Vera Font License"; + }; + + bitTorrent10 = { + spdxId = "BitTorrent-1.0"; + fullName = " BitTorrent Open Source License v1.0"; + }; + + bitTorrent11 = { + spdxId = "BitTorrent-1.1"; + fullName = " BitTorrent Open Source License v1.1"; + }; + + bola11 = { + url = "https://blitiri.com.ar/p/bola/"; + fullName = "Buena Onda License Agreement 1.1"; + }; + + boost = { + spdxId = "BSL-1.0"; + fullName = "Boost Software License 1.0"; + }; + + beerware = { + spdxId = "Beerware"; + fullName = "Beerware License"; + }; + + blueOak100 = { + spdxId = "BlueOak-1.0.0"; + fullName = "Blue Oak Model License 1.0.0"; + }; + + bsd0 = { + spdxId = "0BSD"; + fullName = "BSD Zero Clause License"; + }; + + bsd1 = { + spdxId = "BSD-1-Clause"; + fullName = "BSD 1-Clause License"; + }; + + bsd2 = { + spdxId = "BSD-2-Clause"; + fullName = ''BSD 2-clause "Simplified" License''; + }; + + bsd2Patent = { + spdxId = "BSD-2-Clause-Patent"; + fullName = "BSD-2-Clause Plus Patent License"; + }; + + bsd2WithViews = { + spdxId = "BSD-2-Clause-Views"; + fullName = "BSD 2-Clause with views sentence"; + }; + + bsd3 = { + spdxId = "BSD-3-Clause"; + fullName = ''BSD 3-clause "New" or "Revised" License''; + }; + + bsdOriginal = { + spdxId = "BSD-4-Clause"; + fullName = ''BSD 4-clause "Original" or "Old" License''; + }; + + bsdOriginalShortened = { + spdxId = "BSD-4-Clause-Shortened"; + fullName = "BSD 4 Clause Shortened"; + }; + + bsdOriginalUC = { + spdxId = "BSD-4-Clause-UC"; + fullName = "BSD 4-Clause University of California-Specific"; + }; + + bsdProtection = { + spdxId = "BSD-Protection"; + fullName = "BSD Protection License"; + }; + + bsl11 = { + fullName = "Business Source License 1.1"; + url = "https://mariadb.com/bsl11"; + free = false; + redistributable = true; + }; + + caossl = { + fullName = "Computer Associates Open Source Licence Version 1.0"; + url = "http://jxplorer.org/licence.html"; + }; + + cal10 = { + fullName = "Cryptographic Autonomy License version 1.0 (CAL-1.0)"; + url = "https://opensource.org/licenses/CAL-1.0"; + }; + + caldera = { + spdxId = "Caldera"; + fullName = "Caldera License"; + url = "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf"; + }; + + capec = { + fullName = "Common Attack Pattern Enumeration and Classification"; + url = "https://capec.mitre.org/about/termsofuse.html"; + }; + + clArtistic = { + spdxId = "ClArtistic"; + fullName = "Clarified Artistic License"; + }; + + cc0 = { + spdxId = "CC0-1.0"; + fullName = "Creative Commons Zero v1.0 Universal"; + }; + + cc-by-nc-nd-30 = { + spdxId = "CC-BY-NC-ND-3.0"; + fullName = "Creative Commons Attribution Non Commercial No Derivative Works 3.0 Unported"; + free = false; + }; + + cc-by-nc-nd-40 = { + spdxId = "CC-BY-NC-ND-4.0"; + fullName = "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International"; + free = false; + }; + + cc-by-nc-sa-20 = { + spdxId = "CC-BY-NC-SA-2.0"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 2.0"; + free = false; + }; + + cc-by-nc-sa-25 = { + spdxId = "CC-BY-NC-SA-2.5"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 2.5"; + free = false; + }; + + cc-by-nc-sa-30 = { + spdxId = "CC-BY-NC-SA-3.0"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 3.0"; + free = false; + }; + + cc-by-nc-sa-40 = { + spdxId = "CC-BY-NC-SA-4.0"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 4.0"; + free = false; + }; + + cc-by-nc-30 = { + spdxId = "CC-BY-NC-3.0"; + fullName = "Creative Commons Attribution Non Commercial 3.0 Unported"; + free = false; + }; + + cc-by-nc-40 = { + spdxId = "CC-BY-NC-4.0"; + fullName = "Creative Commons Attribution Non Commercial 4.0 International"; + free = false; + }; + + cc-by-nd-30 = { + spdxId = "CC-BY-ND-3.0"; + fullName = "Creative Commons Attribution-No Derivative Works v3.00"; + free = false; + }; + + cc-by-sa-25 = { + spdxId = "CC-BY-SA-2.5"; + fullName = "Creative Commons Attribution Share Alike 2.5"; + }; + + cc-by-30 = { + spdxId = "CC-BY-3.0"; + fullName = "Creative Commons Attribution 3.0"; + }; + + cc-by-sa-30 = { + spdxId = "CC-BY-SA-3.0"; + fullName = "Creative Commons Attribution Share Alike 3.0"; + }; + + cc-by-40 = { + spdxId = "CC-BY-4.0"; + fullName = "Creative Commons Attribution 4.0"; + }; + + cc-by-sa-40 = { + spdxId = "CC-BY-SA-4.0"; + fullName = "Creative Commons Attribution Share Alike 4.0"; + }; + + cddl = { + spdxId = "CDDL-1.0"; + fullName = "Common Development and Distribution License 1.0"; + }; + + cecill20 = { + spdxId = "CECILL-2.0"; + fullName = "CeCILL Free Software License Agreement v2.0"; + }; + + cecill21 = { + spdxId = "CECILL-2.1"; + fullName = "CeCILL Free Software License Agreement v2.1"; + }; + + cecill-b = { + spdxId = "CECILL-B"; + fullName = "CeCILL-B Free Software License Agreement"; + }; + + cecill-c = { + spdxId = "CECILL-C"; + fullName = "CeCILL-C Free Software License Agreement"; + }; + + cpal10 = { + spdxId = "CPAL-1.0"; + fullName = "Common Public Attribution License 1.0"; + }; + + cpl10 = { + spdxId = "CPL-1.0"; + fullName = "Common Public License 1.0"; + }; + + curl = { + spdxId = "curl"; + fullName = "curl License"; + }; + + doc = { + spdxId = "DOC"; + fullName = "DOC License"; + }; + + drl10 = { + spdxId = "DRL-1.0"; + fullName = "Detection Rule License 1.0"; + }; + + eapl = { + fullName = "EPSON AVASYS PUBLIC LICENSE"; + url = "https://avasys.jp/hp/menu000000700/hpg000000603.htm"; + free = false; + }; + + ecl20 = { + fullName = "Educational Community License, Version 2.0"; + url = "https://opensource.org/licenses/ECL-2.0"; + shortName = "ECL 2.0"; + spdxId = "ECL-2.0"; + }; + + efl10 = { + spdxId = "EFL-1.0"; + fullName = "Eiffel Forum License v1.0"; + }; + + efl20 = { + spdxId = "EFL-2.0"; + fullName = "Eiffel Forum License v2.0"; + }; + + elastic = { + fullName = "ELASTIC LICENSE"; + url = "https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt"; + free = false; + }; + + epl10 = { + spdxId = "EPL-1.0"; + fullName = "Eclipse Public License 1.0"; + }; + + epl20 = { + spdxId = "EPL-2.0"; + fullName = "Eclipse Public License 2.0"; + }; + + epson = { + fullName = "Seiko Epson Corporation Software License Agreement for Linux"; + url = "https://download.ebz.epson.net/dsc/du/02/eula/global/LINUX_EN.html"; + free = false; + }; + + eupl11 = { + spdxId = "EUPL-1.1"; + fullName = "European Union Public License 1.1"; + }; + + eupl12 = { + spdxId = "EUPL-1.2"; + fullName = "European Union Public License 1.2"; + }; + + fdl11Only = { + spdxId = "GFDL-1.1-only"; + fullName = "GNU Free Documentation License v1.1 only"; + }; + + fdl11Plus = { + spdxId = "GFDL-1.1-or-later"; + fullName = "GNU Free Documentation License v1.1 or later"; + }; + + fdl12Only = { + spdxId = "GFDL-1.2-only"; + fullName = "GNU Free Documentation License v1.2 only"; + }; + + fdl12Plus = { + spdxId = "GFDL-1.2-or-later"; + fullName = "GNU Free Documentation License v1.2 or later"; + }; + + fdl13Only = { + spdxId = "GFDL-1.3-only"; + fullName = "GNU Free Documentation License v1.3 only"; + }; + + fdl13Plus = { + spdxId = "GFDL-1.3-or-later"; + fullName = "GNU Free Documentation License v1.3 or later"; + }; + + ffsl = { + fullName = "Floodgap Free Software License"; + url = "https://www.floodgap.com/software/ffsl/license.html"; + free = false; + }; + + free = { + fullName = "Unspecified free software license"; + }; + + ftl = { + spdxId = "FTL"; + fullName = "Freetype Project License"; + }; + + g4sl = { + fullName = "Geant4 Software License"; + url = "https://geant4.web.cern.ch/geant4/license/LICENSE.html"; + }; + + geogebra = { + fullName = "GeoGebra Non-Commercial License Agreement"; + url = "https://www.geogebra.org/license"; + free = false; + }; + + generaluser = { + fullName = "GeneralUser GS License v2.0"; + url = "http://www.schristiancollins.com/generaluser.php"; # license included in sources + }; + + gpl1Only = { + spdxId = "GPL-1.0-only"; + fullName = "GNU General Public License v1.0 only"; + }; + + gpl1Plus = { + spdxId = "GPL-1.0-or-later"; + fullName = "GNU General Public License v1.0 or later"; + }; + + gpl2Only = { + spdxId = "GPL-2.0-only"; + fullName = "GNU General Public License v2.0 only"; + }; + + gpl2Classpath = { + spdxId = "GPL-2.0-with-classpath-exception"; + fullName = "GNU General Public License v2.0 only (with Classpath exception)"; + }; + + gpl2ClasspathPlus = { + fullName = "GNU General Public License v2.0 or later (with Classpath exception)"; + url = "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception"; + }; + + gpl2Oss = { + fullName = "GNU General Public License version 2 only (with OSI approved licenses linking exception)"; + url = "https://www.mysql.com/about/legal/licensing/foss-exception"; + }; + + gpl2Plus = { + spdxId = "GPL-2.0-or-later"; + fullName = "GNU General Public License v2.0 or later"; + }; + + gpl3Only = { + spdxId = "GPL-3.0-only"; + fullName = "GNU General Public License v3.0 only"; + }; + + gpl3Plus = { + spdxId = "GPL-3.0-or-later"; + fullName = "GNU General Public License v3.0 or later"; + }; + + gpl3ClasspathPlus = { + fullName = "GNU General Public License v3.0 or later (with Classpath exception)"; + url = "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception"; + }; + + hpnd = { + spdxId = "HPND"; + fullName = "Historic Permission Notice and Disclaimer"; + }; + + hpndSellVariant = { + fullName = "Historical Permission Notice and Disclaimer - sell variant"; + spdxId = "HPND-sell-variant"; + }; + + # Intel's license, seems free + iasl = { + fullName = "iASL"; + url = "https://old.calculate-linux.org/packages/licenses/iASL"; + }; + + ijg = { + spdxId = "IJG"; + fullName = "Independent JPEG Group License"; + }; + + imagemagick = { + fullName = "ImageMagick License"; + spdxId = "imagemagick"; + }; + + imlib2 = { + spdxId = "Imlib2"; + fullName = "Imlib2 License"; + }; + + info-zip = { + spdxId = "Info-ZIP"; + fullName = "Info-ZIP License"; + url = "http://www.info-zip.org/pub/infozip/license.html"; + }; + + inria-compcert = { + fullName = "INRIA Non-Commercial License Agreement for the CompCert verified compiler"; + url = "https://compcert.org/doc/LICENSE.txt"; + free = false; + }; + + inria-icesl = { + fullName = "INRIA Non-Commercial License Agreement for IceSL"; + url = "https://icesl.loria.fr/assets/pdf/EULA_IceSL_binary.pdf"; + free = false; + }; + + ipa = { + spdxId = "IPA"; + fullName = "IPA Font License"; + }; + + ipl10 = { + spdxId = "IPL-1.0"; + fullName = "IBM Public License v1.0"; + }; + + isc = { + spdxId = "ISC"; + fullName = "ISC License"; + }; + + # Proprietary binaries; free to redistribute without modification. + databricks = { + fullName = "Databricks Proprietary License"; + url = "https://pypi.org/project/databricks-connect"; + free = false; + }; + + databricks-dbx = { + fullName = "DataBricks eXtensions aka dbx License"; + url = "https://github.com/databrickslabs/dbx/blob/743b579a4ac44531f764c6e522dbe5a81a7dc0e4/LICENSE"; + free = false; + redistributable = false; + }; + + fair = { + fullName = "Fair License"; + spdxId = "Fair"; + free = true; + }; + + issl = { + fullName = "Intel Simplified Software License"; + url = "https://software.intel.com/en-us/license/intel-simplified-software-license"; + free = false; + }; + + lal12 = { + spdxId = "LAL-1.2"; + fullName = "Licence Art Libre 1.2"; + }; + + lal13 = { + spdxId = "LAL-1.3"; + fullName = "Licence Art Libre 1.3"; + }; + + lens = { + fullName = "Lens Terms of Service Agreement"; + url = "https://k8slens.dev/licenses/tos"; + free = false; + }; + + lgpl2Only = { + spdxId = "LGPL-2.0-only"; + fullName = "GNU Library General Public License v2 only"; + }; + + lgpl2Plus = { + spdxId = "LGPL-2.0-or-later"; + fullName = "GNU Library General Public License v2 or later"; + }; + + lgpl21Only = { + spdxId = "LGPL-2.1-only"; + fullName = "GNU Lesser General Public License v2.1 only"; + }; + + lgpl21Plus = { + spdxId = "LGPL-2.1-or-later"; + fullName = "GNU Lesser General Public License v2.1 or later"; + }; + + lgpl3Only = { + spdxId = "LGPL-3.0-only"; + fullName = "GNU Lesser General Public License v3.0 only"; + }; + + lgpl3Plus = { + spdxId = "LGPL-3.0-or-later"; + fullName = "GNU Lesser General Public License v3.0 or later"; + }; + + lgpllr = { + spdxId = "LGPLLR"; + fullName = "Lesser General Public License For Linguistic Resources"; + }; + + libpng = { + spdxId = "Libpng"; + fullName = "libpng License"; + }; + + libpng2 = { + spdxId = "libpng-2.0"; # Used since libpng 1.6.36. + fullName = "PNG Reference Library version 2"; + }; + + libssh2 = { + fullName = "libssh2 License"; + url = "https://www.libssh2.org/license.html"; + }; + + libtiff = { + spdxId = "libtiff"; + fullName = "libtiff License"; + }; + + llgpl21 = { + fullName = "Lisp LGPL; GNU Lesser General Public License version 2.1 with Franz Inc. preamble for clarification of LGPL terms in context of Lisp"; + url = "https://opensource.franz.com/preamble.html"; + }; + + lppl12 = { + spdxId = "LPPL-1.2"; + fullName = "LaTeX Project Public License v1.2"; + }; + + lppl13c = { + spdxId = "LPPL-1.3c"; + fullName = "LaTeX Project Public License v1.3c"; + }; + + lpl-102 = { + spdxId = "LPL-1.02"; + fullName = "Lucent Public License v1.02"; + }; + + miros = { + fullName = "MirOS License"; + url = "https://opensource.org/licenses/MirOS"; + }; + + # spdx.org does not (yet) differentiate between the X11 and Expat versions + # for details see https://en.wikipedia.org/wiki/MIT_License#Various_versions + mit = { + spdxId = "MIT"; + fullName = "MIT License"; + }; + # https://spdx.org/licenses/MIT-feh.html + mit-feh = { + spdxId = "MIT-feh"; + fullName = "feh License"; + }; + + mitAdvertising = { + spdxId = "MIT-advertising"; + fullName = "Enlightenment License (e16)"; + }; + + mit0 = { + spdxId = "MIT-0"; + fullName = "MIT No Attribution"; + }; + + mpl10 = { + spdxId = "MPL-1.0"; + fullName = "Mozilla Public License 1.0"; + }; + + mpl11 = { + spdxId = "MPL-1.1"; + fullName = "Mozilla Public License 1.1"; + }; + + mpl20 = { + spdxId = "MPL-2.0"; + fullName = "Mozilla Public License 2.0"; + }; + + mspl = { + spdxId = "MS-PL"; + fullName = "Microsoft Public License"; + }; + + nasa13 = { + spdxId = "NASA-1.3"; + fullName = "NASA Open Source Agreement 1.3"; + free = false; + }; + + ncsa = { + spdxId = "NCSA"; + fullName = "University of Illinois/NCSA Open Source License"; + }; + + nlpl = { + spdxId = "NLPL"; + fullName = "No Limit Public License"; + }; + + nposl3 = { + spdxId = "NPOSL-3.0"; + fullName = "Non-Profit Open Software License 3.0"; + }; + + obsidian = { + fullName = "Obsidian End User Agreement"; + url = "https://obsidian.md/eula"; + free = false; + }; + + ocamlpro_nc = { + fullName = "OCamlPro Non Commercial license version 1"; + url = "https://alt-ergo.ocamlpro.com/http/alt-ergo-2.2.0/OCamlPro-Non-Commercial-License.pdf"; + free = false; + }; + + odbl = { + spdxId = "ODbL-1.0"; + fullName = "Open Data Commons Open Database License v1.0"; + }; + + ofl = { + spdxId = "OFL-1.1"; + fullName = "SIL Open Font License 1.1"; + }; + + oml = { + spdxId = "OML"; + fullName = "Open Market License"; + }; + + openldap = { + spdxId = "OLDAP-2.8"; + fullName = "Open LDAP Public License v2.8"; + }; + + openssl = { + spdxId = "OpenSSL"; + fullName = "OpenSSL License"; + }; + + osl2 = { + spdxId = "OSL-2.0"; + fullName = "Open Software License 2.0"; + }; + + osl21 = { + spdxId = "OSL-2.1"; + fullName = "Open Software License 2.1"; + }; + + osl3 = { + spdxId = "OSL-3.0"; + fullName = "Open Software License 3.0"; + }; + + parity70 = { + spdxId = "Parity-7.0.0"; + fullName = "Parity Public License 7.0.0"; + url = "https://paritylicense.com/versions/7.0.0.html"; + }; + + php301 = { + spdxId = "PHP-3.01"; + fullName = "PHP License v3.01"; + }; + + postgresql = { + spdxId = "PostgreSQL"; + fullName = "PostgreSQL License"; + }; + + postman = { + fullName = "Postman EULA"; + url = "https://www.getpostman.com/licenses/postman_base_app"; + free = false; + }; + + psfl = { + spdxId = "Python-2.0"; + fullName = "Python Software Foundation License version 2"; + url = "https://docs.python.org/license.html"; + }; + + publicDomain = { + fullName = "Public Domain"; + }; + + purdueBsd = { + fullName = " Purdue BSD-Style License"; # also know as lsof license + url = "https://enterprise.dejacode.com/licenses/public/purdue-bsd"; + }; + + prosperity30 = { + fullName = "Prosperity-3.0.0"; + free = false; + url = "https://prosperitylicense.com/versions/3.0.0.html"; + }; + + qhull = { + spdxId = "Qhull"; + fullName = "Qhull License"; + }; + + qpl = { + spdxId = "QPL-1.0"; + fullName = "Q Public License 1.0"; + }; + + qwt = { + fullName = "Qwt License, Version 1.0"; + url = "https://qwt.sourceforge.io/qwtlicense.html"; + }; + + ruby = { + spdxId = "Ruby"; + fullName = "Ruby License"; + }; + + sendmail = { + spdxId = "Sendmail"; + fullName = "Sendmail License"; + }; + + sgi-b-20 = { + spdxId = "SGI-B-2.0"; + fullName = "SGI Free Software License B v2.0"; + }; + + # Gentoo seems to treat it as a license: + # https://gitweb.gentoo.org/repo/gentoo.git/tree/licenses/SGMLUG?id=7d999af4a47bf55e53e54713d98d145f935935c1 + sgmlug = { + fullName = "SGML UG SGML Parser Materials license"; + }; + + sleepycat = { + spdxId = "Sleepycat"; + fullName = "Sleepycat License"; + }; + + smail = { + shortName = "smail"; + fullName = "SMAIL General Public License"; + url = "https://sources.debian.org/copyright/license/debianutils/4.9.1/"; + }; + + sspl = { + shortName = "SSPL"; + fullName = "Server Side Public License"; + url = "https://www.mongodb.com/licensing/server-side-public-license"; + free = false; + # NOTE Debatable. + # The license a slightly modified AGPL but still considered unfree by the + # OSI for what seem like political reasons + redistributable = true; # Definitely redistributable though, it's an AGPL derivative + }; + + stk = { + shortName = "stk"; + fullName = "Synthesis Tool Kit 4.3"; + url = "https://github.com/thestk/stk/blob/master/LICENSE"; + }; + + tsl = { + shortName = "TSL"; + fullName = "Timescale License Agreegment"; + url = "https://github.com/timescale/timescaledb/blob/main/tsl/LICENSE-TIMESCALE"; + unfree = true; + }; + + tcltk = { + spdxId = "TCL"; + fullName = "TCL/TK License"; + }; + + ucd = { + fullName = "Unicode Character Database License"; + url = "https://fedoraproject.org/wiki/Licensing:UCD"; + }; + + ufl = { + fullName = "Ubuntu Font License 1.0"; + url = "https://ubuntu.com/legal/font-licence"; + }; + + unfree = { + fullName = "Unfree"; + free = false; + }; + + unfreeRedistributable = { + fullName = "Unfree redistributable"; + free = false; + redistributable = true; + }; + + unfreeRedistributableFirmware = { + fullName = "Unfree redistributable firmware"; + redistributable = true; + # Note: we currently consider these "free" for inclusion in the + # channel and NixOS images. + }; + + unicode-dfs-2015 = { + spdxId = "Unicode-DFS-2015"; + fullName = "Unicode License Agreement - Data Files and Software (2015)"; + }; + + unicode-dfs-2016 = { + spdxId = "Unicode-DFS-2016"; + fullName = "Unicode License Agreement - Data Files and Software (2016)"; + }; + + unlicense = { + spdxId = "Unlicense"; + fullName = "The Unlicense"; + }; + + upl = { + fullName = "Universal Permissive License"; + url = "https://oss.oracle.com/licenses/upl/"; + }; + + vim = { + spdxId = "Vim"; + fullName = "Vim License"; + }; + + virtualbox-puel = { + fullName = "Oracle VM VirtualBox Extension Pack Personal Use and Evaluation License (PUEL)"; + url = "https://www.virtualbox.org/wiki/VirtualBox_PUEL"; + free = false; + }; + + vol-sl = { + fullName = "Volatility Software License, Version 1.0"; + url = "https://www.volatilityfoundation.org/license/vsl-v1.0"; + }; + + vsl10 = { + spdxId = "VSL-1.0"; + fullName = "Vovida Software License v1.0"; + }; + + watcom = { + spdxId = "Watcom-1.0"; + fullName = "Sybase Open Watcom Public License 1.0"; + }; + + w3c = { + spdxId = "W3C"; + fullName = "W3C Software Notice and License"; + }; + + wadalab = { + fullName = "Wadalab Font License"; + url = "https://fedoraproject.org/wiki/Licensing:Wadalab?rd=Licensing/Wadalab"; + }; + + wtfpl = { + spdxId = "WTFPL"; + fullName = "Do What The F*ck You Want To Public License"; + }; + + wxWindows = { + spdxId = "wxWindows"; + fullName = "wxWindows Library Licence, Version 3.1"; + }; + + x11 = { + spdxId = "X11"; + fullName = "X11 License"; + }; + + xfig = { + fullName = "xfig"; + url = "http://mcj.sourceforge.net/authors.html#xfig"; # https is broken + }; + + zlib = { + spdxId = "Zlib"; + fullName = "zlib License"; + }; + + zpl20 = { + spdxId = "ZPL-2.0"; + fullName = "Zope Public License 2.0"; + }; + + zpl21 = { + spdxId = "ZPL-2.1"; + fullName = "Zope Public License 2.1"; + }; +} // { + # TODO: remove legacy aliases + agpl3 = { + spdxId = "AGPL-3.0"; + fullName = "GNU Affero General Public License v3.0"; + deprecated = true; + }; + gpl2 = { + spdxId = "GPL-2.0"; + fullName = "GNU General Public License v2.0"; + deprecated = true; + }; + gpl3 = { + spdxId = "GPL-3.0"; + fullName = "GNU General Public License v3.0"; + deprecated = true; + }; + lgpl2 = { + spdxId = "LGPL-2.0"; + fullName = "GNU Library General Public License v2"; + deprecated = true; + }; + lgpl21 = { + spdxId = "LGPL-2.1"; + fullName = "GNU Lesser General Public License v2.1"; + deprecated = true; + }; + lgpl3 = { + spdxId = "LGPL-3.0"; + fullName = "GNU Lesser General Public License v3.0"; + deprecated = true; + }; +}) diff --git a/lists.nix b/lists.nix new file mode 100644 index 000000000..bea9bf43c --- /dev/null +++ b/lists.nix @@ -0,0 +1,724 @@ +# General list operations. + +{ lib }: +let + inherit (lib.strings) toInt; + inherit (lib.trivial) compare min; + inherit (lib.attrsets) mapAttrs; +in +rec { + + inherit (builtins) head tail length isList elemAt concatLists filter elem genList map; + + /* Create a list consisting of a single element. `singleton x` is + sometimes more convenient with respect to indentation than `[x]` + when x spans multiple lines. + + Type: singleton :: a -> [a] + + Example: + singleton "foo" + => [ "foo" ] + */ + singleton = x: [x]; + + /* Apply the function to each element in the list. Same as `map`, but arguments + flipped. + + Type: forEach :: [a] -> (a -> b) -> [b] + + Example: + forEach [ 1 2 ] (x: + toString x + ) + => [ "1" "2" ] + */ + forEach = xs: f: map f xs; + + /* “right fold” a binary function `op` between successive elements of + `list` with `nul` as the starting value, i.e., + `foldr op nul [x_1 x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))`. + + Type: foldr :: (a -> b -> b) -> b -> [a] -> b + + Example: + concat = foldr (a: b: a + b) "z" + concat [ "a" "b" "c" ] + => "abcz" + # different types + strange = foldr (int: str: toString (int + 1) + str) "a" + strange [ 1 2 3 4 ] + => "2345a" + */ + foldr = op: nul: list: + let + len = length list; + fold' = n: + if n == len + then nul + else op (elemAt list n) (fold' (n + 1)); + in fold' 0; + + /* `fold` is an alias of `foldr` for historic reasons */ + # FIXME(Profpatsch): deprecate? + fold = foldr; + + + /* “left fold”, like `foldr`, but from the left: + `foldl op nul [x_1 x_2 ... x_n] == op (... (op (op nul x_1) x_2) ... x_n)`. + + Type: foldl :: (b -> a -> b) -> b -> [a] -> b + + Example: + lconcat = foldl (a: b: a + b) "z" + lconcat [ "a" "b" "c" ] + => "zabc" + # different types + lstrange = foldl (str: int: str + toString (int + 1)) "a" + lstrange [ 1 2 3 4 ] + => "a2345" + */ + foldl = op: nul: list: + let + foldl' = n: + if n == -1 + then nul + else op (foldl' (n - 1)) (elemAt list n); + in foldl' (length list - 1); + + /* Strict version of `foldl`. + + The difference is that evaluation is forced upon access. Usually used + with small whole results (in contrast with lazily-generated list or large + lists where only a part is consumed.) + + Type: foldl' :: (b -> a -> b) -> b -> [a] -> b + */ + foldl' = builtins.foldl' or foldl; + + /* Map with index starting from 0 + + Type: imap0 :: (int -> a -> b) -> [a] -> [b] + + Example: + imap0 (i: v: "${v}-${toString i}") ["a" "b"] + => [ "a-0" "b-1" ] + */ + imap0 = f: list: genList (n: f n (elemAt list n)) (length list); + + /* Map with index starting from 1 + + Type: imap1 :: (int -> a -> b) -> [a] -> [b] + + Example: + imap1 (i: v: "${v}-${toString i}") ["a" "b"] + => [ "a-1" "b-2" ] + */ + imap1 = f: list: genList (n: f (n + 1) (elemAt list n)) (length list); + + /* Map and concatenate the result. + + Type: concatMap :: (a -> [b]) -> [a] -> [b] + + Example: + concatMap (x: [x] ++ ["z"]) ["a" "b"] + => [ "a" "z" "b" "z" ] + */ + concatMap = builtins.concatMap or (f: list: concatLists (map f list)); + + /* Flatten the argument into a single list; that is, nested lists are + spliced into the top-level lists. + + Example: + flatten [1 [2 [3] 4] 5] + => [1 2 3 4 5] + flatten 1 + => [1] + */ + flatten = x: + if isList x + then concatMap (y: flatten y) x + else [x]; + + /* Remove elements equal to 'e' from a list. Useful for buildInputs. + + Type: remove :: a -> [a] -> [a] + + Example: + remove 3 [ 1 3 4 3 ] + => [ 1 4 ] + */ + remove = + # Element to remove from the list + e: filter (x: x != e); + + /* Find the sole element in the list matching the specified + predicate, returns `default` if no such element exists, or + `multiple` if there are multiple matching elements. + + Type: findSingle :: (a -> bool) -> a -> a -> [a] -> a + + Example: + findSingle (x: x == 3) "none" "multiple" [ 1 3 3 ] + => "multiple" + findSingle (x: x == 3) "none" "multiple" [ 1 3 ] + => 3 + findSingle (x: x == 3) "none" "multiple" [ 1 9 ] + => "none" + */ + findSingle = + # Predicate + pred: + # Default value to return if element was not found. + default: + # Default value to return if more than one element was found + multiple: + # Input list + list: + let found = filter pred list; len = length found; + in if len == 0 then default + else if len != 1 then multiple + else head found; + + /* Find the first element in the list matching the specified + predicate or return `default` if no such element exists. + + Type: findFirst :: (a -> bool) -> a -> [a] -> a + + Example: + findFirst (x: x > 3) 7 [ 1 6 4 ] + => 6 + findFirst (x: x > 9) 7 [ 1 6 4 ] + => 7 + */ + findFirst = + # Predicate + pred: + # Default value to return + default: + # Input list + list: + let + # A naive recursive implementation would be much simpler, but + # would also overflow the evaluator stack. We use `foldl'` as a workaround + # because it reuses the same stack space, evaluating the function for one + # element after another. We can't return early, so this means that we + # sacrifice early cutoff, but that appears to be an acceptable cost. A + # clever scheme with "exponential search" is possible, but appears over- + # engineered for now. See https://github.com/NixOS/nixpkgs/pull/235267 + + # Invariant: + # - if index < 0 then el == elemAt list (- index - 1) and all elements before el didn't satisfy pred + # - if index >= 0 then pred (elemAt list index) and all elements before (elemAt list index) didn't satisfy pred + # + # We start with index -1 and the 0'th element of the list, which satisfies the invariant + resultIndex = foldl' (index: el: + if index < 0 then + # No match yet before the current index, we need to check the element + if pred el then + # We have a match! Turn it into the actual index to prevent future iterations from modifying it + - index - 1 + else + # Still no match, update the index to the next element (we're counting down, so minus one) + index - 1 + else + # There's already a match, propagate the index without evaluating anything + index + ) (-1) list; + in + if resultIndex < 0 then + default + else + elemAt list resultIndex; + + /* Return true if function `pred` returns true for at least one + element of `list`. + + Type: any :: (a -> bool) -> [a] -> bool + + Example: + any isString [ 1 "a" { } ] + => true + any isString [ 1 { } ] + => false + */ + any = builtins.any or (pred: foldr (x: y: if pred x then true else y) false); + + /* Return true if function `pred` returns true for all elements of + `list`. + + Type: all :: (a -> bool) -> [a] -> bool + + Example: + all (x: x < 3) [ 1 2 ] + => true + all (x: x < 3) [ 1 2 3 ] + => false + */ + all = builtins.all or (pred: foldr (x: y: if pred x then y else false) true); + + /* Count how many elements of `list` match the supplied predicate + function. + + Type: count :: (a -> bool) -> [a] -> int + + Example: + count (x: x == 3) [ 3 2 3 4 6 ] + => 2 + */ + count = + # Predicate + pred: foldl' (c: x: if pred x then c + 1 else c) 0; + + /* Return a singleton list or an empty list, depending on a boolean + value. Useful when building lists with optional elements + (e.g. `++ optional (system == "i686-linux") firefox`). + + Type: optional :: bool -> a -> [a] + + Example: + optional true "foo" + => [ "foo" ] + optional false "foo" + => [ ] + */ + optional = cond: elem: if cond then [elem] else []; + + /* Return a list or an empty list, depending on a boolean value. + + Type: optionals :: bool -> [a] -> [a] + + Example: + optionals true [ 2 3 ] + => [ 2 3 ] + optionals false [ 2 3 ] + => [ ] + */ + optionals = + # Condition + cond: + # List to return if condition is true + elems: if cond then elems else []; + + + /* If argument is a list, return it; else, wrap it in a singleton + list. If you're using this, you should almost certainly + reconsider if there isn't a more "well-typed" approach. + + Example: + toList [ 1 2 ] + => [ 1 2 ] + toList "hi" + => [ "hi "] + */ + toList = x: if isList x then x else [x]; + + /* Return a list of integers from `first` up to and including `last`. + + Type: range :: int -> int -> [int] + + Example: + range 2 4 + => [ 2 3 4 ] + range 3 2 + => [ ] + */ + range = + # First integer in the range + first: + # Last integer in the range + last: + if first > last then + [] + else + genList (n: first + n) (last - first + 1); + + /* Return a list with `n` copies of an element. + + Type: replicate :: int -> a -> [a] + + Example: + replicate 3 "a" + => [ "a" "a" "a" ] + replicate 2 true + => [ true true ] + */ + replicate = n: elem: genList (_: elem) n; + + /* Splits the elements of a list in two lists, `right` and + `wrong`, depending on the evaluation of a predicate. + + Type: (a -> bool) -> [a] -> { right :: [a]; wrong :: [a]; } + + Example: + partition (x: x > 2) [ 5 1 2 3 4 ] + => { right = [ 5 3 4 ]; wrong = [ 1 2 ]; } + */ + partition = builtins.partition or (pred: + foldr (h: t: + if pred h + then { right = [h] ++ t.right; wrong = t.wrong; } + else { right = t.right; wrong = [h] ++ t.wrong; } + ) { right = []; wrong = []; }); + + /* Splits the elements of a list into many lists, using the return value of a predicate. + Predicate should return a string which becomes keys of attrset `groupBy` returns. + + `groupBy'` allows to customise the combining function and initial value + + Example: + groupBy (x: boolToString (x > 2)) [ 5 1 2 3 4 ] + => { true = [ 5 3 4 ]; false = [ 1 2 ]; } + groupBy (x: x.name) [ {name = "icewm"; script = "icewm &";} + {name = "xfce"; script = "xfce4-session &";} + {name = "icewm"; script = "icewmbg &";} + {name = "mate"; script = "gnome-session &";} + ] + => { icewm = [ { name = "icewm"; script = "icewm &"; } + { name = "icewm"; script = "icewmbg &"; } ]; + mate = [ { name = "mate"; script = "gnome-session &"; } ]; + xfce = [ { name = "xfce"; script = "xfce4-session &"; } ]; + } + + groupBy' builtins.add 0 (x: boolToString (x > 2)) [ 5 1 2 3 4 ] + => { true = 12; false = 3; } + */ + groupBy' = op: nul: pred: lst: mapAttrs (name: foldl op nul) (groupBy pred lst); + + groupBy = builtins.groupBy or ( + pred: foldl' (r: e: + let + key = pred e; + in + r // { ${key} = (r.${key} or []) ++ [e]; } + ) {}); + + /* Merges two lists of the same size together. If the sizes aren't the same + the merging stops at the shortest. How both lists are merged is defined + by the first argument. + + Type: zipListsWith :: (a -> b -> c) -> [a] -> [b] -> [c] + + Example: + zipListsWith (a: b: a + b) ["h" "l"] ["e" "o"] + => ["he" "lo"] + */ + zipListsWith = + # Function to zip elements of both lists + f: + # First list + fst: + # Second list + snd: + genList + (n: f (elemAt fst n) (elemAt snd n)) (min (length fst) (length snd)); + + /* Merges two lists of the same size together. If the sizes aren't the same + the merging stops at the shortest. + + Type: zipLists :: [a] -> [b] -> [{ fst :: a; snd :: b; }] + + Example: + zipLists [ 1 2 ] [ "a" "b" ] + => [ { fst = 1; snd = "a"; } { fst = 2; snd = "b"; } ] + */ + zipLists = zipListsWith (fst: snd: { inherit fst snd; }); + + /* Reverse the order of the elements of a list. + + Type: reverseList :: [a] -> [a] + + Example: + + reverseList [ "b" "o" "j" ] + => [ "j" "o" "b" ] + */ + reverseList = xs: + let l = length xs; in genList (n: elemAt xs (l - n - 1)) l; + + /* Depth-First Search (DFS) for lists `list != []`. + + `before a b == true` means that `b` depends on `a` (there's an + edge from `b` to `a`). + + Example: + listDfs true hasPrefix [ "/home/user" "other" "/" "/home" ] + == { minimal = "/"; # minimal element + visited = [ "/home/user" ]; # seen elements (in reverse order) + rest = [ "/home" "other" ]; # everything else + } + + listDfs true hasPrefix [ "/home/user" "other" "/" "/home" "/" ] + == { cycle = "/"; # cycle encountered at this element + loops = [ "/" ]; # and continues to these elements + visited = [ "/" "/home/user" ]; # elements leading to the cycle (in reverse order) + rest = [ "/home" "other" ]; # everything else + + */ + listDfs = stopOnCycles: before: list: + let + dfs' = us: visited: rest: + let + c = filter (x: before x us) visited; + b = partition (x: before x us) rest; + in if stopOnCycles && (length c > 0) + then { cycle = us; loops = c; inherit visited rest; } + else if length b.right == 0 + then # nothing is before us + { minimal = us; inherit visited rest; } + else # grab the first one before us and continue + dfs' (head b.right) + ([ us ] ++ visited) + (tail b.right ++ b.wrong); + in dfs' (head list) [] (tail list); + + /* Sort a list based on a partial ordering using DFS. This + implementation is O(N^2), if your ordering is linear, use `sort` + instead. + + `before a b == true` means that `b` should be after `a` + in the result. + + Example: + + toposort hasPrefix [ "/home/user" "other" "/" "/home" ] + == { result = [ "/" "/home" "/home/user" "other" ]; } + + toposort hasPrefix [ "/home/user" "other" "/" "/home" "/" ] + == { cycle = [ "/home/user" "/" "/" ]; # path leading to a cycle + loops = [ "/" ]; } # loops back to these elements + + toposort hasPrefix [ "other" "/home/user" "/home" "/" ] + == { result = [ "other" "/" "/home" "/home/user" ]; } + + toposort (a: b: a < b) [ 3 2 1 ] == { result = [ 1 2 3 ]; } + + */ + toposort = before: list: + let + dfsthis = listDfs true before list; + toporest = toposort before (dfsthis.visited ++ dfsthis.rest); + in + if length list < 2 + then # finish + { result = list; } + else if dfsthis ? cycle + then # there's a cycle, starting from the current vertex, return it + { cycle = reverseList ([ dfsthis.cycle ] ++ dfsthis.visited); + inherit (dfsthis) loops; } + else if toporest ? cycle + then # there's a cycle somewhere else in the graph, return it + toporest + # Slow, but short. Can be made a bit faster with an explicit stack. + else # there are no cycles + { result = [ dfsthis.minimal ] ++ toporest.result; }; + + /* Sort a list based on a comparator function which compares two + elements and returns true if the first argument is strictly below + the second argument. The returned list is sorted in an increasing + order. The implementation does a quick-sort. + + Example: + sort (a: b: a < b) [ 5 3 7 ] + => [ 3 5 7 ] + */ + sort = builtins.sort or ( + strictLess: list: + let + len = length list; + first = head list; + pivot' = n: acc@{ left, right }: let el = elemAt list n; next = pivot' (n + 1); in + if n == len + then acc + else if strictLess first el + then next { inherit left; right = [ el ] ++ right; } + else + next { left = [ el ] ++ left; inherit right; }; + pivot = pivot' 1 { left = []; right = []; }; + in + if len < 2 then list + else (sort strictLess pivot.left) ++ [ first ] ++ (sort strictLess pivot.right)); + + /* Compare two lists element-by-element. + + Example: + compareLists compare [] [] + => 0 + compareLists compare [] [ "a" ] + => -1 + compareLists compare [ "a" ] [] + => 1 + compareLists compare [ "a" "b" ] [ "a" "c" ] + => -1 + */ + compareLists = cmp: a: b: + if a == [] + then if b == [] + then 0 + else -1 + else if b == [] + then 1 + else let rel = cmp (head a) (head b); in + if rel == 0 + then compareLists cmp (tail a) (tail b) + else rel; + + /* Sort list using "Natural sorting". + Numeric portions of strings are sorted in numeric order. + + Example: + naturalSort ["disk11" "disk8" "disk100" "disk9"] + => ["disk8" "disk9" "disk11" "disk100"] + naturalSort ["10.46.133.149" "10.5.16.62" "10.54.16.25"] + => ["10.5.16.62" "10.46.133.149" "10.54.16.25"] + naturalSort ["v0.2" "v0.15" "v0.0.9"] + => [ "v0.0.9" "v0.2" "v0.15" ] + */ + naturalSort = lst: + let + vectorise = s: map (x: if isList x then toInt (head x) else x) (builtins.split "(0|[1-9][0-9]*)" s); + prepared = map (x: [ (vectorise x) x ]) lst; # remember vectorised version for O(n) regex splits + less = a: b: (compareLists compare (head a) (head b)) < 0; + in + map (x: elemAt x 1) (sort less prepared); + + /* Return the first (at most) N elements of a list. + + Type: take :: int -> [a] -> [a] + + Example: + take 2 [ "a" "b" "c" "d" ] + => [ "a" "b" ] + take 2 [ ] + => [ ] + */ + take = + # Number of elements to take + count: sublist 0 count; + + /* Remove the first (at most) N elements of a list. + + Type: drop :: int -> [a] -> [a] + + Example: + drop 2 [ "a" "b" "c" "d" ] + => [ "c" "d" ] + drop 2 [ ] + => [ ] + */ + drop = + # Number of elements to drop + count: + # Input list + list: sublist count (length list) list; + + commonPrefixLength = lhs: rhs: + let + minLength = min (length lhs) (length rhs); + recurse = index: + if index >= minLength || elemAt lhs index != elemAt rhs index then + index + else + recurse (index + 1); + in recurse 0; + + commonPrefix = lhs: rhs: + take (commonPrefixLength lhs rhs) lhs; + + /* Return a list consisting of at most `count` elements of `list`, + starting at index `start`. + + Type: sublist :: int -> int -> [a] -> [a] + + Example: + sublist 1 3 [ "a" "b" "c" "d" "e" ] + => [ "b" "c" "d" ] + sublist 1 3 [ ] + => [ ] + */ + sublist = + # Index at which to start the sublist + start: + # Number of elements to take + count: + # Input list + list: + let len = length list; in + genList + (n: elemAt list (n + start)) + (if start >= len then 0 + else if start + count > len then len - start + else count); + + /* Return the last element of a list. + + This function throws an error if the list is empty. + + Type: last :: [a] -> a + + Example: + last [ 1 2 3 ] + => 3 + */ + last = list: + assert lib.assertMsg (list != []) "lists.last: list must not be empty!"; + elemAt list (length list - 1); + + /* Return all elements but the last. + + This function throws an error if the list is empty. + + Type: init :: [a] -> [a] + + Example: + init [ 1 2 3 ] + => [ 1 2 ] + */ + init = list: + assert lib.assertMsg (list != []) "lists.init: list must not be empty!"; + take (length list - 1) list; + + + /* Return the image of the cross product of some lists by a function. + + Example: + crossLists (x:y: "${toString x}${toString y}") [[1 2] [3 4]] + => [ "13" "14" "23" "24" ] + */ + crossLists = builtins.trace + "lib.crossLists is deprecated, use lib.cartesianProductOfSets instead" + (f: foldl (fs: args: concatMap (f: map f args) fs) [f]); + + + /* Remove duplicate elements from the list. O(n^2) complexity. + + Type: unique :: [a] -> [a] + + Example: + unique [ 3 2 3 4 ] + => [ 3 2 4 ] + */ + unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) []; + + /* Intersects list 'e' and another list. O(nm) complexity. + + Example: + intersectLists [ 1 2 3 ] [ 6 3 2 ] + => [ 3 2 ] + */ + intersectLists = e: filter (x: elem x e); + + /* Subtracts list 'e' from another list. O(nm) complexity. + + Example: + subtractLists [ 3 2 ] [ 1 2 3 4 5 3 ] + => [ 1 4 5 ] + */ + subtractLists = e: filter (x: !(elem x e)); + + /* Test if two lists have no common element. + It should be slightly more efficient than (intersectLists a b == []) + */ + mutuallyExclusive = a: b: length a == 0 || !(any (x: elem x a) b); + +} diff --git a/meta.nix b/meta.nix new file mode 100644 index 000000000..5fd55c4e9 --- /dev/null +++ b/meta.nix @@ -0,0 +1,148 @@ +/* Some functions for manipulating meta attributes, as well as the + name attribute. */ + +{ lib }: + +rec { + + + /* Add to or override the meta attributes of the given + derivation. + + Example: + addMetaAttrs {description = "Bla blah";} somePkg + */ + addMetaAttrs = newAttrs: drv: + drv // { meta = (drv.meta or {}) // newAttrs; }; + + + /* Disable Hydra builds of given derivation. + */ + dontDistribute = drv: addMetaAttrs { hydraPlatforms = []; } drv; + + + /* Change the symbolic name of a package for presentation purposes + (i.e., so that nix-env users can tell them apart). + */ + setName = name: drv: drv // {inherit name;}; + + + /* Like `setName`, but takes the previous name as an argument. + + Example: + updateName (oldName: oldName + "-experimental") somePkg + */ + updateName = updater: drv: drv // {name = updater (drv.name);}; + + + /* Append a suffix to the name of a package (before the version + part). */ + appendToName = suffix: updateName (name: + let x = builtins.parseDrvName name; in "${x.name}-${suffix}-${x.version}"); + + + /* Apply a function to each derivation and only to derivations in an attrset. + */ + mapDerivationAttrset = f: set: lib.mapAttrs (name: pkg: if lib.isDerivation pkg then (f pkg) else pkg) set; + + /* Set the nix-env priority of the package. + */ + setPrio = priority: addMetaAttrs { inherit priority; }; + + /* Decrease the nix-env priority of the package, i.e., other + versions/variants of the package will be preferred. + */ + lowPrio = setPrio 10; + + /* Apply lowPrio to an attrset with derivations + */ + lowPrioSet = set: mapDerivationAttrset lowPrio set; + + + /* Increase the nix-env priority of the package, i.e., this + version/variant of the package will be preferred. + */ + hiPrio = setPrio (-10); + + /* Apply hiPrio to an attrset with derivations + */ + hiPrioSet = set: mapDerivationAttrset hiPrio set; + + + /* Check to see if a platform is matched by the given `meta.platforms` + element. + + A `meta.platform` pattern is either + + 1. (legacy) a system string. + + 2. (modern) a pattern for the entire platform structure (see `lib.systems.inspect.platformPatterns`). + + 3. (modern) a pattern for the platform `parsed` field (see `lib.systems.inspect.patterns`). + + We can inject these into a pattern for the whole of a structured platform, + and then match that. + */ + platformMatch = platform: elem: let + pattern = + if builtins.isString elem + then { system = elem; } + else if elem?parsed + then elem + else { parsed = elem; }; + in lib.matchAttrs pattern platform; + + /* Check if a package is available on a given platform. + + A package is available on a platform if both + + 1. One of `meta.platforms` pattern matches the given + platform, or `meta.platforms` is not present. + + 2. None of `meta.badPlatforms` pattern matches the given platform. + */ + availableOn = platform: pkg: + ((!pkg?meta.platforms) || lib.any (platformMatch platform) pkg.meta.platforms) && + lib.all (elem: !platformMatch platform elem) (pkg.meta.badPlatforms or []); + + /* Get the corresponding attribute in lib.licenses + from the SPDX ID. + For SPDX IDs, see + https://spdx.org/licenses + + Type: + getLicenseFromSpdxId :: str -> AttrSet + + Example: + lib.getLicenseFromSpdxId "MIT" == lib.licenses.mit + => true + lib.getLicenseFromSpdxId "mIt" == lib.licenses.mit + => true + lib.getLicenseFromSpdxId "MY LICENSE" + => trace: warning: getLicenseFromSpdxId: No license matches the given SPDX ID: MY LICENSE + => { shortName = "MY LICENSE"; } + */ + getLicenseFromSpdxId = + let + spdxLicenses = lib.mapAttrs (id: ls: assert lib.length ls == 1; builtins.head ls) + (lib.groupBy (l: lib.toLower l.spdxId) (lib.filter (l: l ? spdxId) (lib.attrValues lib.licenses))); + in licstr: + spdxLicenses.${ lib.toLower licstr } or ( + lib.warn "getLicenseFromSpdxId: No license matches the given SPDX ID: ${licstr}" + { shortName = licstr; } + ); + + /* Get the path to the main program of a derivation with either + meta.mainProgram or pname or name + + Type: getExe :: derivation -> string + + Example: + getExe pkgs.hello + => "/nix/store/g124820p9hlv4lj8qplzxw1c44dxaw1k-hello-2.12/bin/hello" + getExe pkgs.mustache-go + => "/nix/store/am9ml4f4ywvivxnkiaqwr0hyxka1xjsf-mustache-go-1.3.0/bin/mustache" + */ + getExe = x: + "${lib.getBin x}/bin/${x.meta.mainProgram or (lib.getName x)}"; +} diff --git a/minver.nix b/minver.nix new file mode 100644 index 000000000..507d45bba --- /dev/null +++ b/minver.nix @@ -0,0 +1,2 @@ +# Expose the minimum required version for evaluating Nixpkgs +"2.3" diff --git a/modules.nix b/modules.nix new file mode 100644 index 000000000..4dc8c663b --- /dev/null +++ b/modules.nix @@ -0,0 +1,1287 @@ +{ lib }: + +let + inherit (lib) + all + any + attrByPath + attrNames + catAttrs + concatLists + concatMap + concatStringsSep + elem + filter + foldl' + getAttrFromPath + head + id + imap1 + isAttrs + isBool + isFunction + isList + isPath + isString + length + mapAttrs + mapAttrsToList + mapAttrsRecursiveCond + min + optional + optionalAttrs + optionalString + recursiveUpdate + reverseList sort + setAttrByPath + types + warnIf + zipAttrsWith + ; + inherit (lib.options) + isOption + mkOption + showDefs + showFiles + showOption + unknownModule + ; + inherit (lib.strings) + isConvertibleWithToString + ; + + showDeclPrefix = loc: decl: prefix: + " - option(s) with prefix `${showOption (loc ++ [prefix])}' in module `${decl._file}'"; + showRawDecls = loc: decls: + concatStringsSep "\n" + (sort (a: b: a < b) + (concatMap + (decl: map + (showDeclPrefix loc decl) + (attrNames decl.options) + ) + decls + )); + + /* See https://nixos.org/manual/nixpkgs/unstable/#module-system-lib-evalModules + or file://./../doc/module-system/module-system.chapter.md + + !!! Please think twice before adding to this argument list! The more + that is specified here instead of in the modules themselves the harder + it is to transparently move a set of modules to be a submodule of another + config (as the proper arguments need to be replicated at each call to + evalModules) and the less declarative the module set is. */ + evalModules = evalModulesArgs@ + { modules + , prefix ? [] + , # This should only be used for special arguments that need to be evaluated + # when resolving module structure (like in imports). For everything else, + # there's _module.args. If specialArgs.modulesPath is defined it will be + # used as the base path for disabledModules. + specialArgs ? {} + , # `class`: + # A nominal type for modules. When set and non-null, this adds a check to + # make sure that only compatible modules are imported. + # This would be remove in the future, Prefer _module.args option instead. + class ? null + , args ? {} + , # This would be remove in the future, Prefer _module.check option instead. + check ? true + }: + let + withWarnings = x: + lib.warnIf (evalModulesArgs?args) "The args argument to evalModules is deprecated. Please set config._module.args instead." + lib.warnIf (evalModulesArgs?check) "The check argument to evalModules is deprecated. Please set config._module.check instead." + x; + + legacyModules = + optional (evalModulesArgs?args) { + config = { + _module.args = args; + }; + } + ++ optional (evalModulesArgs?check) { + config = { + _module.check = mkDefault check; + }; + }; + regularModules = modules ++ legacyModules; + + # This internal module declare internal options under the `_module' + # attribute. These options are fragile, as they are used by the + # module system to change the interpretation of modules. + # + # When extended with extendModules or moduleType, a fresh instance of + # this module is used, to avoid conflicts and allow chaining of + # extendModules. + internalModule = rec { + _file = "lib/modules.nix"; + + key = _file; + + options = { + _module.args = mkOption { + # Because things like `mkIf` are entirely useless for + # `_module.args` (because there's no way modules can check which + # arguments were passed), we'll use `lazyAttrsOf` which drops + # support for that, in turn it's lazy in its values. This means e.g. + # a `_module.args.pkgs = import (fetchTarball { ... }) {}` won't + # start a download when `pkgs` wasn't evaluated. + type = types.lazyAttrsOf types.raw; + # Only render documentation once at the root of the option tree, + # not for all individual submodules. + # Allow merging option decls to make this internal regardless. + ${if prefix == [] + then null # unset => visible + else "internal"} = true; + # TODO: hidden during the markdown transition to not expose downstream + # users of the docs infra to markdown if they're not ready for it. + # we don't make this visible conditionally because it can impact + # performance (https://github.com/NixOS/nixpkgs/pull/208407#issuecomment-1368246192) + visible = false; + # TODO: Change the type of this option to a submodule with a + # freeformType, so that individual arguments can be documented + # separately + description = lib.mdDoc '' + Additional arguments passed to each module in addition to ones + like `lib`, `config`, + and `pkgs`, `modulesPath`. + + This option is also available to all submodules. Submodules do not + inherit args from their parent module, nor do they provide args to + their parent module or sibling submodules. The sole exception to + this is the argument `name` which is provided by + parent modules to a submodule and contains the attribute name + the submodule is bound to, or a unique generated name if it is + not bound to an attribute. + + Some arguments are already passed by default, of which the + following *cannot* be changed with this option: + - {var}`lib`: The nixpkgs library. + - {var}`config`: The results of all options after merging the values from all modules together. + - {var}`options`: The options declared in all modules. + - {var}`specialArgs`: The `specialArgs` argument passed to `evalModules`. + - All attributes of {var}`specialArgs` + + Whereas option values can generally depend on other option values + thanks to laziness, this does not apply to `imports`, which + must be computed statically before anything else. + + For this reason, callers of the module system can provide `specialArgs` + which are available during import resolution. + + For NixOS, `specialArgs` includes + {var}`modulesPath`, which allows you to import + extra modules from the nixpkgs package tree without having to + somehow make the module aware of the location of the + `nixpkgs` or NixOS directories. + ``` + { modulesPath, ... }: { + imports = [ + (modulesPath + "/profiles/minimal.nix") + ]; + } + ``` + + For NixOS, the default value for this option includes at least this argument: + - {var}`pkgs`: The nixpkgs package set according to + the {option}`nixpkgs.pkgs` option. + ''; + }; + + _module.check = mkOption { + type = types.bool; + internal = true; + default = true; + description = lib.mdDoc "Whether to check whether all option definitions have matching declarations."; + }; + + _module.freeformType = mkOption { + type = types.nullOr types.optionType; + internal = true; + default = null; + description = lib.mdDoc '' + If set, merge all definitions that don't have an associated option + together using this type. The result then gets combined with the + values of all declared options to produce the final ` + config` value. + + If this is `null`, definitions without an option + will throw an error unless {option}`_module.check` is + turned off. + ''; + }; + + _module.specialArgs = mkOption { + readOnly = true; + internal = true; + description = lib.mdDoc '' + Externally provided module arguments that can't be modified from + within a configuration, but can be used in module imports. + ''; + }; + }; + + config = { + _module.args = { + inherit extendModules; + moduleType = type; + }; + _module.specialArgs = specialArgs; + }; + }; + + merged = + let collected = collectModules + class + (specialArgs.modulesPath or "") + (regularModules ++ [ internalModule ]) + ({ inherit lib options config specialArgs; } // specialArgs); + in mergeModules prefix (reverseList collected); + + options = merged.matchedOptions; + + config = + let + + # For definitions that have an associated option + declaredConfig = mapAttrsRecursiveCond (v: ! isOption v) (_: v: v.value) options; + + # If freeformType is set, this is for definitions that don't have an associated option + freeformConfig = + let + defs = map (def: { + file = def.file; + value = setAttrByPath def.prefix def.value; + }) merged.unmatchedDefns; + in if defs == [] then {} + else declaredConfig._module.freeformType.merge prefix defs; + + in if declaredConfig._module.freeformType == null then declaredConfig + # Because all definitions that had an associated option ended in + # declaredConfig, freeformConfig can only contain the non-option + # paths, meaning recursiveUpdate will never override any value + else recursiveUpdate freeformConfig declaredConfig; + + checkUnmatched = + if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then + let + firstDef = head merged.unmatchedDefns; + baseMsg = + let + optText = showOption (prefix ++ firstDef.prefix); + defText = + builtins.addErrorContext + "while evaluating the error message for definitions for `${optText}', which is an option that does not exist" + (builtins.addErrorContext + "while evaluating a definition from `${firstDef.file}'" + ( showDefs [ firstDef ]) + ); + in + "The option `${optText}' does not exist. Definition values:${defText}"; + in + if attrNames options == [ "_module" ] + then + let + optionName = showOption prefix; + in + if optionName == "" + then throw '' + ${baseMsg} + + It seems as if you're trying to declare an option by placing it into `config' rather than `options'! + '' + else + throw '' + ${baseMsg} + + However there are no options defined in `${showOption prefix}'. Are you sure you've + declared your options properly? This can happen if you e.g. declared your options in `types.submodule' + under `config' rather than `options'. + '' + else throw baseMsg + else null; + + checked = builtins.seq checkUnmatched; + + extendModules = extendArgs@{ + modules ? [], + specialArgs ? {}, + prefix ? [], + }: + evalModules (evalModulesArgs // { + inherit class; + modules = regularModules ++ modules; + specialArgs = evalModulesArgs.specialArgs or {} // specialArgs; + prefix = extendArgs.prefix or evalModulesArgs.prefix or []; + }); + + type = lib.types.submoduleWith { + inherit modules specialArgs class; + }; + + result = withWarnings { + _type = "configuration"; + options = checked options; + config = checked (removeAttrs config [ "_module" ]); + _module = checked (config._module); + inherit extendModules type; + class = class; + }; + in result; + + # collectModules :: (class: String) -> (modulesPath: String) -> (modules: [ Module ]) -> (args: Attrs) -> [ Module ] + # + # Collects all modules recursively through `import` statements, filtering out + # all modules in disabledModules. + collectModules = class: let + + # Like unifyModuleSyntax, but also imports paths and calls functions if necessary + loadModule = args: fallbackFile: fallbackKey: m: + if isFunction m then + unifyModuleSyntax fallbackFile fallbackKey (applyModuleArgs fallbackKey m args) + else if isAttrs m then + if m._type or "module" == "module" then + unifyModuleSyntax fallbackFile fallbackKey m + else if m._type == "if" || m._type == "override" then + loadModule args fallbackFile fallbackKey { config = m; } + else + throw ( + "Could not load a value as a module, because it is of type ${lib.strings.escapeNixString m._type}" + + lib.optionalString (fallbackFile != unknownModule) ", in file ${toString fallbackFile}." + + lib.optionalString (m._type == "configuration") " If you do intend to import this configuration, please only import the modules that make up the configuration. You may have to create a `let` binding, file or attribute to give yourself access to the relevant modules.\nWhile loading a configuration into the module system is a very sensible idea, it can not be done cleanly in practice." + # Extended explanation: That's because a finalized configuration is more than just a set of modules. For instance, it has its own `specialArgs` that, by the nature of `specialArgs` can't be loaded through `imports` or the the `modules` argument. So instead, we have to ask you to extract the relevant modules and use those instead. This way, we keep the module system comparatively simple, and hopefully avoid a bad surprise down the line. + ) + else if isList m then + let defs = [{ file = fallbackFile; value = m; }]; in + throw "Module imports can't be nested lists. Perhaps you meant to remove one level of lists? Definitions: ${showDefs defs}" + else unifyModuleSyntax (toString m) (toString m) (applyModuleArgsIfFunction (toString m) (import m) args); + + checkModule = + if class != null + then + m: + if m._class != null -> m._class == class + then m + else + throw "The module ${m._file or m.key} was imported into ${class} instead of ${m._class}." + else + m: m; + + /* + Collects all modules recursively into the form + + { + disabled = [ <list of disabled modules> ]; + # All modules of the main module list + modules = [ + { + key = <key1>; + module = <module for key1>; + # All modules imported by the module for key1 + modules = [ + { + key = <key1-1>; + module = <module for key1-1>; + # All modules imported by the module for key1-1 + modules = [ ... ]; + } + ... + ]; + } + ... + ]; + } + */ + collectStructuredModules = + let + collectResults = modules: { + disabled = concatLists (catAttrs "disabled" modules); + inherit modules; + }; + in parentFile: parentKey: initialModules: args: collectResults (imap1 (n: x: + let + module = checkModule (loadModule args parentFile "${parentKey}:anon-${toString n}" x); + collectedImports = collectStructuredModules module._file module.key module.imports args; + in { + key = module.key; + module = module; + modules = collectedImports.modules; + disabled = (if module.disabledModules != [] then [{ file = module._file; disabled = module.disabledModules; }] else []) ++ collectedImports.disabled; + }) initialModules); + + # filterModules :: String -> { disabled, modules } -> [ Module ] + # + # Filters a structure as emitted by collectStructuredModules by removing all disabled + # modules recursively. It returns the final list of unique-by-key modules + filterModules = modulesPath: { disabled, modules }: + let + moduleKey = file: m: + if isString m + then + if builtins.substring 0 1 m == "/" + then m + else toString modulesPath + "/" + m + + else if isConvertibleWithToString m + then + if m?key && m.key != toString m + then + throw "Module `${file}` contains a disabledModules item that is an attribute set that can be converted to a string (${toString m}) but also has a `.key` attribute (${m.key}) with a different value. This makes it ambiguous which module should be disabled." + else + toString m + + else if m?key + then + m.key + + else if isAttrs m + then throw "Module `${file}` contains a disabledModules item that is an attribute set, presumably a module, that does not have a `key` attribute. This means that the module system doesn't have any means to identify the module that should be disabled. Make sure that you've put the correct value in disabledModules: a string path relative to modulesPath, a path value, or an attribute set with a `key` attribute." + else throw "Each disabledModules item must be a path, string, or a attribute set with a key attribute, or a value supported by toString. However, one of the disabledModules items in `${toString file}` is none of that, but is of type ${builtins.typeOf m}."; + + disabledKeys = concatMap ({ file, disabled }: map (moduleKey file) disabled) disabled; + keyFilter = filter (attrs: ! elem attrs.key disabledKeys); + in map (attrs: attrs.module) (builtins.genericClosure { + startSet = keyFilter modules; + operator = attrs: keyFilter attrs.modules; + }); + + in modulesPath: initialModules: args: + filterModules modulesPath (collectStructuredModules unknownModule "" initialModules args); + + /* Wrap a module with a default location for reporting errors. */ + setDefaultModuleLocation = file: m: + { _file = file; imports = [ m ]; }; + + /* Massage a module into canonical form, that is, a set consisting + of ‘options’, ‘config’ and ‘imports’ attributes. */ + unifyModuleSyntax = file: key: m: + let + addMeta = config: if m ? meta + then mkMerge [ config { meta = m.meta; } ] + else config; + addFreeformType = config: if m ? freeformType + then mkMerge [ config { _module.freeformType = m.freeformType; } ] + else config; + in + if m ? config || m ? options then + let badAttrs = removeAttrs m ["_class" "_file" "key" "disabledModules" "imports" "options" "config" "meta" "freeformType"]; in + if badAttrs != {} then + throw "Module `${key}' has an unsupported attribute `${head (attrNames badAttrs)}'. This is caused by introducing a top-level `config' or `options' attribute. Add configuration attributes immediately on the top level instead, or move all of them (namely: ${toString (attrNames badAttrs)}) into the explicit `config' attribute." + else + { _file = toString m._file or file; + _class = m._class or null; + key = toString m.key or key; + disabledModules = m.disabledModules or []; + imports = m.imports or []; + options = m.options or {}; + config = addFreeformType (addMeta (m.config or {})); + } + else + # shorthand syntax + lib.throwIfNot (isAttrs m) "module ${file} (${key}) does not look like a module." + { _file = toString m._file or file; + _class = m._class or null; + key = toString m.key or key; + disabledModules = m.disabledModules or []; + imports = m.require or [] ++ m.imports or []; + options = {}; + config = addFreeformType (removeAttrs m ["_class" "_file" "key" "disabledModules" "require" "imports" "freeformType"]); + }; + + applyModuleArgsIfFunction = key: f: args@{ config, options, lib, ... }: + if isFunction f then applyModuleArgs key f args else f; + + applyModuleArgs = key: f: args@{ config, options, lib, ... }: + let + # Module arguments are resolved in a strict manner when attribute set + # deconstruction is used. As the arguments are now defined with the + # config._module.args option, the strictness used on the attribute + # set argument would cause an infinite loop, if the result of the + # option is given as argument. + # + # To work-around the strictness issue on the deconstruction of the + # attributes set argument, we create a new attribute set which is + # constructed to satisfy the expected set of attributes. Thus calling + # a module will resolve strictly the attributes used as argument but + # not their values. The values are forwarding the result of the + # evaluation of the option. + context = name: ''while evaluating the module argument `${name}' in "${key}":''; + extraArgs = builtins.mapAttrs (name: _: + builtins.addErrorContext (context name) + (args.${name} or config._module.args.${name}) + ) (lib.functionArgs f); + + # Note: we append in the opposite order such that we can add an error + # context on the explicit arguments of "args" too. This update + # operator is used to make the "args@{ ... }: with args.lib;" notation + # works. + in f (args // extraArgs); + + /* Merge a list of modules. This will recurse over the option + declarations in all modules, combining them into a single set. + At the same time, for each option declaration, it will merge the + corresponding option definitions in all machines, returning them + in the ‘value’ attribute of each option. + + This returns a set like + { + # A recursive set of options along with their final values + matchedOptions = { + foo = { _type = "option"; value = "option value of foo"; ... }; + bar.baz = { _type = "option"; value = "option value of bar.baz"; ... }; + ... + }; + # A list of definitions that weren't matched by any option + unmatchedDefns = [ + { file = "file.nix"; prefix = [ "qux" ]; value = "qux"; } + ... + ]; + } + */ + mergeModules = prefix: modules: + mergeModules' prefix modules + (concatMap (m: map (config: { file = m._file; inherit config; }) (pushDownProperties m.config)) modules); + + mergeModules' = prefix: options: configs: + let + /* byName is like foldAttrs, but will look for attributes to merge in the + specified attribute name. + + byName "foo" (module: value: ["module.hidden=${module.hidden},value=${value}"]) + [ + { + hidden="baz"; + foo={qux="bar"; gla="flop";}; + } + { + hidden="fli"; + foo={qux="gne"; gli="flip";}; + } + ] + ===> + { + gla = [ "module.hidden=baz,value=flop" ]; + gli = [ "module.hidden=fli,value=flip" ]; + qux = [ "module.hidden=baz,value=bar" "module.hidden=fli,value=gne" ]; + } + */ + byName = attr: f: modules: + zipAttrsWith (n: concatLists) + (map (module: let subtree = module.${attr}; in + if !(builtins.isAttrs subtree) then + throw (if attr == "config" then '' + You're trying to define a value of type `${builtins.typeOf subtree}' + rather than an attribute set for the option + `${builtins.concatStringsSep "." prefix}'! + + This usually happens if `${builtins.concatStringsSep "." prefix}' has option + definitions inside that are not matched. Please check how to properly define + this option by e.g. referring to `man 5 configuration.nix'! + '' else '' + An option declaration for `${builtins.concatStringsSep "." prefix}' has type + `${builtins.typeOf subtree}' rather than an attribute set. + Did you mean to define this outside of `options'? + '') + else + mapAttrs (n: f module) subtree + ) modules); + # an attrset 'name' => list of submodules that declare ‘name’. + declsByName = byName "options" (module: option: + [{ inherit (module) _file; options = option; }] + ) options; + # an attrset 'name' => list of submodules that define ‘name’. + defnsByName = byName "config" (module: value: + map (config: { inherit (module) file; inherit config; }) (pushDownProperties value) + ) configs; + # extract the definitions for each loc + defnsByName' = byName "config" (module: value: + [{ inherit (module) file; inherit value; }] + ) configs; + + # Convert an option tree decl to a submodule option decl + optionTreeToOption = decl: + if isOption decl.options + then decl + else decl // { + options = mkOption { + type = types.submoduleWith { + modules = [ { options = decl.options; } ]; + # `null` is not intended for use by modules. It is an internal + # value that means "whatever the user has declared elsewhere". + # This might become obsolete with https://github.com/NixOS/nixpkgs/issues/162398 + shorthandOnlyDefinesConfig = null; + }; + }; + }; + + resultsByName = mapAttrs (name: decls: + # We're descending into attribute ‘name’. + let + loc = prefix ++ [name]; + defns = defnsByName.${name} or []; + defns' = defnsByName'.${name} or []; + optionDecls = filter (m: isOption m.options) decls; + in + if length optionDecls == length decls then + let opt = fixupOptionType loc (mergeOptionDecls loc decls); + in { + matchedOptions = evalOptionValue loc opt defns'; + unmatchedDefns = []; + } + else if optionDecls != [] then + if all (x: x.options.type.name == "submodule") optionDecls + # Raw options can only be merged into submodules. Merging into + # attrsets might be nice, but ambiguous. Suppose we have + # attrset as a `attrsOf submodule`. User declares option + # attrset.foo.bar, this could mean: + # a. option `bar` is only available in `attrset.foo` + # b. option `foo.bar` is available in all `attrset.*` + # c. reject and require "<name>" as a reminder that it behaves like (b). + # d. magically combine (a) and (c). + # All of the above are merely syntax sugar though. + then + let opt = fixupOptionType loc (mergeOptionDecls loc (map optionTreeToOption decls)); + in { + matchedOptions = evalOptionValue loc opt defns'; + unmatchedDefns = []; + } + else + let + nonOptions = filter (m: !isOption m.options) decls; + in + throw "The option `${showOption loc}' in module `${(lib.head optionDecls)._file}' would be a parent of the following options, but its type `${(lib.head optionDecls).options.type.description or "<no description>"}' does not support nested options.\n${ + showRawDecls loc nonOptions + }" + else + mergeModules' loc decls defns) declsByName; + + matchedOptions = mapAttrs (n: v: v.matchedOptions) resultsByName; + + # an attrset 'name' => list of unmatched definitions for 'name' + unmatchedDefnsByName = + # Propagate all unmatched definitions from nested option sets + mapAttrs (n: v: v.unmatchedDefns) resultsByName + # Plus the definitions for the current prefix that don't have a matching option + // removeAttrs defnsByName' (attrNames matchedOptions); + in { + inherit matchedOptions; + + # Transforms unmatchedDefnsByName into a list of definitions + unmatchedDefns = + if configs == [] + then + # When no config values exist, there can be no unmatched config, so + # we short circuit and avoid evaluating more _options_ than necessary. + [] + else + concatLists (mapAttrsToList (name: defs: + map (def: def // { + # Set this so we know when the definition first left unmatched territory + prefix = [name] ++ (def.prefix or []); + }) defs + ) unmatchedDefnsByName); + }; + + /* Merge multiple option declarations into a single declaration. In + general, there should be only one declaration of each option. + The exception is the ‘options’ attribute, which specifies + sub-options. These can be specified multiple times to allow one + module to add sub-options to an option declared somewhere else + (e.g. multiple modules define sub-options for ‘fileSystems’). + + 'loc' is the list of attribute names where the option is located. + + 'opts' is a list of modules. Each module has an options attribute which + correspond to the definition of 'loc' in 'opt.file'. */ + mergeOptionDecls = + loc: opts: + foldl' (res: opt: + let t = res.type; + t' = opt.options.type; + mergedType = t.typeMerge t'.functor; + typesMergeable = mergedType != null; + typeSet = if (bothHave "type") && typesMergeable + then { type = mergedType; } + else {}; + bothHave = k: opt.options ? ${k} && res ? ${k}; + in + if bothHave "default" || + bothHave "example" || + bothHave "description" || + bothHave "apply" || + (bothHave "type" && (! typesMergeable)) + then + throw "The option `${showOption loc}' in `${opt._file}' is already declared in ${showFiles res.declarations}." + else + let + getSubModules = opt.options.type.getSubModules or null; + submodules = + if getSubModules != null then map (setDefaultModuleLocation opt._file) getSubModules ++ res.options + else res.options; + in opt.options // res // + { declarations = res.declarations ++ [opt._file]; + options = submodules; + } // typeSet + ) { inherit loc; declarations = []; options = []; } opts; + + /* Merge all the definitions of an option to produce the final + config value. */ + evalOptionValue = loc: opt: defs: + let + # Add in the default value for this option, if any. + defs' = + (optional (opt ? default) + { file = head opt.declarations; value = mkOptionDefault opt.default; }) ++ defs; + + # Handle properties, check types, and merge everything together. + res = + if opt.readOnly or false && length defs' > 1 then + let + # For a better error message, evaluate all readOnly definitions as + # if they were the only definition. + separateDefs = map (def: def // { + value = (mergeDefinitions loc opt.type [ def ]).mergedValue; + }) defs'; + in throw "The option `${showOption loc}' is read-only, but it's set multiple times. Definition values:${showDefs separateDefs}" + else + mergeDefinitions loc opt.type defs'; + + # Apply the 'apply' function to the merged value. This allows options to + # yield a value computed from the definitions + value = if opt ? apply then opt.apply res.mergedValue else res.mergedValue; + + warnDeprecation = + warnIf (opt.type.deprecationMessage != null) + "The type `types.${opt.type.name}' of option `${showOption loc}' defined in ${showFiles opt.declarations} is deprecated. ${opt.type.deprecationMessage}"; + + in warnDeprecation opt // + { value = builtins.addErrorContext "while evaluating the option `${showOption loc}':" value; + inherit (res.defsFinal') highestPrio; + definitions = map (def: def.value) res.defsFinal; + files = map (def: def.file) res.defsFinal; + definitionsWithLocations = res.defsFinal; + inherit (res) isDefined; + # This allows options to be correctly displayed using `${options.path.to.it}` + __toString = _: showOption loc; + }; + + # Merge definitions of a value of a given type. + mergeDefinitions = loc: type: defs: rec { + defsFinal' = + let + # Process mkMerge and mkIf properties. + defs' = concatMap (m: + map (value: { inherit (m) file; inherit value; }) (builtins.addErrorContext "while evaluating definitions from `${m.file}':" (dischargeProperties m.value)) + ) defs; + + # Process mkOverride properties. + defs'' = filterOverrides' defs'; + + # Sort mkOrder properties. + defs''' = + # Avoid sorting if we don't have to. + if any (def: def.value._type or "" == "order") defs''.values + then sortProperties defs''.values + else defs''.values; + in { + values = defs'''; + inherit (defs'') highestPrio; + }; + defsFinal = defsFinal'.values; + + # Type-check the remaining definitions, and merge them. Or throw if no definitions. + mergedValue = + if isDefined then + if all (def: type.check def.value) defsFinal then type.merge loc defsFinal + else let allInvalid = filter (def: ! type.check def.value) defsFinal; + in throw "A definition for option `${showOption loc}' is not of type `${type.description}'. Definition values:${showDefs allInvalid}" + else + # (nixos-option detects this specific error message and gives it special + # handling. If changed here, please change it there too.) + throw "The option `${showOption loc}' is used but not defined."; + + isDefined = defsFinal != []; + + optionalValue = + if isDefined then { value = mergedValue; } + else {}; + }; + + /* Given a config set, expand mkMerge properties, and push down the + other properties into the children. The result is a list of + config sets that do not have properties at top-level. For + example, + + mkMerge [ { boot = set1; } (mkIf cond { boot = set2; services = set3; }) ] + + is transformed into + + [ { boot = set1; } { boot = mkIf cond set2; services = mkIf cond set3; } ]. + + This transform is the critical step that allows mkIf conditions + to refer to the full configuration without creating an infinite + recursion. + */ + pushDownProperties = cfg: + if cfg._type or "" == "merge" then + concatMap pushDownProperties cfg.contents + else if cfg._type or "" == "if" then + map (mapAttrs (n: v: mkIf cfg.condition v)) (pushDownProperties cfg.content) + else if cfg._type or "" == "override" then + map (mapAttrs (n: v: mkOverride cfg.priority v)) (pushDownProperties cfg.content) + else # FIXME: handle mkOrder? + [ cfg ]; + + /* Given a config value, expand mkMerge properties, and discharge + any mkIf conditions. That is, this is the place where mkIf + conditions are actually evaluated. The result is a list of + config values. For example, ‘mkIf false x’ yields ‘[]’, + ‘mkIf true x’ yields ‘[x]’, and + + mkMerge [ 1 (mkIf true 2) (mkIf true (mkIf false 3)) ] + + yields ‘[ 1 2 ]’. + */ + dischargeProperties = def: + if def._type or "" == "merge" then + concatMap dischargeProperties def.contents + else if def._type or "" == "if" then + if isBool def.condition then + if def.condition then + dischargeProperties def.content + else + [ ] + else + throw "‘mkIf’ called with a non-Boolean condition" + else + [ def ]; + + /* Given a list of config values, process the mkOverride properties, + that is, return the values that have the highest (that is, + numerically lowest) priority, and strip the mkOverride + properties. For example, + + [ { file = "/1"; value = mkOverride 10 "a"; } + { file = "/2"; value = mkOverride 20 "b"; } + { file = "/3"; value = "z"; } + { file = "/4"; value = mkOverride 10 "d"; } + ] + + yields + + [ { file = "/1"; value = "a"; } + { file = "/4"; value = "d"; } + ] + + Note that "z" has the default priority 100. + */ + filterOverrides = defs: (filterOverrides' defs).values; + + filterOverrides' = defs: + let + getPrio = def: if def.value._type or "" == "override" then def.value.priority else defaultOverridePriority; + highestPrio = foldl' (prio: def: min (getPrio def) prio) 9999 defs; + strip = def: if def.value._type or "" == "override" then def // { value = def.value.content; } else def; + in { + values = concatMap (def: if getPrio def == highestPrio then [(strip def)] else []) defs; + inherit highestPrio; + }; + + /* Sort a list of properties. The sort priority of a property is + defaultOrderPriority by default, but can be overridden by wrapping the property + using mkOrder. */ + sortProperties = defs: + let + strip = def: + if def.value._type or "" == "order" + then def // { value = def.value.content; inherit (def.value) priority; } + else def; + defs' = map strip defs; + compare = a: b: (a.priority or defaultOrderPriority) < (b.priority or defaultOrderPriority); + in sort compare defs'; + + # This calls substSubModules, whose entire purpose is only to ensure that + # option declarations in submodules have accurate position information. + # TODO: Merge this into mergeOptionDecls + fixupOptionType = loc: opt: + if opt.type.getSubModules or null == null + then opt // { type = opt.type or types.unspecified; } + else opt // { type = opt.type.substSubModules opt.options; options = []; }; + + + /* Properties. */ + + mkIf = condition: content: + { _type = "if"; + inherit condition content; + }; + + mkAssert = assertion: message: content: + mkIf + (if assertion then true else throw "\nFailed assertion: ${message}") + content; + + mkMerge = contents: + { _type = "merge"; + inherit contents; + }; + + mkOverride = priority: content: + { _type = "override"; + inherit priority content; + }; + + mkOptionDefault = mkOverride 1500; # priority of option defaults + mkDefault = mkOverride 1000; # used in config sections of non-user modules to set a default + defaultOverridePriority = 100; + mkImageMediaOverride = mkOverride 60; # image media profiles can be derived by inclusion into host config, hence needing to override host config, but do allow user to mkForce + mkForce = mkOverride 50; + mkVMOverride = mkOverride 10; # used by ‘nixos-rebuild build-vm’ + + defaultPriority = lib.warnIf (lib.isInOldestRelease 2305) "lib.modules.defaultPriority is deprecated, please use lib.modules.defaultOverridePriority instead." defaultOverridePriority; + + mkFixStrictness = lib.warn "lib.mkFixStrictness has no effect and will be removed. It returns its argument unmodified, so you can just remove any calls." id; + + mkOrder = priority: content: + { _type = "order"; + inherit priority content; + }; + + mkBefore = mkOrder 500; + defaultOrderPriority = 1000; + mkAfter = mkOrder 1500; + + # Convenient property used to transfer all definitions and their + # properties from one option to another. This property is useful for + # renaming options, and also for including properties from another module + # system, including sub-modules. + # + # { config, options, ... }: + # + # { + # # 'bar' might not always be defined in the current module-set. + # config.foo.enable = mkAliasDefinitions (options.bar.enable or {}); + # + # # 'barbaz' has to be defined in the current module-set. + # config.foobar.paths = mkAliasDefinitions options.barbaz.paths; + # } + # + # Note, this is different than taking the value of the option and using it + # as a definition, as the new definition will not keep the mkOverride / + # mkDefault properties of the previous option. + # + mkAliasDefinitions = mkAliasAndWrapDefinitions id; + mkAliasAndWrapDefinitions = wrap: option: + mkAliasIfDef option (wrap (mkMerge option.definitions)); + + # Similar to mkAliasAndWrapDefinitions but copies over the priority from the + # option as well. + # + # If a priority is not set, it assumes a priority of defaultOverridePriority. + mkAliasAndWrapDefsWithPriority = wrap: option: + let + prio = option.highestPrio or defaultOverridePriority; + defsWithPrio = map (mkOverride prio) option.definitions; + in mkAliasIfDef option (wrap (mkMerge defsWithPrio)); + + mkAliasIfDef = option: + mkIf (isOption option && option.isDefined); + + /* Compatibility. */ + fixMergeModules = modules: args: evalModules { inherit modules args; check = false; }; + + + /* Return a module that causes a warning to be shown if the + specified option is defined. For example, + + mkRemovedOptionModule [ "boot" "loader" "grub" "bootDevice" ] "<replacement instructions>" + + causes a assertion if the user defines boot.loader.grub.bootDevice. + + replacementInstructions is a string that provides instructions on + how to achieve the same functionality without the removed option, + or alternatively a reasoning why the functionality is not needed. + replacementInstructions SHOULD be provided! + */ + mkRemovedOptionModule = optionName: replacementInstructions: + { options, ... }: + { options = setAttrByPath optionName (mkOption { + visible = false; + apply = x: throw "The option `${showOption optionName}' can no longer be used since it's been removed. ${replacementInstructions}"; + }); + config.assertions = + let opt = getAttrFromPath optionName options; in [{ + assertion = !opt.isDefined; + message = '' + The option definition `${showOption optionName}' in ${showFiles opt.files} no longer has any effect; please remove it. + ${replacementInstructions} + ''; + }]; + }; + + /* Return a module that causes a warning to be shown if the + specified "from" option is defined; the defined value is however + forwarded to the "to" option. This can be used to rename options + while providing backward compatibility. For example, + + mkRenamedOptionModule [ "boot" "copyKernels" ] [ "boot" "loader" "grub" "copyKernels" ] + + forwards any definitions of boot.copyKernels to + boot.loader.grub.copyKernels while printing a warning. + + This also copies over the priority from the aliased option to the + non-aliased option. + */ + mkRenamedOptionModule = from: to: doRename { + inherit from to; + visible = false; + warn = true; + use = builtins.trace "Obsolete option `${showOption from}' is used. It was renamed to `${showOption to}'."; + }; + + mkRenamedOptionModuleWith = { + /* Old option path as list of strings. */ + from, + /* New option path as list of strings. */ + to, + + /* + Release number of the first release that contains the rename, ignoring backports. + Set it to the upcoming release, matching the nixpkgs/.version file. + */ + sinceRelease, + + }: doRename { + inherit from to; + visible = false; + warn = lib.isInOldestRelease sinceRelease; + use = lib.warnIf (lib.isInOldestRelease sinceRelease) + "Obsolete option `${showOption from}' is used. It was renamed to `${showOption to}'."; + }; + + /* Return a module that causes a warning to be shown if any of the "from" + option is defined; the defined values can be used in the "mergeFn" to set + the "to" value. + This function can be used to merge multiple options into one that has a + different type. + + "mergeFn" takes the module "config" as a parameter and must return a value + of "to" option type. + + mkMergedOptionModule + [ [ "a" "b" "c" ] + [ "d" "e" "f" ] ] + [ "x" "y" "z" ] + (config: + let value = p: getAttrFromPath p config; + in + if (value [ "a" "b" "c" ]) == true then "foo" + else if (value [ "d" "e" "f" ]) == true then "bar" + else "baz") + + - options.a.b.c is a removed boolean option + - options.d.e.f is a removed boolean option + - options.x.y.z is a new str option that combines a.b.c and d.e.f + functionality + + This show a warning if any a.b.c or d.e.f is set, and set the value of + x.y.z to the result of the merge function + */ + mkMergedOptionModule = from: to: mergeFn: + { config, options, ... }: + { + options = foldl' recursiveUpdate {} (map (path: setAttrByPath path (mkOption { + visible = false; + # To use the value in mergeFn without triggering errors + default = "_mkMergedOptionModule"; + })) from); + + config = { + warnings = filter (x: x != "") (map (f: + let val = getAttrFromPath f config; + opt = getAttrFromPath f options; + in + optionalString + (val != "_mkMergedOptionModule") + "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly." + ) from); + } // setAttrByPath to (mkMerge + (optional + (any (f: (getAttrFromPath f config) != "_mkMergedOptionModule") from) + (mergeFn config))); + }; + + /* Single "from" version of mkMergedOptionModule. + Return a module that causes a warning to be shown if the "from" option is + defined; the defined value can be used in the "mergeFn" to set the "to" + value. + This function can be used to change an option into another that has a + different type. + + "mergeFn" takes the module "config" as a parameter and must return a value of + "to" option type. + + mkChangedOptionModule [ "a" "b" "c" ] [ "x" "y" "z" ] + (config: + let value = getAttrFromPath [ "a" "b" "c" ] config; + in + if value > 100 then "high" + else "normal") + + - options.a.b.c is a removed int option + - options.x.y.z is a new str option that supersedes a.b.c + + This show a warning if a.b.c is set, and set the value of x.y.z to the + result of the change function + */ + mkChangedOptionModule = from: to: changeFn: + mkMergedOptionModule [ from ] to changeFn; + + /* Like ‘mkRenamedOptionModule’, but doesn't show a warning. */ + mkAliasOptionModule = from: to: doRename { + inherit from to; + visible = true; + warn = false; + use = id; + }; + + /* Transitional version of mkAliasOptionModule that uses MD docs. */ + mkAliasOptionModuleMD = from: to: doRename { + inherit from to; + visible = true; + warn = false; + use = id; + markdown = true; + }; + + /* mkDerivedConfig : Option a -> (a -> Definition b) -> Definition b + + Create config definitions with the same priority as the definition of another option. + This should be used for option definitions where one option sets the value of another as a convenience. + For instance a config file could be set with a `text` or `source` option, where text translates to a `source` + value using `mkDerivedConfig options.text (pkgs.writeText "filename.conf")`. + + It takes care of setting the right priority using `mkOverride`. + */ + # TODO: make the module system error message include information about `opt` in + # error messages about conflicts. E.g. introduce a variation of `mkOverride` which + # adds extra location context to the definition object. This will allow context to be added + # to all messages that report option locations "this value was derived from <full option name> + # which was defined in <locations>". It can provide a trace of options that contributed + # to definitions. + mkDerivedConfig = opt: f: + mkOverride + (opt.highestPrio or defaultOverridePriority) + (f opt.value); + + doRename = { from, to, visible, warn, use, withPriority ? true, markdown ? false }: + { config, options, ... }: + let + fromOpt = getAttrFromPath from options; + toOf = attrByPath to + (abort "Renaming error: option `${showOption to}' does not exist."); + toType = let opt = attrByPath to {} options; in opt.type or (types.submodule {}); + in + { + options = setAttrByPath from (mkOption { + inherit visible; + description = if markdown + then lib.mdDoc "Alias of {option}`${showOption to}`." + else "Alias of <option>${showOption to}</option>."; + apply = x: use (toOf config); + } // optionalAttrs (toType != null) { + type = toType; + }); + config = mkMerge [ + (optionalAttrs (options ? warnings) { + warnings = optional (warn && fromOpt.isDefined) + "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; + }) + (if withPriority + then mkAliasAndWrapDefsWithPriority (setAttrByPath to) fromOpt + else mkAliasAndWrapDefinitions (setAttrByPath to) fromOpt) + ]; + }; + + /* Use this function to import a JSON file as NixOS configuration. + + modules.importJSON :: path -> attrs + */ + importJSON = file: { + _file = file; + config = lib.importJSON file; + }; + + /* Use this function to import a TOML file as NixOS configuration. + + modules.importTOML :: path -> attrs + */ + importTOML = file: { + _file = file; + config = lib.importTOML file; + }; + + private = lib.mapAttrs + (k: lib.warn "External use of `lib.modules.${k}` is deprecated. If your use case isn't covered by non-deprecated functions, we'd like to know more and perhaps support your use case well, instead of providing access to these low level functions. In this case please open an issue in https://github.com/nixos/nixpkgs/issues/.") + { + inherit + applyModuleArgsIfFunction + dischargeProperties + evalOptionValue + mergeModules + mergeModules' + pushDownProperties + unifyModuleSyntax + ; + collectModules = collectModules null; + }; + +in +private // +{ + # NOTE: not all of these functions are necessarily public interfaces; some + # are just needed by types.nix, but are not meant to be consumed + # externally. + inherit + defaultOrderPriority + defaultOverridePriority + defaultPriority + doRename + evalModules + filterOverrides + filterOverrides' + fixMergeModules + fixupOptionType # should be private? + importJSON + importTOML + mergeDefinitions + mergeOptionDecls # should be private? + mkAfter + mkAliasAndWrapDefinitions + mkAliasAndWrapDefsWithPriority + mkAliasDefinitions + mkAliasIfDef + mkAliasOptionModule + mkAliasOptionModuleMD + mkAssert + mkBefore + mkChangedOptionModule + mkDefault + mkDerivedConfig + mkFixStrictness + mkForce + mkIf + mkImageMediaOverride + mkMerge + mkMergedOptionModule + mkOptionDefault + mkOrder + mkOverride + mkRemovedOptionModule + mkRenamedOptionModule + mkRenamedOptionModuleWith + mkVMOverride + setDefaultModuleLocation + sortProperties; +} diff --git a/options.nix b/options.nix new file mode 100644 index 000000000..af7914bb5 --- /dev/null +++ b/options.nix @@ -0,0 +1,432 @@ +# Nixpkgs/NixOS option handling. +{ lib }: + +let + inherit (lib) + all + collect + concatLists + concatMap + concatMapStringsSep + filter + foldl' + head + tail + isAttrs + isBool + isDerivation + isFunction + isInt + isList + isString + length + mapAttrs + optional + optionals + take + ; + inherit (lib.attrsets) + attrByPath + optionalAttrs + ; + inherit (lib.strings) + concatMapStrings + concatStringsSep + ; + inherit (lib.types) + mkOptionType + ; + inherit (lib.lists) + last + ; + prioritySuggestion = '' + Use `lib.mkForce value` or `lib.mkDefault value` to change the priority on any of these definitions. + ''; +in +rec { + + /* Returns true when the given argument is an option + + Type: isOption :: a -> bool + + Example: + isOption 1 // => false + isOption (mkOption {}) // => true + */ + isOption = lib.isType "option"; + + /* Creates an Option attribute set. mkOption accepts an attribute set with the following keys: + + All keys default to `null` when not given. + + Example: + mkOption { } // => { _type = "option"; } + mkOption { default = "foo"; } // => { _type = "option"; default = "foo"; } + */ + mkOption = + { + # Default value used when no definition is given in the configuration. + default ? null, + # Textual representation of the default, for the manual. + defaultText ? null, + # Example value used in the manual. + example ? null, + # String describing the option. + description ? null, + # Related packages used in the manual (see `genRelatedPackages` in ../nixos/lib/make-options-doc/default.nix). + relatedPackages ? null, + # Option type, providing type-checking and value merging. + type ? null, + # Function that converts the option value to something else. + apply ? null, + # Whether the option is for NixOS developers only. + internal ? null, + # Whether the option shows up in the manual. Default: true. Use false to hide the option and any sub-options from submodules. Use "shallow" to hide only sub-options. + visible ? null, + # Whether the option can be set only once + readOnly ? null, + } @ attrs: + attrs // { _type = "option"; }; + + /* Creates an Option attribute set for a boolean value option i.e an + option to be toggled on or off: + + Example: + mkEnableOption "foo" + => { _type = "option"; default = false; description = "Whether to enable foo."; example = true; type = { ... }; } + */ + mkEnableOption = + # Name for the created option + name: mkOption { + default = false; + example = true; + description = + if name ? _type && name._type == "mdDoc" + then lib.mdDoc "Whether to enable ${name.text}." + else "Whether to enable ${name}."; + type = lib.types.bool; + }; + + /* Creates an Option attribute set for an option that specifies the + package a module should use for some purpose. + + The package is specified in the third argument under `default` as a list of strings + representing its attribute path in nixpkgs (or another package set). + Because of this, you need to pass nixpkgs itself (or a subset) as the first argument. + + The second argument may be either a string or a list of strings. + It provides the display name of the package in the description of the generated option + (using only the last element if the passed value is a list) + and serves as the fallback value for the `default` argument. + + To include extra information in the description, pass `extraDescription` to + append arbitrary text to the generated description. + You can also pass an `example` value, either a literal string or an attribute path. + + The default argument can be omitted if the provided name is + an attribute of pkgs (if name is a string) or a + valid attribute path in pkgs (if name is a list). + + If you wish to explicitly provide no default, pass `null` as `default`. + + Type: mkPackageOption :: pkgs -> (string|[string]) -> { default? :: [string], example? :: null|string|[string], extraDescription? :: string } -> option + + Example: + mkPackageOption pkgs "hello" { } + => { _type = "option"; default = «derivation /nix/store/3r2vg51hlxj3cx5vscp0vkv60bqxkaq0-hello-2.10.drv»; defaultText = { ... }; description = "The hello package to use."; type = { ... }; } + + Example: + mkPackageOption pkgs "GHC" { + default = [ "ghc" ]; + example = "pkgs.haskell.packages.ghc92.ghc.withPackages (hkgs: [ hkgs.primes ])"; + } + => { _type = "option"; default = «derivation /nix/store/jxx55cxsjrf8kyh3fp2ya17q99w7541r-ghc-8.10.7.drv»; defaultText = { ... }; description = "The GHC package to use."; example = { ... }; type = { ... }; } + + Example: + mkPackageOption pkgs [ "python39Packages" "pytorch" ] { + extraDescription = "This is an example and doesn't actually do anything."; + } + => { _type = "option"; default = «derivation /nix/store/gvqgsnc4fif9whvwd9ppa568yxbkmvk8-python3.9-pytorch-1.10.2.drv»; defaultText = { ... }; description = "The pytorch package to use. This is an example and doesn't actually do anything."; type = { ... }; } + + */ + mkPackageOption = + # Package set (a specific version of nixpkgs or a subset) + pkgs: + # Name for the package, shown in option description + name: + { + # Whether the package can be null, for example to disable installing a package altogether. + nullable ? false, + # The attribute path where the default package is located (may be omitted) + default ? name, + # A string or an attribute path to use as an example (may be omitted) + example ? null, + # Additional text to include in the option description (may be omitted) + extraDescription ? "", + }: + let + name' = if isList name then last name else name; + in mkOption ({ + type = with lib.types; (if nullable then nullOr else lib.id) package; + description = "The ${name'} package to use." + + (if extraDescription == "" then "" else " ") + extraDescription; + } // (if default != null then let + default' = if isList default then default else [ default ]; + defaultPath = concatStringsSep "." default'; + defaultValue = attrByPath default' + (throw "${defaultPath} cannot be found in pkgs") pkgs; + in { + default = defaultValue; + defaultText = literalExpression ("pkgs." + defaultPath); + } else if nullable then { + default = null; + } else { }) // lib.optionalAttrs (example != null) { + example = literalExpression + (if isList example then "pkgs." + concatStringsSep "." example else example); + }); + + /* Like mkPackageOption, but emit an mdDoc description instead of DocBook. */ + mkPackageOptionMD = pkgs: name: extra: + let option = mkPackageOption pkgs name extra; + in option // { description = lib.mdDoc option.description; }; + + /* This option accepts anything, but it does not produce any result. + + This is useful for sharing a module across different module sets + without having to implement similar features as long as the + values of the options are not accessed. */ + mkSinkUndeclaredOptions = attrs: mkOption ({ + internal = true; + visible = false; + default = false; + description = "Sink for option definitions."; + type = mkOptionType { + name = "sink"; + check = x: true; + merge = loc: defs: false; + }; + apply = x: throw "Option value is not readable because the option is not declared."; + } // attrs); + + mergeDefaultOption = loc: defs: + let list = getValues defs; in + if length list == 1 then head list + else if all isFunction list then x: mergeDefaultOption loc (map (f: f x) list) + else if all isList list then concatLists list + else if all isAttrs list then foldl' lib.mergeAttrs {} list + else if all isBool list then foldl' lib.or false list + else if all isString list then lib.concatStrings list + else if all isInt list && all (x: x == head list) list then head list + else throw "Cannot merge definitions of `${showOption loc}'. Definition values:${showDefs defs}"; + + mergeOneOption = mergeUniqueOption { message = ""; }; + + mergeUniqueOption = { message }: loc: defs: + if length defs == 1 + then (head defs).value + else assert length defs > 1; + throw "The option `${showOption loc}' is defined multiple times while it's expected to be unique.\n${message}\nDefinition values:${showDefs defs}\n${prioritySuggestion}"; + + /* "Merge" option definitions by checking that they all have the same value. */ + mergeEqualOption = loc: defs: + if defs == [] then abort "This case should never happen." + # Return early if we only have one element + # This also makes it work for functions, because the foldl' below would try + # to compare the first element with itself, which is false for functions + else if length defs == 1 then (head defs).value + else (foldl' (first: def: + if def.value != first.value then + throw "The option `${showOption loc}' has conflicting definition values:${showDefs [ first def ]}\n${prioritySuggestion}" + else + first) (head defs) (tail defs)).value; + + /* Extracts values of all "value" keys of the given list. + + Type: getValues :: [ { value :: a; } ] -> [a] + + Example: + getValues [ { value = 1; } { value = 2; } ] // => [ 1 2 ] + getValues [ ] // => [ ] + */ + getValues = map (x: x.value); + + /* Extracts values of all "file" keys of the given list + + Type: getFiles :: [ { file :: a; } ] -> [a] + + Example: + getFiles [ { file = "file1"; } { file = "file2"; } ] // => [ "file1" "file2" ] + getFiles [ ] // => [ ] + */ + getFiles = map (x: x.file); + + # Generate documentation template from the list of option declaration like + # the set generated with filterOptionSets. + optionAttrSetToDocList = optionAttrSetToDocList' []; + + optionAttrSetToDocList' = _: options: + concatMap (opt: + let + name = showOption opt.loc; + docOption = { + loc = opt.loc; + inherit name; + description = opt.description or null; + declarations = filter (x: x != unknownModule) opt.declarations; + internal = opt.internal or false; + visible = + if (opt?visible && opt.visible == "shallow") + then true + else opt.visible or true; + readOnly = opt.readOnly or false; + type = opt.type.description or "unspecified"; + } + // optionalAttrs (opt ? example) { + example = + builtins.addErrorContext "while evaluating the example of option `${name}`" ( + renderOptionValue opt.example + ); + } + // optionalAttrs (opt ? defaultText || opt ? default) { + default = + builtins.addErrorContext "while evaluating the ${if opt?defaultText then "defaultText" else "default value"} of option `${name}`" ( + renderOptionValue (opt.defaultText or opt.default) + ); + } + // optionalAttrs (opt ? relatedPackages && opt.relatedPackages != null) { inherit (opt) relatedPackages; }; + + subOptions = + let ss = opt.type.getSubOptions opt.loc; + in if ss != {} then optionAttrSetToDocList' opt.loc ss else []; + subOptionsVisible = docOption.visible && opt.visible or null != "shallow"; + in + # To find infinite recursion in NixOS option docs: + # builtins.trace opt.loc + [ docOption ] ++ optionals subOptionsVisible subOptions) (collect isOption options); + + + /* This function recursively removes all derivation attributes from + `x` except for the `name` attribute. + + This is to make the generation of `options.xml` much more + efficient: the XML representation of derivations is very large + (on the order of megabytes) and is not actually used by the + manual generator. + + This function was made obsolete by renderOptionValue and is kept for + compatibility with out-of-tree code. + */ + scrubOptionValue = x: + if isDerivation x then + { type = "derivation"; drvPath = x.name; outPath = x.name; name = x.name; } + else if isList x then map scrubOptionValue x + else if isAttrs x then mapAttrs (n: v: scrubOptionValue v) (removeAttrs x ["_args"]) + else x; + + + /* Ensures that the given option value (default or example) is a `_type`d string + by rendering Nix values to `literalExpression`s. + */ + renderOptionValue = v: + if v ? _type && v ? text then v + else literalExpression (lib.generators.toPretty { + multiline = true; + allowPrettyValues = true; + } v); + + + /* For use in the `defaultText` and `example` option attributes. Causes the + given string to be rendered verbatim in the documentation as Nix code. This + is necessary for complex values, e.g. functions, or values that depend on + other values or packages. + */ + literalExpression = text: + if ! isString text then throw "literalExpression expects a string." + else { _type = "literalExpression"; inherit text; }; + + literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalDocBook for a non-Nix description." literalExpression; + + + /* For use in the `defaultText` and `example` option attributes. Causes the + given DocBook text to be inserted verbatim in the documentation, for when + a `literalExpression` would be too hard to read. + */ + literalDocBook = text: + if ! isString text then throw "literalDocBook expects a string." + else + lib.warnIf (lib.isInOldestRelease 2211) + "literalDocBook is deprecated, use literalMD instead" + { _type = "literalDocBook"; inherit text; }; + + /* Transition marker for documentation that's already migrated to markdown + syntax. + */ + mdDoc = text: + if ! isString text then throw "mdDoc expects a string." + else { _type = "mdDoc"; inherit text; }; + + /* For use in the `defaultText` and `example` option attributes. Causes the + given MD text to be inserted verbatim in the documentation, for when + a `literalExpression` would be too hard to read. + */ + literalMD = text: + if ! isString text then throw "literalMD expects a string." + else { _type = "literalMD"; inherit text; }; + + # Helper functions. + + /* Convert an option, described as a list of the option parts to a + human-readable version. + + Example: + (showOption ["foo" "bar" "baz"]) == "foo.bar.baz" + (showOption ["foo" "bar.baz" "tux"]) == "foo.\"bar.baz\".tux" + (showOption ["windowManager" "2bwm" "enable"]) == "windowManager.\"2bwm\".enable" + + Placeholders will not be quoted as they are not actual values: + (showOption ["foo" "*" "bar"]) == "foo.*.bar" + (showOption ["foo" "<name>" "bar"]) == "foo.<name>.bar" + */ + showOption = parts: let + escapeOptionPart = part: + let + # We assume that these are "special values" and not real configuration data. + # If it is real configuration data, it is rendered incorrectly. + specialIdentifiers = [ + "<name>" # attrsOf (submodule {}) + "*" # listOf (submodule {}) + "<function body>" # functionTo + ]; + in if builtins.elem part specialIdentifiers + then part + else lib.strings.escapeNixIdentifier part; + in (concatStringsSep ".") (map escapeOptionPart parts); + showFiles = files: concatStringsSep " and " (map (f: "`${f}'") files); + + showDefs = defs: concatMapStrings (def: + let + # Pretty print the value for display, if successful + prettyEval = builtins.tryEval + (lib.generators.toPretty { } + (lib.generators.withRecursion { depthLimit = 10; throwOnDepthLimit = false; } def.value)); + # Split it into its lines + lines = filter (v: ! isList v) (builtins.split "\n" prettyEval.value); + # Only display the first 5 lines, and indent them for better visibility + value = concatStringsSep "\n " (take 5 lines ++ optional (length lines > 5) "..."); + result = + # Don't print any value if evaluating the value strictly fails + if ! prettyEval.success then "" + # Put it on a new line if it consists of multiple + else if length lines > 1 then ":\n " + value + else ": " + value; + in "\n- In `${def.file}'${result}" + ) defs; + + showOptionWithDefLocs = opt: '' + ${showOption opt.loc}, with values defined in: + ${concatMapStringsSep "\n" (defFile: " - ${defFile}") opt.files} + ''; + + unknownModule = "<unknown-file>"; + +} diff --git a/path/README.md b/path/README.md new file mode 100644 index 000000000..87e552d12 --- /dev/null +++ b/path/README.md @@ -0,0 +1,196 @@ +# Path library + +This document explains why the `lib.path` library is designed the way it is. + +The purpose of this library is to process [filesystem paths]. It does not read files from the filesystem. +It exists to support the native Nix [path value type] with extra functionality. + +[filesystem paths]: https://en.m.wikipedia.org/wiki/Path_(computing) +[path value type]: https://nixos.org/manual/nix/stable/language/values.html#type-path + +As an extension of the path value type, it inherits the same intended use cases and limitations: +- Only use paths to access files at evaluation time, such as the local project source. +- Paths cannot point to derivations, so they are unfit to represent dependencies. +- A path implicitly imports the referenced files into the Nix store when interpolated to a string. Therefore paths are not suitable to access files at build- or run-time, as you risk importing the path from the evaluation system instead. + +Overall, this library works with two types of paths: +- Absolute paths are represented with the Nix [path value type]. Nix automatically normalises these paths. +- Subpaths are represented with the [string value type] since path value types don't support relative paths. This library normalises these paths as safely as possible. Absolute paths in strings are not supported. + + A subpath refers to a specific file or directory within an absolute base directory. + It is a stricter form of a relative path, notably [without support for `..` components][parents] since those could escape the base directory. + +[string value type]: https://nixos.org/manual/nix/stable/language/values.html#type-string + +This library is designed to be as safe and intuitive as possible, throwing errors when operations are attempted that would produce surprising results, and giving the expected result otherwise. + +This library is designed to work well as a dependency for the `lib.filesystem` and `lib.sources` library components. Contrary to these library components, `lib.path` does not read any paths from the filesystem. + +This library makes only these assumptions about paths and no others: +- `dirOf path` returns the path to the parent directory of `path`, unless `path` is the filesystem root, in which case `path` is returned. + - There can be multiple filesystem roots: `p == dirOf p` and `q == dirOf q` does not imply `p == q`. + - While there's only a single filesystem root in stable Nix, the [lazy trees feature](https://github.com/NixOS/nix/pull/6530) introduces [additional filesystem roots](https://github.com/NixOS/nix/pull/6530#discussion_r1041442173). +- `path + ("/" + string)` returns the path to the `string` subdirectory in `path`. + - If `string` contains no `/` characters, then `dirOf (path + ("/" + string)) == path`. + - If `string` contains no `/` characters, then `baseNameOf (path + ("/" + string)) == string`. +- `path1 == path2` returns `true` only if `path1` points to the same filesystem path as `path2`. + +Notably we do not make the assumption that we can turn paths into strings using `toString path`. + +## Design decisions + +Each subsection here contains a decision along with arguments and counter-arguments for (+) and against (-) that decision. + +### Leading dots for relative paths +[leading-dots]: #leading-dots-for-relative-paths + +Observing: Since subpaths are a form of relative paths, they can have a leading `./` to indicate it being a relative path, this is generally not necessary for tools though. + +Considering: Paths should be as explicit, consistent and unambiguous as possible. + +Decision: Returned subpaths should always have a leading `./`. + +<details> +<summary>Arguments</summary> + +- (+) In shells, just running `foo` as a command wouldn't execute the file `foo`, whereas `./foo` would execute the file. In contrast, `foo/bar` does execute that file without the need for `./`. This can lead to confusion about when a `./` needs to be prefixed. If a `./` is always included, this becomes a non-issue. This effectively then means that paths don't overlap with command names. +- (+) Prepending with `./` makes the subpaths always valid as relative Nix path expressions. +- (+) Using paths in command line arguments could give problems if not escaped properly, e.g. if a path was `--version`. This is not a problem with `./--version`. This effectively then means that paths don't overlap with GNU-style command line options. +- (-) `./` is not required to resolve relative paths, resolution always has an implicit `./` as prefix. +- (-) It's less noisy without the `./`, e.g. in error messages. + - (+) But similarly, it could be confusing whether something was even a path. + e.g. `foo` could be anything, but `./foo` is more clearly a path. +- (+) Makes it more uniform with absolute paths (those always start with `/`). + - (-) That is not relevant for practical purposes. +- (+) `find` also outputs results with `./`. + - (-) But only if you give it an argument of `.`. If you give it the argument `some-directory`, it won't prefix that. +- (-) `realpath --relative-to` doesn't prefix relative paths with `./`. + - (+) There is no need to return the same result as `realpath`. + +</details> + +### Representation of the current directory +[curdir]: #representation-of-the-current-directory + +Observing: The subpath that produces the base directory can be represented with `.` or `./` or `./.`. + +Considering: Paths should be as consistent and unambiguous as possible. + +Decision: It should be `./.`. + +<details> +<summary>Arguments</summary> + +- (+) `./` would be inconsistent with [the decision to not persist trailing slashes][trailing-slashes]. +- (-) `.` is how `realpath` normalises paths. +- (+) `.` can be interpreted as a shell command (it's a builtin for sourcing files in `bash` and `zsh`). +- (+) `.` would be the only path without a `/`. It could not be used as a Nix path expression, since those require at least one `/` to be parsed as such. +- (-) `./.` is rather long. + - (-) We don't require users to type this though, as it's only output by the library. + As inputs all three variants are supported for subpaths (and we can't do anything about absolute paths) +- (-) `builtins.dirOf "foo" == "."`, so `.` would be consistent with that. +- (+) `./.` is consistent with the [decision to have leading `./`][leading-dots]. +- (+) `./.` is a valid Nix path expression, although this property does not hold for every relative path or subpath. + +</details> + +### Subpath representation +[relrepr]: #subpath-representation + +Observing: Subpaths such as `foo/bar` can be represented in various ways: +- string: `"foo/bar"` +- list with all the components: `[ "foo" "bar" ]` +- attribute set: `{ type = "relative-path"; components = [ "foo" "bar" ]; }` + +Considering: Paths should be as safe to use as possible. We should generate string outputs in the library and not encourage users to do that themselves. + +Decision: Paths are represented as strings. + +<details> +<summary>Arguments</summary> + +- (+) It's simpler for the users of the library. One doesn't have to convert a path a string before it can be used. + - (+) Naively converting the list representation to a string with `concatStringsSep "/"` would break for `[]`, requiring library users to be more careful. +- (+) It doesn't encourage people to do their own path processing and instead use the library. + With a list representation it would seem easy to just use `lib.lists.init` to get the parent directory, but then it breaks for `.`, which would be represented as `[ ]`. +- (+) `+` is convenient and doesn't work on lists and attribute sets. + - (-) Shouldn't use `+` anyways, we export safer functions for path manipulation. + +</details> + +### Parent directory +[parents]: #parent-directory + +Observing: Relative paths can have `..` components, which refer to the parent directory. + +Considering: Paths should be as safe and unambiguous as possible. + +Decision: `..` path components in string paths are not supported, neither as inputs nor as outputs. Hence, string paths are called subpaths, rather than relative paths. + +<details> +<summary>Arguments</summary> + +- (+) If we wanted relative paths to behave according to the "physical" interpretation (as a directory tree with relations between nodes), it would require resolving symlinks, since e.g. `foo/..` would not be the same as `.` if `foo` is a symlink. + - (-) The "logical" interpretation is also valid (treating paths as a sequence of names), and is used by some software. It is simpler, and not using symlinks at all is safer. + - (+) Mixing both models can lead to surprises. + - (+) We can't resolve symlinks without filesystem access. + - (+) Nix also doesn't support reading symlinks at evaluation time. + - (-) We could just not handle such cases, e.g. `equals "foo" "foo/bar/.. == false`. The paths are different, we don't need to check whether the paths point to the same thing. + - (+) Assume we said `relativeTo /foo /bar == "../bar"`. If this is used like `/bar/../foo` in the end, and `bar` turns out to be a symlink to somewhere else, this won't be accurate. + - (-) We could decide to not support such ambiguous operations, or mark them as such, e.g. the normal `relativeTo` will error on such a case, but there could be `extendedRelativeTo` supporting that. +- (-) `..` are a part of paths, a path library should therefore support it. + - (+) If we can convincingly argue that all such use cases are better done e.g. with runtime tools, the library not supporting it can nudge people towards using those. +- (-) We could allow "..", but only in the prefix. + - (+) Then we'd have to throw an error for doing `append /some/path "../foo"`, making it non-composable. + - (+) The same is for returning paths with `..`: `relativeTo /foo /bar => "../bar"` would produce a non-composable path. +- (+) We argue that `..` is not needed at the Nix evaluation level, since we'd always start evaluation from the project root and don't go up from there. + - (+) `..` is supported in Nix paths, turning them into absolute paths. + - (-) This is ambiguous in the presence of symlinks. +- (+) If you need `..` for building or runtime, you can use build-/run-time tooling to create those (e.g. `realpath` with `--relative-to`), or use absolute paths instead. + This also gives you the ability to correctly handle symlinks. + +</details> + +### Trailing slashes +[trailing-slashes]: #trailing-slashes + +Observing: Subpaths can contain trailing slashes, like `foo/`, indicating that the path points to a directory and not a file. + +Considering: Paths should be as consistent as possible, there should only be a single normalisation for the same path. + +Decision: All functions remove trailing slashes in their results. + +<details> +<summary>Arguments</summary> + +- (+) It allows normalisations to be unique, in that there's only a single normalisation for the same path. If trailing slashes were preserved, both `foo/bar` and `foo/bar/` would be valid but different normalisations for the same path. +- Comparison to other frameworks to figure out the least surprising behavior: + - (+) Nix itself doesn't support trailing slashes when parsing and doesn't preserve them when appending paths. + - (-) [Rust's std::path](https://doc.rust-lang.org/std/path/index.html) does preserve them during [construction](https://doc.rust-lang.org/std/path/struct.Path.html#method.new). + - (+) Doesn't preserve them when returning individual [components](https://doc.rust-lang.org/std/path/struct.Path.html#method.components). + - (+) Doesn't preserve them when [canonicalizing](https://doc.rust-lang.org/std/path/struct.Path.html#method.canonicalize). + - (+) [Python 3's pathlib](https://docs.python.org/3/library/pathlib.html#module-pathlib) doesn't preserve them during [construction](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath). + - Notably it represents the individual components as a list internally. + - (-) [Haskell's filepath](https://hackage.haskell.org/package/filepath-1.4.100.0) has [explicit support](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html#g:6) for handling trailing slashes. + - (-) Does preserve them for [normalisation](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html#v:normalise). + - (-) [NodeJS's Path library](https://nodejs.org/api/path.html) preserves trailing slashes for [normalisation](https://nodejs.org/api/path.html#pathnormalizepath). + - (+) For [parsing a path](https://nodejs.org/api/path.html#pathparsepath) into its significant elements, trailing slashes are not preserved. +- (+) Nix's builtin function `dirOf` gives an unexpected result for paths with trailing slashes: `dirOf "foo/bar/" == "foo/bar"`. + Inconsistently, `baseNameOf` works correctly though: `baseNameOf "foo/bar/" == "bar"`. + - (-) We are writing a path library to improve handling of paths though, so we shouldn't use these functions and discourage their use. +- (-) Unexpected result when normalising intermediate paths, like `relative.normalise ("foo" + "/") + "bar" == "foobar"`. + - (+) This is not a practical use case though. + - (+) Don't use `+` to append paths, this library has a `join` function for that. + - (-) Users might use `+` out of habit though. +- (+) The `realpath` command also removes trailing slashes. +- (+) Even with a trailing slash, the path is the same, it's only an indication that it's a directory. + +</details> + +## Other implementations and references + +- [Rust](https://doc.rust-lang.org/std/path/struct.Path.html) +- [Python](https://docs.python.org/3/library/pathlib.html) +- [Haskell](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html) +- [Nodejs](https://nodejs.org/api/path.html) +- [POSIX.1-2017](https://pubs.opengroup.org/onlinepubs/9699919799/nframe.html) diff --git a/path/default.nix b/path/default.nix new file mode 100644 index 000000000..ce5d77599 --- /dev/null +++ b/path/default.nix @@ -0,0 +1,433 @@ +# Functions for working with paths, see ./path.md +{ lib }: +let + + inherit (builtins) + isString + isPath + split + match + typeOf + ; + + inherit (lib.lists) + length + head + last + genList + elemAt + all + concatMap + foldl' + take + drop + ; + + inherit (lib.strings) + concatStringsSep + substring + ; + + inherit (lib.asserts) + assertMsg + ; + + inherit (lib.path.subpath) + isValid + ; + + # Return the reason why a subpath is invalid, or `null` if it's valid + subpathInvalidReason = value: + if ! isString value then + "The given value is of type ${builtins.typeOf value}, but a string was expected" + else if value == "" then + "The given string is empty" + else if substring 0 1 value == "/" then + "The given string \"${value}\" starts with a `/`, representing an absolute path" + # We don't support ".." components, see ./path.md#parent-directory + else if match "(.*/)?\\.\\.(/.*)?" value != null then + "The given string \"${value}\" contains a `..` component, which is not allowed in subpaths" + else null; + + # Split and normalise a relative path string into its components. + # Error for ".." components and doesn't include "." components + splitRelPath = path: + let + # Split the string into its parts using regex for efficiency. This regex + # matches patterns like "/", "/./", "/././", with arbitrarily many "/"s + # together. These are the main special cases: + # - Leading "./" gets split into a leading "." part + # - Trailing "/." or "/" get split into a trailing "." or "" + # part respectively + # + # These are the only cases where "." and "" parts can occur + parts = split "/+(\\./+)*" path; + + # `split` creates a list of 2 * k + 1 elements, containing the k + + # 1 parts, interleaved with k matches where k is the number of + # (non-overlapping) matches. This calculation here gets the number of parts + # back from the list length + # floor( (2 * k + 1) / 2 ) + 1 == floor( k + 1/2 ) + 1 == k + 1 + partCount = length parts / 2 + 1; + + # To assemble the final list of components we want to: + # - Skip a potential leading ".", normalising "./foo" to "foo" + # - Skip a potential trailing "." or "", normalising "foo/" and "foo/." to + # "foo". See ./path.md#trailing-slashes + skipStart = if head parts == "." then 1 else 0; + skipEnd = if last parts == "." || last parts == "" then 1 else 0; + + # We can now know the length of the result by removing the number of + # skipped parts from the total number + componentCount = partCount - skipEnd - skipStart; + + in + # Special case of a single "." path component. Such a case leaves a + # componentCount of -1 due to the skipStart/skipEnd not verifying that + # they don't refer to the same character + if path == "." then [] + + # Generate the result list directly. This is more efficient than a + # combination of `filter`, `init` and `tail`, because here we don't + # allocate any intermediate lists + else genList (index: + # To get to the element we need to add the number of parts we skip and + # multiply by two due to the interleaved layout of `parts` + elemAt parts ((skipStart + index) * 2) + ) componentCount; + + # Join relative path components together + joinRelPath = components: + # Always return relative paths with `./` as a prefix (./path.md#leading-dots-for-relative-paths) + "./" + + # An empty string is not a valid relative path, so we need to return a `.` when we have no components + (if components == [] then "." else concatStringsSep "/" components); + + # Deconstruct a path value type into: + # - root: The filesystem root of the path, generally `/` + # - components: All the path's components + # + # This is similar to `splitString "/" (toString path)` but safer + # because it can distinguish different filesystem roots + deconstructPath = + let + recurse = components: path: + # If the parent of a path is the path itself, then it's a filesystem root + if path == dirOf path then { root = path; inherit components; } + else recurse ([ (baseNameOf path) ] ++ components) (dirOf path); + in recurse []; + + # Used as an abstraction between `hasPrefix`, `hasProperPrefix` and `removePrefix` + # + # Takes four arguments: + # - context: A string describing the callee, for error messages + # - continue: A function with four arguments + # - path1: A path + # - path2: Another path + # + # The function checks whether `path1` is a path and computes its + # decomposition (`deconPath1`) before taking `path2` as an argument, which + # allows the computation to be cached in a thunk between multiple calls. + # With `path2` also provided, it checks whether it's also a path, computes + # the decomposition (`deconPath2`), checks whether both paths have the same + # filesystem root, and then calls `continue path1 deconPath1 path2 deconPath2`, + # allowing the caller to decide what to do with these values. + withTwoDeconstructedPaths = context: continue: + path1: + assert assertMsg + (isPath path1) + "${context}: First argument is of type ${typeOf path1}, but a path was expected"; + let + deconPath1 = deconstructPath path1; + in + path2: + assert assertMsg + (isPath path2) + "${context}: Second argument is of type ${typeOf path2}, but a path was expected"; + let + deconPath2 = deconstructPath path2; + in + assert assertMsg + (deconPath1.root == deconPath2.root) '' + ${context}: Filesystem roots must be the same for both paths, but paths with different roots were given: + first argument: "${toString path1}" (root "${toString deconPath1.root}") + second argument: "${toString path2}" (root "${toString deconPath2.root}")''; + continue path1 deconPath1 path2 deconPath2; + +in /* No rec! Add dependencies on this file at the top. */ { + + /* Append a subpath string to a path. + + Like `path + ("/" + string)` but safer, because it errors instead of returning potentially surprising results. + More specifically, it checks that the first argument is a [path value type](https://nixos.org/manual/nix/stable/language/values.html#type-path"), + and that the second argument is a valid subpath string (see `lib.path.subpath.isValid`). + + Type: + append :: Path -> String -> Path + + Example: + append /foo "bar/baz" + => /foo/bar/baz + + # subpaths don't need to be normalised + append /foo "./bar//baz/./" + => /foo/bar/baz + + # can append to root directory + append /. "foo/bar" + => /foo/bar + + # first argument needs to be a path value type + append "/foo" "bar" + => <error> + + # second argument needs to be a valid subpath string + append /foo /bar + => <error> + append /foo "" + => <error> + append /foo "/bar" + => <error> + append /foo "../bar" + => <error> + */ + append = + # The absolute path to append to + path: + # The subpath string to append + subpath: + assert assertMsg (isPath path) '' + lib.path.append: The first argument is of type ${builtins.typeOf path}, but a path was expected''; + assert assertMsg (isValid subpath) '' + lib.path.append: Second argument is not a valid subpath string: + ${subpathInvalidReason subpath}''; + path + ("/" + subpath); + + hasPrefix = withTwoDeconstructedPaths "lib.path.hasPrefix" (prefix: deconPrefix: path: deconPath: + take (length deconPrefix.components) deconPath.components == deconPrefix.components + ); + + removePrefix = withTwoDeconstructedPaths "lib.path.removePrefix" (prefix: deconPrefix: path: deconPath: + if take (length deconPrefix.components) deconPath.components == deconPrefix.components + then drop (length deconPrefix.components) deconPath.components + else throw '' + lib.path.removePrefix: The first prefix path argument (${toString prefix}) is not a prefix of the second path argument (${toString path})'' + ); + + /* Whether a value is a valid subpath string. + + - The value is a string + + - The string is not empty + + - The string doesn't start with a `/` + + - The string doesn't contain any `..` path components + + Type: + subpath.isValid :: String -> Bool + + Example: + # Not a string + subpath.isValid null + => false + + # Empty string + subpath.isValid "" + => false + + # Absolute path + subpath.isValid "/foo" + => false + + # Contains a `..` path component + subpath.isValid "../foo" + => false + + # Valid subpath + subpath.isValid "foo/bar" + => true + + # Doesn't need to be normalised + subpath.isValid "./foo//bar/" + => true + */ + subpath.isValid = + # The value to check + value: + subpathInvalidReason value == null; + + + /* Join subpath strings together using `/`, returning a normalised subpath string. + + Like `concatStringsSep "/"` but safer, specifically: + + - All elements must be valid subpath strings, see `lib.path.subpath.isValid` + + - The result gets normalised, see `lib.path.subpath.normalise` + + - The edge case of an empty list gets properly handled by returning the neutral subpath `"./."` + + Laws: + + - Associativity: + + subpath.join [ x (subpath.join [ y z ]) ] == subpath.join [ (subpath.join [ x y ]) z ] + + - Identity - `"./."` is the neutral element for normalised paths: + + subpath.join [ ] == "./." + subpath.join [ (subpath.normalise p) "./." ] == subpath.normalise p + subpath.join [ "./." (subpath.normalise p) ] == subpath.normalise p + + - Normalisation - the result is normalised according to `lib.path.subpath.normalise`: + + subpath.join ps == subpath.normalise (subpath.join ps) + + - For non-empty lists, the implementation is equivalent to normalising the result of `concatStringsSep "/"`. + Note that the above laws can be derived from this one. + + ps != [] -> subpath.join ps == subpath.normalise (concatStringsSep "/" ps) + + Type: + subpath.join :: [ String ] -> String + + Example: + subpath.join [ "foo" "bar/baz" ] + => "./foo/bar/baz" + + # normalise the result + subpath.join [ "./foo" "." "bar//./baz/" ] + => "./foo/bar/baz" + + # passing an empty list results in the current directory + subpath.join [ ] + => "./." + + # elements must be valid subpath strings + subpath.join [ /foo ] + => <error> + subpath.join [ "" ] + => <error> + subpath.join [ "/foo" ] + => <error> + subpath.join [ "../foo" ] + => <error> + */ + subpath.join = + # The list of subpaths to join together + subpaths: + # Fast in case all paths are valid + if all isValid subpaths + then joinRelPath (concatMap splitRelPath subpaths) + else + # Otherwise we take our time to gather more info for a better error message + # Strictly go through each path, throwing on the first invalid one + # Tracks the list index in the fold accumulator + foldl' (i: path: + if isValid path + then i + 1 + else throw '' + lib.path.subpath.join: Element at index ${toString i} is not a valid subpath string: + ${subpathInvalidReason path}'' + ) 0 subpaths; + + /* Normalise a subpath. Throw an error if the subpath isn't valid, see + `lib.path.subpath.isValid` + + - Limit repeating `/` to a single one + + - Remove redundant `.` components + + - Remove trailing `/` and `/.` + + - Add leading `./` + + Laws: + + - Idempotency - normalising multiple times gives the same result: + + subpath.normalise (subpath.normalise p) == subpath.normalise p + + - Uniqueness - there's only a single normalisation for the paths that lead to the same file system node: + + subpath.normalise p != subpath.normalise q -> $(realpath ${p}) != $(realpath ${q}) + + - Don't change the result when appended to a Nix path value: + + base + ("/" + p) == base + ("/" + subpath.normalise p) + + - Don't change the path according to `realpath`: + + $(realpath ${p}) == $(realpath ${subpath.normalise p}) + + - Only error on invalid subpaths: + + builtins.tryEval (subpath.normalise p)).success == subpath.isValid p + + Type: + subpath.normalise :: String -> String + + Example: + # limit repeating `/` to a single one + subpath.normalise "foo//bar" + => "./foo/bar" + + # remove redundant `.` components + subpath.normalise "foo/./bar" + => "./foo/bar" + + # add leading `./` + subpath.normalise "foo/bar" + => "./foo/bar" + + # remove trailing `/` + subpath.normalise "foo/bar/" + => "./foo/bar" + + # remove trailing `/.` + subpath.normalise "foo/bar/." + => "./foo/bar" + + # Return the current directory as `./.` + subpath.normalise "." + => "./." + + # error on `..` path components + subpath.normalise "foo/../bar" + => <error> + + # error on empty string + subpath.normalise "" + => <error> + + # error on absolute path + subpath.normalise "/foo" + => <error> + */ + subpath.normalise = + # The subpath string to normalise + subpath: + assert assertMsg (isValid subpath) '' + lib.path.subpath.normalise: Argument is not a valid subpath string: + ${subpathInvalidReason subpath}''; + joinRelPath (splitRelPath subpath); + + commonAncestor = a: b: + let + a' = deconstructPath a; + b' = deconstructPath b; + in + if a'.root != b'.root then + throw "lib.path.commonAncestor: Given paths don't have the same filesystem root" + else + a'.root + ("/" + concatStringsSep "/" (lib.lists.commonPrefix a'.components b'.components)); + + deconstruct = path: deconstructPath path; + construct = { root, components }: root + ("/" + concatStringsSep "/" components); + + components = { + toSubpath = joinRelPath; + fromSubpath = splitRelPath; + }; +} diff --git a/path/tests/default.nix b/path/tests/default.nix new file mode 100644 index 000000000..9a31e4282 --- /dev/null +++ b/path/tests/default.nix @@ -0,0 +1,34 @@ +{ + nixpkgs ? ../../.., + system ? builtins.currentSystem, + pkgs ? import nixpkgs { + config = {}; + overlays = []; + inherit system; + }, + libpath ? ../.., + # Random seed + seed ? null, +}: +pkgs.runCommand "lib-path-tests" { + nativeBuildInputs = with pkgs; [ + nix + jq + bc + ]; +} '' + # Needed to make Nix evaluation work + export NIX_STATE_DIR=$(mktemp -d) + + cp -r ${libpath} lib + export TEST_LIB=$PWD/lib + + echo "Running unit tests lib/path/tests/unit.nix" + nix-instantiate --eval lib/path/tests/unit.nix \ + --argstr libpath "$TEST_LIB" + + echo "Running property tests lib/path/tests/prop.sh" + bash lib/path/tests/prop.sh ${toString seed} + + touch $out +'' diff --git a/path/tests/generate.awk b/path/tests/generate.awk new file mode 100644 index 000000000..811dd0c46 --- /dev/null +++ b/path/tests/generate.awk @@ -0,0 +1,64 @@ +# Generate random path-like strings, separated by null characters. +# +# Invocation: +# +# awk -f ./generate.awk -v <variable>=<value> | tr '\0' '\n' +# +# Customizable variables (all default to 0): +# - seed: Deterministic random seed to use for generation +# - count: Number of paths to generate +# - extradotweight: Give extra weight to dots being generated +# - extraslashweight: Give extra weight to slashes being generated +# - extranullweight: Give extra weight to null being generated, making paths shorter +BEGIN { + # Random seed, passed explicitly for reproducibility + srand(seed) + + # Don't include special characters below 32 + minascii = 32 + # Don't include DEL at 128 + maxascii = 127 + upperascii = maxascii - minascii + + # add extra weight for ., in addition to the one weight from the ascii range + upperdot = upperascii + extradotweight + + # add extra weight for /, in addition to the one weight from the ascii range + upperslash = upperdot + extraslashweight + + # add extra weight for null, indicating the end of the string + # Must be at least 1 to have strings end at all + total = upperslash + 1 + extranullweight + + # new=1 indicates that it's a new string + new=1 + while (count > 0) { + + # Random integer between [0, total) + value = int(rand() * total) + + if (value < upperascii) { + # Ascii range + printf("%c", value + minascii) + new=0 + + } else if (value < upperdot) { + # Dot range + printf "." + new=0 + + } else if (value < upperslash) { + # If it's the start of a new path, only generate a / in 10% of cases + # This is always an invalid subpath, which is not a very interesting case + if (new && rand() > 0.1) continue + printf "/" + + } else { + # Do not generate empty strings + if (new) continue + printf "\x00" + count-- + new=1 + } + } +} diff --git a/path/tests/prop.nix b/path/tests/prop.nix new file mode 100644 index 000000000..67e5c1e9d --- /dev/null +++ b/path/tests/prop.nix @@ -0,0 +1,60 @@ +# Given a list of path-like strings, check some properties of the path library +# using those paths and return a list of attribute sets of the following form: +# +# { <string> = <lib.path.subpath.normalise string>; } +# +# If `normalise` fails to evaluate, the attribute value is set to `""`. +# If not, the resulting value is normalised again and an appropriate attribute set added to the output list. +{ + # The path to the nixpkgs lib to use + libpath, + # A flat directory containing files with randomly-generated + # path-like values + dir, +}: +let + lib = import libpath; + + # read each file into a string + strings = map (name: + builtins.readFile (dir + "/${name}") + ) (builtins.attrNames (builtins.readDir dir)); + + inherit (lib.path.subpath) normalise isValid; + inherit (lib.asserts) assertMsg; + + normaliseAndCheck = str: + let + originalValid = isValid str; + + tryOnce = builtins.tryEval (normalise str); + tryTwice = builtins.tryEval (normalise tryOnce.value); + + absConcatOrig = /. + ("/" + str); + absConcatNormalised = /. + ("/" + tryOnce.value); + in + # Check the lib.path.subpath.normalise property to only error on invalid subpaths + assert assertMsg + (originalValid -> tryOnce.success) + "Even though string \"${str}\" is valid as a subpath, the normalisation for it failed"; + assert assertMsg + (! originalValid -> ! tryOnce.success) + "Even though string \"${str}\" is invalid as a subpath, the normalisation for it succeeded"; + + # Check normalisation idempotency + assert assertMsg + (originalValid -> tryTwice.success) + "For valid subpath \"${str}\", the normalisation \"${tryOnce.value}\" was not a valid subpath"; + assert assertMsg + (originalValid -> tryOnce.value == tryTwice.value) + "For valid subpath \"${str}\", normalising it once gives \"${tryOnce.value}\" but normalising it twice gives a different result: \"${tryTwice.value}\""; + + # Check that normalisation doesn't change a string when appended to an absolute Nix path value + assert assertMsg + (originalValid -> absConcatOrig == absConcatNormalised) + "For valid subpath \"${str}\", appending to an absolute Nix path value gives \"${absConcatOrig}\", but appending the normalised result \"${tryOnce.value}\" gives a different value \"${absConcatNormalised}\""; + + # Return an empty string when failed + if tryOnce.success then tryOnce.value else ""; + +in lib.genAttrs strings normaliseAndCheck diff --git a/path/tests/prop.sh b/path/tests/prop.sh new file mode 100755 index 000000000..e48c6667f --- /dev/null +++ b/path/tests/prop.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +# Property tests for the `lib.path` library +# +# It generates random path-like strings and runs the functions on +# them, checking that the expected laws of the functions hold + +set -euo pipefail +shopt -s inherit_errexit + +# https://stackoverflow.com/a/246128 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if test -z "${TEST_LIB:-}"; then + TEST_LIB=$SCRIPT_DIR/../.. +fi + +tmp="$(mktemp -d)" +clean_up() { + rm -rf "$tmp" +} +trap clean_up EXIT +mkdir -p "$tmp/work" +cd "$tmp/work" + +# Defaulting to a random seed but the first argument can override this +seed=${1:-$RANDOM} +echo >&2 "Using seed $seed, use \`lib/path/tests/prop.sh $seed\` to reproduce this result" + +# The number of random paths to generate. This specific number was chosen to +# be fast enough while still generating enough variety to detect bugs. +count=500 + +debug=0 +# debug=1 # print some extra info +# debug=2 # print generated values + +# Fine tuning parameters to balance the number of generated invalid paths +# to the variance in generated paths. +extradotweight=64 # Larger value: more dots +extraslashweight=64 # Larger value: more slashes +extranullweight=16 # Larger value: shorter strings + +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if [[ "$debug" -ge 1 ]]; then + echo >&2 "Generating $count random path-like strings" +fi + +# Read stream of null-terminated strings entry-by-entry into bash, +# write it to a file and the `strings` array. +declare -a strings=() +mkdir -p "$tmp/strings" +while IFS= read -r -d $'\0' str; do + printf "%s" "$str" > "$tmp/strings/${#strings[@]}" + strings+=("$str") +done < <(awk \ + -f "$SCRIPT_DIR"/generate.awk \ + -v seed="$seed" \ + -v count="$count" \ + -v extradotweight="$extradotweight" \ + -v extraslashweight="$extraslashweight" \ + -v extranullweight="$extranullweight") + +if [[ "$debug" -ge 1 ]]; then + echo >&2 "Trying to normalise the generated path-like strings with Nix" +fi + +# Precalculate all normalisations with a single Nix call. Calling Nix for each +# string individually would take way too long +nix-instantiate --eval --strict --json \ + --argstr libpath "$TEST_LIB" \ + --argstr dir "$tmp/strings" \ + "$SCRIPT_DIR"/prop.nix \ + >"$tmp/result.json" + +# Uses some jq magic to turn the resulting attribute set into an associative +# bash array assignment +declare -A normalised_result="($(jq ' + to_entries + | map("[\(.key | @sh)]=\(.value | @sh)") + | join(" \n")' -r < "$tmp/result.json"))" + +# Looks up a normalisation result for a string +# Checks that the normalisation is only failing iff it's an invalid subpath +# For valid subpaths, returns 0 and prints the normalisation result +# For invalid subpaths, returns 1 +normalise() { + local str=$1 + # Uses the same check for validity as in the library implementation + if [[ "$str" == "" || "$str" == /* || "$str" =~ ^(.*/)?\.\.(/.*)?$ ]]; then + valid= + else + valid=1 + fi + + normalised=${normalised_result[$str]} + # An empty string indicates failure, this is encoded in ./prop.nix + if [[ -n "$normalised" ]]; then + if [[ -n "$valid" ]]; then + echo "$normalised" + else + die "For invalid subpath \"$str\", lib.path.subpath.normalise returned this result: \"$normalised\"" + fi + else + if [[ -n "$valid" ]]; then + die "For valid subpath \"$str\", lib.path.subpath.normalise failed" + else + if [[ "$debug" -ge 2 ]]; then + echo >&2 "String \"$str\" is not a valid subpath" + fi + # Invalid and it correctly failed, we let the caller continue if they catch the exit code + return 1 + fi + fi +} + +# Intermediate result populated by test_idempotency_realpath +# and used in test_normalise_uniqueness +# +# Contains a mapping from a normalised subpath to the realpath result it represents +declare -A norm_to_real + +test_idempotency_realpath() { + if [[ "$debug" -ge 1 ]]; then + echo >&2 "Checking idempotency of each result and making sure the realpath result isn't changed" + fi + + # Count invalid subpaths to display stats + invalid=0 + for str in "${strings[@]}"; do + if ! result=$(normalise "$str"); then + ((invalid++)) || true + continue + fi + + # Check the law that it doesn't change the result of a realpath + mkdir -p -- "$str" "$result" + real_orig=$(realpath -- "$str") + real_norm=$(realpath -- "$result") + + if [[ "$real_orig" != "$real_norm" ]]; then + die "realpath of the original string \"$str\" (\"$real_orig\") is not the same as realpath of the normalisation \"$result\" (\"$real_norm\")" + fi + + if [[ "$debug" -ge 2 ]]; then + echo >&2 "String \"$str\" gets normalised to \"$result\" and file path \"$real_orig\"" + fi + norm_to_real["$result"]="$real_orig" + done + if [[ "$debug" -ge 1 ]]; then + echo >&2 "$(bc <<< "scale=1; 100 / $count * $invalid")% of the total $count generated strings were invalid subpath strings, and were therefore ignored" + fi +} + +test_normalise_uniqueness() { + if [[ "$debug" -ge 1 ]]; then + echo >&2 "Checking for the uniqueness law" + fi + + for norm_p in "${!norm_to_real[@]}"; do + real_p=${norm_to_real["$norm_p"]} + for norm_q in "${!norm_to_real[@]}"; do + real_q=${norm_to_real["$norm_q"]} + # Checks normalisation uniqueness law for each pair of values + if [[ "$norm_p" != "$norm_q" && "$real_p" == "$real_q" ]]; then + die "Normalisations \"$norm_p\" and \"$norm_q\" are different, but the realpath of them is the same: \"$real_p\"" + fi + done + done +} + +test_idempotency_realpath +test_normalise_uniqueness + +echo >&2 tests ok diff --git a/path/tests/unit.nix b/path/tests/unit.nix new file mode 100644 index 000000000..61c4ab4d6 --- /dev/null +++ b/path/tests/unit.nix @@ -0,0 +1,193 @@ +# Unit tests for lib.path functions. Use `nix-build` in this directory to +# run these +{ libpath }: +let + lib = import libpath; + inherit (lib.path) append subpath; + + cases = lib.runTests { + # Test examples from the lib.path.append documentation + testAppendExample1 = { + expr = append /foo "bar/baz"; + expected = /foo/bar/baz; + }; + testAppendExample2 = { + expr = append /foo "./bar//baz/./"; + expected = /foo/bar/baz; + }; + testAppendExample3 = { + expr = append /. "foo/bar"; + expected = /foo/bar; + }; + testAppendExample4 = { + expr = (builtins.tryEval (append "/foo" "bar")).success; + expected = false; + }; + testAppendExample5 = { + expr = (builtins.tryEval (append /foo /bar)).success; + expected = false; + }; + testAppendExample6 = { + expr = (builtins.tryEval (append /foo "")).success; + expected = false; + }; + testAppendExample7 = { + expr = (builtins.tryEval (append /foo "/bar")).success; + expected = false; + }; + testAppendExample8 = { + expr = (builtins.tryEval (append /foo "../bar")).success; + expected = false; + }; + + # Test examples from the lib.path.subpath.isValid documentation + testSubpathIsValidExample1 = { + expr = subpath.isValid null; + expected = false; + }; + testSubpathIsValidExample2 = { + expr = subpath.isValid ""; + expected = false; + }; + testSubpathIsValidExample3 = { + expr = subpath.isValid "/foo"; + expected = false; + }; + testSubpathIsValidExample4 = { + expr = subpath.isValid "../foo"; + expected = false; + }; + testSubpathIsValidExample5 = { + expr = subpath.isValid "foo/bar"; + expected = true; + }; + testSubpathIsValidExample6 = { + expr = subpath.isValid "./foo//bar/"; + expected = true; + }; + # Some extra tests + testSubpathIsValidTwoDotsEnd = { + expr = subpath.isValid "foo/.."; + expected = false; + }; + testSubpathIsValidTwoDotsMiddle = { + expr = subpath.isValid "foo/../bar"; + expected = false; + }; + testSubpathIsValidTwoDotsPrefix = { + expr = subpath.isValid "..foo"; + expected = true; + }; + testSubpathIsValidTwoDotsSuffix = { + expr = subpath.isValid "foo.."; + expected = true; + }; + testSubpathIsValidTwoDotsPrefixComponent = { + expr = subpath.isValid "foo/..bar/baz"; + expected = true; + }; + testSubpathIsValidTwoDotsSuffixComponent = { + expr = subpath.isValid "foo/bar../baz"; + expected = true; + }; + testSubpathIsValidThreeDots = { + expr = subpath.isValid "..."; + expected = true; + }; + testSubpathIsValidFourDots = { + expr = subpath.isValid "...."; + expected = true; + }; + testSubpathIsValidThreeDotsComponent = { + expr = subpath.isValid "foo/.../bar"; + expected = true; + }; + testSubpathIsValidFourDotsComponent = { + expr = subpath.isValid "foo/..../bar"; + expected = true; + }; + + # Test examples from the lib.path.subpath.join documentation + testSubpathJoinExample1 = { + expr = subpath.join [ "foo" "bar/baz" ]; + expected = "./foo/bar/baz"; + }; + testSubpathJoinExample2 = { + expr = subpath.join [ "./foo" "." "bar//./baz/" ]; + expected = "./foo/bar/baz"; + }; + testSubpathJoinExample3 = { + expr = subpath.join [ ]; + expected = "./."; + }; + testSubpathJoinExample4 = { + expr = (builtins.tryEval (subpath.join [ /foo ])).success; + expected = false; + }; + testSubpathJoinExample5 = { + expr = (builtins.tryEval (subpath.join [ "" ])).success; + expected = false; + }; + testSubpathJoinExample6 = { + expr = (builtins.tryEval (subpath.join [ "/foo" ])).success; + expected = false; + }; + testSubpathJoinExample7 = { + expr = (builtins.tryEval (subpath.join [ "../foo" ])).success; + expected = false; + }; + + # Test examples from the lib.path.subpath.normalise documentation + testSubpathNormaliseExample1 = { + expr = subpath.normalise "foo//bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample2 = { + expr = subpath.normalise "foo/./bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample3 = { + expr = subpath.normalise "foo/bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample4 = { + expr = subpath.normalise "foo/bar/"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample5 = { + expr = subpath.normalise "foo/bar/."; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample6 = { + expr = subpath.normalise "."; + expected = "./."; + }; + testSubpathNormaliseExample7 = { + expr = (builtins.tryEval (subpath.normalise "foo/../bar")).success; + expected = false; + }; + testSubpathNormaliseExample8 = { + expr = (builtins.tryEval (subpath.normalise "")).success; + expected = false; + }; + testSubpathNormaliseExample9 = { + expr = (builtins.tryEval (subpath.normalise "/foo")).success; + expected = false; + }; + # Some extra tests + testSubpathNormaliseIsValidDots = { + expr = subpath.normalise "./foo/.bar/.../baz...qux"; + expected = "./foo/.bar/.../baz...qux"; + }; + testSubpathNormaliseWrongType = { + expr = (builtins.tryEval (subpath.normalise null)).success; + expected = false; + }; + testSubpathNormaliseTwoDots = { + expr = (builtins.tryEval (subpath.normalise "..")).success; + expected = false; + }; + }; +in + if cases == [] then "Unit tests successful" + else throw "Path unit tests failed: ${lib.generators.toPretty {} cases}" diff --git a/source-types.nix b/source-types.nix new file mode 100644 index 000000000..c4f263dcf --- /dev/null +++ b/source-types.nix @@ -0,0 +1,19 @@ +{ lib }: + +let + defaultSourceType = tname: { + shortName = tname; + isSource = false; + }; +in lib.mapAttrs (tname: tset: defaultSourceType tname // tset) { + + fromSource = { + isSource = true; + }; + + binaryNativeCode = {}; + + binaryBytecode = {}; + + binaryFirmware = {}; +} diff --git a/sources.nix b/sources.nix new file mode 100644 index 000000000..d990777c6 --- /dev/null +++ b/sources.nix @@ -0,0 +1,292 @@ +# Functions for copying sources to the Nix store. +{ lib }: + +# Tested in lib/tests/sources.sh +let + inherit (builtins) + match + readDir + split + storeDir + tryEval + ; + inherit (lib) + boolToString + filter + getAttr + isString + pathExists + readFile + ; + inherit (lib.filesystem) + pathType + pathIsDirectory + pathIsRegularFile + ; + + /* + A basic filter for `cleanSourceWith` that removes + directories of version control system, backup files (*~) + and some generated files. + */ + cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! ( + # Filter out version control software files/directories + (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) || + # Filter out editor backup / swap files. + lib.hasSuffix "~" baseName || + match "^\\.sw[a-z]$" baseName != null || + match "^\\..*\\.sw[a-z]$" baseName != null || + + # Filter out generates files. + lib.hasSuffix ".o" baseName || + lib.hasSuffix ".so" baseName || + # Filter out nix-build result symlinks + (type == "symlink" && lib.hasPrefix "result" baseName) || + # Filter out sockets and other types of files we can't have in the store. + (type == "unknown") + ); + + /* + Filters a source tree removing version control files and directories using cleanSourceFilter. + + Example: + cleanSource ./. + */ + cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; }; + + /* + Like `builtins.filterSource`, except it will compose with itself, + allowing you to chain multiple calls together without any + intermediate copies being put in the nix store. + + Example: + lib.cleanSourceWith { + filter = f; + src = lib.cleanSourceWith { + filter = g; + src = ./.; + }; + } + # Succeeds! + + builtins.filterSource f (builtins.filterSource g ./.) + # Fails! + + */ + cleanSourceWith = + { + # A path or cleanSourceWith result to filter and/or rename. + src, + # Optional with default value: constant true (include everything) + # The function will be combined with the && operator such + # that src.filter is called lazily. + # For implementing a filter, see + # https://nixos.org/nix/manual/#builtin-filterSource + # Type: A function (path -> type -> bool) + filter ? _path: _type: true, + # Optional name to use as part of the store path. + # This defaults to `src.name` or otherwise `"source"`. + name ? null + }: + let + orig = toSourceAttributes src; + in fromSourceAttributes { + inherit (orig) origSrc; + filter = path: type: filter path type && orig.filter path type; + name = if name != null then name else orig.name; + }; + + /* + Add logging to a source, for troubleshooting the filtering behavior. + Type: + sources.trace :: sourceLike -> Source + */ + trace = + # Source to debug. The returned source will behave like this source, but also log its filter invocations. + src: + let + attrs = toSourceAttributes src; + in + fromSourceAttributes ( + attrs // { + filter = path: type: + let + r = attrs.filter path type; + in + builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r; + } + ) // { + satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant; + }; + + /* + Filter sources by a list of regular expressions. + + Example: src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"] + */ + sourceByRegex = src: regexes: + let + isFiltered = src ? _isLibCleanSourceWith; + origSrc = if isFiltered then src.origSrc else src; + in lib.cleanSourceWith { + filter = (path: type: + let relPath = lib.removePrefix (toString origSrc + "/") (toString path); + in lib.any (re: match re relPath != null) regexes); + inherit src; + }; + + /* + Get all files ending with the specified suffices from the given + source directory or its descendants, omitting files that do not match + any suffix. The result of the example below will include files like + `./dir/module.c` and `./dir/subdir/doc.xml` if present. + + Type: sourceLike -> [String] -> Source + + Example: + sourceFilesBySuffices ./. [ ".xml" ".c" ] + */ + sourceFilesBySuffices = + # Path or source containing the files to be returned + src: + # A list of file suffix strings + exts: + let filter = name: type: + let base = baseNameOf (toString name); + in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts; + in cleanSourceWith { inherit filter src; }; + + pathIsGitRepo = path: (_commitIdFromGitRepoOrError path)?value; + + /* + Get the commit id of a git repo. + + Example: commitIdFromGitRepo <nixpkgs/.git> + */ + commitIdFromGitRepo = path: + let commitIdOrError = _commitIdFromGitRepoOrError path; + in commitIdOrError.value or (throw commitIdOrError.error); + + # Get the commit id of a git repo. + + # Returns `{ value = commitHash }` or `{ error = "... message ..." }`. + + # Example: commitIdFromGitRepo <nixpkgs/.git> + # not exported, used for commitIdFromGitRepo + _commitIdFromGitRepoOrError = + let readCommitFromFile = file: path: + let fileName = path + "/${file}"; + packedRefsName = path + "/packed-refs"; + absolutePath = base: path: + if lib.hasPrefix "/" path + then path + else toString (/. + "${base}/${path}"); + in if pathIsRegularFile path + # Resolve git worktrees. See gitrepository-layout(5) + then + let m = match "^gitdir: (.*)$" (lib.fileContents path); + in if m == null + then { error = "File contains no gitdir reference: " + path; } + else + let gitDir = absolutePath (dirOf path) (lib.head m); + commonDir'' = if pathIsRegularFile "${gitDir}/commondir" + then lib.fileContents "${gitDir}/commondir" + else gitDir; + commonDir' = lib.removeSuffix "/" commonDir''; + commonDir = absolutePath gitDir commonDir'; + refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}"; + in readCommitFromFile refFile commonDir + + else if pathIsRegularFile fileName + # Sometimes git stores the commitId directly in the file but + # sometimes it stores something like: «ref: refs/heads/branch-name» + then + let fileContent = lib.fileContents fileName; + matchRef = match "^ref: (.*)$" fileContent; + in if matchRef == null + then { value = fileContent; } + else readCommitFromFile (lib.head matchRef) path + + else if pathIsRegularFile packedRefsName + # Sometimes, the file isn't there at all and has been packed away in the + # packed-refs file, so we have to grep through it: + then + let fileContent = readFile packedRefsName; + matchRef = match "([a-z0-9]+) ${file}"; + isRef = s: isString s && (matchRef s) != null; + # there is a bug in libstdc++ leading to stackoverflow for long strings: + # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795 + refs = filter isRef (split "\n" fileContent); + in if refs == [] + then { error = "Could not find " + file + " in " + packedRefsName; } + else { value = lib.head (matchRef (lib.head refs)); } + + else { error = "Not a .git directory: " + toString path; }; + in readCommitFromFile "HEAD"; + + pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir); + + canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src)); + + # -------------------------------------------------------------------------- # + # Internal functions + # + + # toSourceAttributes : sourceLike -> SourceAttrs + # + # Convert any source-like object into a simple, singular representation. + # We don't expose this representation in order to avoid having a fifth path- + # like class of objects in the wild. + # (Existing ones being: paths, strings, sources and x//{outPath}) + # So instead of exposing internals, we build a library of combinator functions. + toSourceAttributes = src: + let + isFiltered = src ? _isLibCleanSourceWith; + in + { + # The original path + origSrc = if isFiltered then src.origSrc else src; + filter = if isFiltered then src.filter else _: _: true; + name = if isFiltered then src.name else "source"; + }; + + # fromSourceAttributes : SourceAttrs -> Source + # + # Inverse of toSourceAttributes for Source objects. + fromSourceAttributes = { origSrc, filter, name }: + { + _isLibCleanSourceWith = true; + inherit origSrc filter name; + outPath = builtins.path { inherit filter name; path = origSrc; }; + }; + +in { + + pathType = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathType has been moved to lib.filesystem.pathType." + lib.filesystem.pathType; + + pathIsDirectory = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory." + lib.filesystem.pathIsDirectory; + + pathIsRegularFile = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile." + lib.filesystem.pathIsRegularFile; + + inherit + pathIsGitRepo + commitIdFromGitRepo + + cleanSource + cleanSourceWith + cleanSourceFilter + pathHasContext + canCleanSource + + sourceByRegex + sourceFilesBySuffices + + trace + ; +} diff --git a/strings-with-deps.nix b/strings-with-deps.nix new file mode 100644 index 000000000..7b88b018d --- /dev/null +++ b/strings-with-deps.nix @@ -0,0 +1,84 @@ +{ lib }: +/* +Usage: + + You define you custom builder script by adding all build steps to a list. + for example: + builder = writeScript "fsg-4.4-builder" + (textClosure [doUnpack addInputs preBuild doMake installPhase doForceShare]); + + a step is defined by noDepEntry, fullDepEntry or packEntry. + To ensure that prerequisite are met those are added before the task itself by + textClosureDupList. Duplicated items are removed again. + + See trace/nixpkgs/trunk/pkgs/top-level/builder-defs.nix for some predefined build steps + + Attention: + + let + pkgs = (import <nixpkgs>) {}; + in let + inherit (pkgs.stringsWithDeps) fullDepEntry packEntry noDepEntry textClosureMap; + inherit (pkgs.lib) id; + + nameA = noDepEntry "Text a"; + nameB = fullDepEntry "Text b" ["nameA"]; + nameC = fullDepEntry "Text c" ["nameA"]; + + stages = { + nameHeader = noDepEntry "#! /bin/sh \n"; + inherit nameA nameB nameC; + }; + in + textClosureMap id stages + [ "nameHeader" "nameA" "nameB" "nameC" + nameC # <- added twice. add a dep entry if you know that it will be added once only [1] + "nameB" # <- this will not be added again because the attr name (reference) is used + ] + + # result: Str("#! /bin/sh \n\nText a\nText b\nText c\nText c",[]) + + [1] maybe this behaviour should be removed to keep things simple (?) +*/ + +let + inherit (lib) + concatStringsSep + head + isAttrs + listToAttrs + tail + ; +in +rec { + + /* !!! The interface of this function is kind of messed up, since + it's way too overloaded and almost but not quite computes a + topological sort of the depstrings. */ + + textClosureList = predefined: arg: + let + f = done: todo: + if todo == [] then {result = []; inherit done;} + else + let entry = head todo; in + if isAttrs entry then + let x = f done entry.deps; + y = f x.done (tail todo); + in { result = x.result ++ [entry.text] ++ y.result; + done = y.done; + } + else if done ? ${entry} then f done (tail todo) + else f (done // listToAttrs [{name = entry; value = 1;}]) ([predefined.${entry}] ++ tail todo); + in (f {} arg).result; + + textClosureMap = f: predefined: names: + concatStringsSep "\n" (map f (textClosureList predefined names)); + + noDepEntry = text: {inherit text; deps = [];}; + fullDepEntry = text: deps: {inherit text deps;}; + packEntry = deps: {inherit deps; text="";}; + + stringAfter = deps: text: { inherit text deps; }; + +} diff --git a/strings.nix b/strings.nix new file mode 100644 index 000000000..e875520c6 --- /dev/null +++ b/strings.nix @@ -0,0 +1,1243 @@ +/* String manipulation functions. */ +{ lib }: +let + + inherit (builtins) length; + + inherit (lib.trivial) warnIf; + +asciiTable = import ./ascii-table.nix; + +in + +rec { + + inherit (builtins) + compareVersions + elem + elemAt + filter + fromJSON + head + isInt + isList + isAttrs + isPath + isString + match + parseDrvName + readFile + replaceStrings + split + storeDir + stringLength + substring + tail + toJSON + typeOf + unsafeDiscardStringContext + ; + + /* Concatenate a list of strings. + + Type: concatStrings :: [string] -> string + + Example: + concatStrings ["foo" "bar"] + => "foobar" + */ + concatStrings = builtins.concatStringsSep ""; + + /* Map a function over a list and concatenate the resulting strings. + + Type: concatMapStrings :: (a -> string) -> [a] -> string + + Example: + concatMapStrings (x: "a" + x) ["foo" "bar"] + => "afooabar" + */ + concatMapStrings = f: list: concatStrings (map f list); + + /* Like `concatMapStrings` except that the f functions also gets the + position as a parameter. + + Type: concatImapStrings :: (int -> a -> string) -> [a] -> string + + Example: + concatImapStrings (pos: x: "${toString pos}-${x}") ["foo" "bar"] + => "1-foo2-bar" + */ + concatImapStrings = f: list: concatStrings (lib.imap1 f list); + + /* Place an element between each element of a list + + Type: intersperse :: a -> [a] -> [a] + + Example: + intersperse "/" ["usr" "local" "bin"] + => ["usr" "/" "local" "/" "bin"]. + */ + intersperse = + # Separator to add between elements + separator: + # Input list + list: + if list == [] || length list == 1 + then list + else tail (lib.concatMap (x: [separator x]) list); + + /* Concatenate a list of strings with a separator between each element + + Type: concatStringsSep :: string -> [string] -> string + + Example: + concatStringsSep "/" ["usr" "local" "bin"] + => "usr/local/bin" + */ + concatStringsSep = builtins.concatStringsSep or (separator: list: + lib.foldl' (x: y: x + y) "" (intersperse separator list)); + + /* Maps a function over a list of strings and then concatenates the + result with the specified separator interspersed between + elements. + + Type: concatMapStringsSep :: string -> (a -> string) -> [a] -> string + + Example: + concatMapStringsSep "-" (x: toUpper x) ["foo" "bar" "baz"] + => "FOO-BAR-BAZ" + */ + concatMapStringsSep = + # Separator to add between elements + sep: + # Function to map over the list + f: + # List of input strings + list: concatStringsSep sep (map f list); + + /* Same as `concatMapStringsSep`, but the mapping function + additionally receives the position of its argument. + + Type: concatIMapStringsSep :: string -> (int -> a -> string) -> [a] -> string + + Example: + concatImapStringsSep "-" (pos: x: toString (x / pos)) [ 6 6 6 ] + => "6-3-2" + */ + concatImapStringsSep = + # Separator to add between elements + sep: + # Function that receives elements and their positions + f: + # List of input strings + list: concatStringsSep sep (lib.imap1 f list); + + /* Concatenate a list of strings, adding a newline at the end of each one. + Defined as `concatMapStrings (s: s + "\n")`. + + Type: concatLines :: [string] -> string + + Example: + concatLines [ "foo" "bar" ] + => "foo\nbar\n" + */ + concatLines = concatMapStrings (s: s + "\n"); + + /* Construct a Unix-style, colon-separated search path consisting of + the given `subDir` appended to each of the given paths. + + Type: makeSearchPath :: string -> [string] -> string + + Example: + makeSearchPath "bin" ["/root" "/usr" "/usr/local"] + => "/root/bin:/usr/bin:/usr/local/bin" + makeSearchPath "bin" [""] + => "/bin" + */ + makeSearchPath = + # Directory name to append + subDir: + # List of base paths + paths: + concatStringsSep ":" (map (path: path + "/" + subDir) (filter (x: x != null) paths)); + + /* Construct a Unix-style search path by appending the given + `subDir` to the specified `output` of each of the packages. If no + output by the given name is found, fallback to `.out` and then to + the default. + + Type: string -> string -> [package] -> string + + Example: + makeSearchPathOutput "dev" "bin" [ pkgs.openssl pkgs.zlib ] + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev/bin:/nix/store/wwh7mhwh269sfjkm6k5665b5kgp7jrk2-zlib-1.2.8/bin" + */ + makeSearchPathOutput = + # Package output to use + output: + # Directory name to append + subDir: + # List of packages + pkgs: makeSearchPath subDir (map (lib.getOutput output) pkgs); + + /* Construct a library search path (such as RPATH) containing the + libraries for a set of packages + + Example: + makeLibraryPath [ "/usr" "/usr/local" ] + => "/usr/lib:/usr/local/lib" + pkgs = import <nixpkgs> { } + makeLibraryPath [ pkgs.openssl pkgs.zlib ] + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r/lib:/nix/store/wwh7mhwh269sfjkm6k5665b5kgp7jrk2-zlib-1.2.8/lib" + */ + makeLibraryPath = makeSearchPathOutput "lib" "lib"; + + /* Construct a binary search path (such as $PATH) containing the + binaries for a set of packages. + + Example: + makeBinPath ["/root" "/usr" "/usr/local"] + => "/root/bin:/usr/bin:/usr/local/bin" + */ + makeBinPath = makeSearchPathOutput "bin" "bin"; + + /* Normalize path, removing extraneous /s + + Type: normalizePath :: string -> string + + Example: + normalizePath "/a//b///c/" + => "/a/b/c/" + */ + normalizePath = s: + warnIf + (isPath s) + '' + lib.strings.normalizePath: The argument (${toString s}) is a path value, but only strings are supported. + Path values are always normalised in Nix, so there's no need to call this function on them. + This function also copies the path to the Nix store and returns the store path, the same as "''${path}" will, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + ( + builtins.foldl' + (x: y: if y == "/" && hasSuffix "/" x then x else x+y) + "" + (stringToCharacters s) + ); + + /* Depending on the boolean `cond', return either the given string + or the empty string. Useful to concatenate against a bigger string. + + Type: optionalString :: bool -> string -> string + + Example: + optionalString true "some-string" + => "some-string" + optionalString false "some-string" + => "" + */ + optionalString = + # Condition + cond: + # String to return if condition is true + string: if cond then string else ""; + + /* Determine whether a string has given prefix. + + Type: hasPrefix :: string -> string -> bool + + Example: + hasPrefix "foo" "foobar" + => true + hasPrefix "foo" "barfoo" + => false + */ + hasPrefix = + # Prefix to check for + pref: + # Input string + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath pref) + '' + lib.strings.hasPrefix: The first argument (${toString pref}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (substring 0 (stringLength pref) str == pref); + + /* Determine whether a string has given suffix. + + Type: hasSuffix :: string -> string -> bool + + Example: + hasSuffix "foo" "foobar" + => false + hasSuffix "foo" "barfoo" + => true + */ + hasSuffix = + # Suffix to check for + suffix: + # Input string + content: + let + lenContent = stringLength content; + lenSuffix = stringLength suffix; + in + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath suffix) + '' + lib.strings.hasSuffix: The first argument (${toString suffix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + ( + lenContent >= lenSuffix + && substring (lenContent - lenSuffix) lenContent content == suffix + ); + + /* Determine whether a string contains the given infix + + Type: hasInfix :: string -> string -> bool + + Example: + hasInfix "bc" "abcd" + => true + hasInfix "ab" "abcd" + => true + hasInfix "cd" "abcd" + => true + hasInfix "foo" "abcd" + => false + */ + hasInfix = infix: content: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath infix) + '' + lib.strings.hasInfix: The first argument (${toString infix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (builtins.match ".*${escapeRegex infix}.*" "${content}" != null); + + /* Convert a string to a list of characters (i.e. singleton strings). + This allows you to, e.g., map a function over each character. However, + note that this will likely be horribly inefficient; Nix is not a + general purpose programming language. Complex string manipulations + should, if appropriate, be done in a derivation. + Also note that Nix treats strings as a list of bytes and thus doesn't + handle unicode. + + Type: stringToCharacters :: string -> [string] + + Example: + stringToCharacters "" + => [ ] + stringToCharacters "abc" + => [ "a" "b" "c" ] + stringToCharacters "🦄" + => [ "�" "�" "�" "�" ] + */ + stringToCharacters = s: + map (p: substring p 1 s) (lib.range 0 (stringLength s - 1)); + + /* Manipulate a string character by character and replace them by + strings before concatenating the results. + + Type: stringAsChars :: (string -> string) -> string -> string + + Example: + stringAsChars (x: if x == "a" then "i" else x) "nax" + => "nix" + */ + stringAsChars = + # Function to map over each individual character + f: + # Input string + s: concatStrings ( + map f (stringToCharacters s) + ); + + /* Convert char to ascii value, must be in printable range + + Type: charToInt :: string -> int + + Example: + charToInt "A" + => 65 + charToInt "(" + => 40 + + */ + charToInt = c: builtins.getAttr c asciiTable; + + /* Escape occurrence of the elements of `list` in `string` by + prefixing it with a backslash. + + Type: escape :: [string] -> string -> string + + Example: + escape ["(" ")"] "(foo)" + => "\\(foo\\)" + */ + escape = list: replaceStrings list (map (c: "\\${c}") list); + + /* Escape occurrence of the element of `list` in `string` by + converting to its ASCII value and prefixing it with \\x. + Only works for printable ascii characters. + + Type: escapeC = [string] -> string -> string + + Example: + escapeC [" "] "foo bar" + => "foo\\x20bar" + + */ + escapeC = list: replaceStrings list (map (c: "\\x${ toLower (lib.toHexString (charToInt c))}") list); + + /* Escape the string so it can be safely placed inside a URL + query. + + Type: escapeURL :: string -> string + + Example: + escapeURL "foo/bar baz" + => "foo%2Fbar%20baz" + */ + escapeURL = let + unreserved = [ "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_" "." "~" ]; + toEscape = builtins.removeAttrs asciiTable unreserved; + in + replaceStrings (builtins.attrNames toEscape) (lib.mapAttrsToList (_: c: "%${fixedWidthString 2 "0" (lib.toHexString c)}") toEscape); + + /* Quote string to be used safely within the Bourne shell. + + Type: escapeShellArg :: string -> string + + Example: + escapeShellArg "esc'ape\nme" + => "'esc'\\''ape\nme'" + */ + escapeShellArg = arg: "'${replaceStrings ["'"] ["'\\''"] (toString arg)}'"; + + /* Quote all arguments to be safely passed to the Bourne shell. + + Type: escapeShellArgs :: [string] -> string + + Example: + escapeShellArgs ["one" "two three" "four'five"] + => "'one' 'two three' 'four'\\''five'" + */ + escapeShellArgs = concatMapStringsSep " " escapeShellArg; + + /* Test whether the given name is a valid POSIX shell variable name. + + Type: string -> bool + + Example: + isValidPosixName "foo_bar000" + => true + isValidPosixName "0-bad.jpg" + => false + */ + isValidPosixName = name: match "[a-zA-Z_][a-zA-Z0-9_]*" name != null; + + /* Translate a Nix value into a shell variable declaration, with proper escaping. + + The value can be a string (mapped to a regular variable), a list of strings + (mapped to a Bash-style array) or an attribute set of strings (mapped to a + Bash-style associative array). Note that "string" includes string-coercible + values like paths or derivations. + + Strings are translated into POSIX sh-compatible code; lists and attribute sets + assume a shell that understands Bash syntax (e.g. Bash or ZSH). + + Type: string -> (string | listOf string | attrsOf string) -> string + + Example: + '' + ${toShellVar "foo" "some string"} + [[ "$foo" == "some string" ]] + '' + */ + toShellVar = name: value: + lib.throwIfNot (isValidPosixName name) "toShellVar: ${name} is not a valid shell variable name" ( + if isAttrs value && ! isStringLike value then + "declare -A ${name}=(${ + concatStringsSep " " (lib.mapAttrsToList (n: v: + "[${escapeShellArg n}]=${escapeShellArg v}" + ) value) + })" + else if isList value then + "declare -a ${name}=(${escapeShellArgs value})" + else + "${name}=${escapeShellArg value}" + ); + + /* Translate an attribute set into corresponding shell variable declarations + using `toShellVar`. + + Type: attrsOf (string | listOf string | attrsOf string) -> string + + Example: + let + foo = "value"; + bar = foo; + in '' + ${toShellVars { inherit foo bar; }} + [[ "$foo" == "$bar" ]] + '' + */ + toShellVars = vars: concatStringsSep "\n" (lib.mapAttrsToList toShellVar vars); + + /* Turn a string into a Nix expression representing that string + + Type: string -> string + + Example: + escapeNixString "hello\${}\n" + => "\"hello\\\${}\\n\"" + */ + escapeNixString = s: escape ["$"] (toJSON s); + + /* Turn a string into an exact regular expression + + Type: string -> string + + Example: + escapeRegex "[^a-z]*" + => "\\[\\^a-z]\\*" + */ + escapeRegex = escape (stringToCharacters "\\[{()^$?*+|."); + + /* Quotes a string if it can't be used as an identifier directly. + + Type: string -> string + + Example: + escapeNixIdentifier "hello" + => "hello" + escapeNixIdentifier "0abc" + => "\"0abc\"" + */ + escapeNixIdentifier = s: + # Regex from https://github.com/NixOS/nix/blob/d048577909e383439c2549e849c5c2f2016c997e/src/libexpr/lexer.l#L91 + if match "[a-zA-Z_][a-zA-Z0-9_'-]*" s != null + then s else escapeNixString s; + + /* Escapes a string such that it is safe to include verbatim in an XML + document. + + Type: string -> string + + Example: + escapeXML ''"test" 'test' < & >'' + => ""test" 'test' < & >" + */ + escapeXML = builtins.replaceStrings + ["\"" "'" "<" ">" "&"] + [""" "'" "<" ">" "&"]; + + # warning added 12-12-2022 + replaceChars = lib.warn "replaceChars is a deprecated alias of replaceStrings, replace usages of it with replaceStrings." builtins.replaceStrings; + + # Case conversion utilities. + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + /* Converts an ASCII string to lower-case. + + Type: toLower :: string -> string + + Example: + toLower "HOME" + => "home" + */ + toLower = replaceStrings upperChars lowerChars; + + /* Converts an ASCII string to upper-case. + + Type: toUpper :: string -> string + + Example: + toUpper "home" + => "HOME" + */ + toUpper = replaceStrings lowerChars upperChars; + + /* Appends string context from another string. This is an implementation + detail of Nix and should be used carefully. + + Strings in Nix carry an invisible `context` which is a list of strings + representing store paths. If the string is later used in a derivation + attribute, the derivation will properly populate the inputDrvs and + inputSrcs. + + Example: + pkgs = import <nixpkgs> { }; + addContextFrom pkgs.coreutils "bar" + => "bar" + */ + addContextFrom = a: b: substring 0 0 a + b; + + /* Cut a string with a separator and produces a list of strings which + were separated by this separator. + + Example: + splitString "." "foo.bar.baz" + => [ "foo" "bar" "baz" ] + splitString "/" "/usr/local/bin" + => [ "" "usr" "local" "bin" ] + */ + splitString = sep: s: + let + splits = builtins.filter builtins.isString (builtins.split (escapeRegex (toString sep)) (toString s)); + in + map (addContextFrom s) splits; + + /* Return a string without the specified prefix, if the prefix matches. + + Type: string -> string -> string + + Example: + removePrefix "foo." "foo.bar.baz" + => "bar.baz" + removePrefix "xxx" "foo.bar.baz" + => "foo.bar.baz" + */ + removePrefix = + # Prefix to remove if it matches + prefix: + # Input string + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath prefix) + '' + lib.strings.removePrefix: The first argument (${toString prefix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function never removes any prefix in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (let + preLen = stringLength prefix; + sLen = stringLength str; + in + if substring 0 preLen str == prefix then + substring preLen (sLen - preLen) str + else + str); + + /* Return a string without the specified suffix, if the suffix matches. + + Type: string -> string -> string + + Example: + removeSuffix "front" "homefront" + => "home" + removeSuffix "xxx" "homefront" + => "homefront" + */ + removeSuffix = + # Suffix to remove if it matches + suffix: + # Input string + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath suffix) + '' + lib.strings.removeSuffix: The first argument (${toString suffix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function never removes any suffix in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (let + sufLen = stringLength suffix; + sLen = stringLength str; + in + if sufLen <= sLen && suffix == substring (sLen - sufLen) sufLen str then + substring 0 (sLen - sufLen) str + else + str); + + /* Return true if string v1 denotes a version older than v2. + + Example: + versionOlder "1.1" "1.2" + => true + versionOlder "1.1" "1.1" + => false + */ + versionOlder = v1: v2: compareVersions v2 v1 == 1; + + /* Return true if string v1 denotes a version equal to or newer than v2. + + Example: + versionAtLeast "1.1" "1.0" + => true + versionAtLeast "1.1" "1.1" + => true + versionAtLeast "1.1" "1.2" + => false + */ + versionAtLeast = v1: v2: !versionOlder v1 v2; + + /* This function takes an argument that's either a derivation or a + derivation's "name" attribute and extracts the name part from that + argument. + + Example: + getName "youtube-dl-2016.01.01" + => "youtube-dl" + getName pkgs.youtube-dl + => "youtube-dl" + */ + getName = x: + let + parse = drv: (parseDrvName drv).name; + in if isString x + then parse x + else x.pname or (parse x.name); + + /* This function takes an argument that's either a derivation or a + derivation's "name" attribute and extracts the version part from that + argument. + + Example: + getVersion "youtube-dl-2016.01.01" + => "2016.01.01" + getVersion pkgs.youtube-dl + => "2016.01.01" + */ + getVersion = x: + let + parse = drv: (parseDrvName drv).version; + in if isString x + then parse x + else x.version or (parse x.name); + + /* Extract name with version from URL. Ask for separator which is + supposed to start extension. + + Example: + nameFromURL "https://nixos.org/releases/nix/nix-1.7/nix-1.7-x86_64-linux.tar.bz2" "-" + => "nix" + nameFromURL "https://nixos.org/releases/nix/nix-1.7/nix-1.7-x86_64-linux.tar.bz2" "_" + => "nix-1.7-x86" + */ + nameFromURL = url: sep: + let + components = splitString "/" url; + filename = lib.last components; + name = head (splitString sep filename); + in assert name != filename; name; + + /* Create a -D<feature>=<value> string that can be passed to typical Meson + invocations. + + Type: mesonOption :: string -> string -> string + + @param feature The feature to be set + @param value The desired value + + Example: + mesonOption "engine" "opengl" + => "-Dengine=opengl" + */ + mesonOption = feature: value: + assert (lib.isString feature); + assert (lib.isString value); + "-D${feature}=${value}"; + + /* Create a -D<condition>={true,false} string that can be passed to typical + Meson invocations. + + Type: mesonBool :: string -> bool -> string + + @param condition The condition to be made true or false + @param flag The controlling flag of the condition + + Example: + mesonBool "hardened" true + => "-Dhardened=true" + mesonBool "static" false + => "-Dstatic=false" + */ + mesonBool = condition: flag: + assert (lib.isString condition); + assert (lib.isBool flag); + mesonOption condition (lib.boolToString flag); + + /* Create a -D<feature>={enabled,disabled} string that can be passed to + typical Meson invocations. + + Type: mesonEnable :: string -> bool -> string + + @param feature The feature to be enabled or disabled + @param flag The controlling flag + + Example: + mesonEnable "docs" true + => "-Ddocs=enabled" + mesonEnable "savage" false + => "-Dsavage=disabled" + */ + mesonEnable = feature: flag: + assert (lib.isString feature); + assert (lib.isBool flag); + mesonOption feature (if flag then "enabled" else "disabled"); + + /* Create an --{enable,disable}-<feat> string that can be passed to + standard GNU Autoconf scripts. + + Example: + enableFeature true "shared" + => "--enable-shared" + enableFeature false "shared" + => "--disable-shared" + */ + enableFeature = enable: feat: + assert isString feat; # e.g. passing openssl instead of "openssl" + "--${if enable then "enable" else "disable"}-${feat}"; + + /* Create an --{enable-<feat>=<value>,disable-<feat>} string that can be passed to + standard GNU Autoconf scripts. + + Example: + enableFeatureAs true "shared" "foo" + => "--enable-shared=foo" + enableFeatureAs false "shared" (throw "ignored") + => "--disable-shared" + */ + enableFeatureAs = enable: feat: value: enableFeature enable feat + optionalString enable "=${value}"; + + /* Create an --{with,without}-<feat> string that can be passed to + standard GNU Autoconf scripts. + + Example: + withFeature true "shared" + => "--with-shared" + withFeature false "shared" + => "--without-shared" + */ + withFeature = with_: feat: + assert isString feat; # e.g. passing openssl instead of "openssl" + "--${if with_ then "with" else "without"}-${feat}"; + + /* Create an --{with-<feat>=<value>,without-<feat>} string that can be passed to + standard GNU Autoconf scripts. + + Example: + withFeatureAs true "shared" "foo" + => "--with-shared=foo" + withFeatureAs false "shared" (throw "ignored") + => "--without-shared" + */ + withFeatureAs = with_: feat: value: withFeature with_ feat + optionalString with_ "=${value}"; + + /* Create a fixed width string with additional prefix to match + required width. + + This function will fail if the input string is longer than the + requested length. + + Type: fixedWidthString :: int -> string -> string -> string + + Example: + fixedWidthString 5 "0" (toString 15) + => "00015" + */ + fixedWidthString = width: filler: str: + let + strw = lib.stringLength str; + reqWidth = width - (lib.stringLength filler); + in + assert lib.assertMsg (strw <= width) + "fixedWidthString: requested string length (${ + toString width}) must not be shorter than actual length (${ + toString strw})"; + if strw == width then str else filler + fixedWidthString reqWidth filler str; + + /* Format a number adding leading zeroes up to fixed width. + + Example: + fixedWidthNumber 5 15 + => "00015" + */ + fixedWidthNumber = width: n: fixedWidthString width "0" (toString n); + + /* Convert a float to a string, but emit a warning when precision is lost + during the conversion + + Example: + floatToString 0.000001 + => "0.000001" + floatToString 0.0000001 + => trace: warning: Imprecise conversion from float to string 0.000000 + "0.000000" + */ + floatToString = float: let + result = toString float; + precise = float == fromJSON result; + in lib.warnIf (!precise) "Imprecise conversion from float to string ${result}" + result; + + /* Soft-deprecated function. While the original implementation is available as + isConvertibleWithToString, consider using isStringLike instead, if suitable. */ + isCoercibleToString = lib.warnIf (lib.isInOldestRelease 2305) + "lib.strings.isCoercibleToString is deprecated in favor of either isStringLike or isConvertibleWithToString. Only use the latter if it needs to return true for null, numbers, booleans and list of similarly coercibles." + isConvertibleWithToString; + + /* Check whether a list or other value can be passed to toString. + + Many types of value are coercible to string this way, including int, float, + null, bool, list of similarly coercible values. + */ + isConvertibleWithToString = x: + isStringLike x || + elem (typeOf x) [ "null" "int" "float" "bool" ] || + (isList x && lib.all isConvertibleWithToString x); + + /* Check whether a value can be coerced to a string. + The value must be a string, path, or attribute set. + + String-like values can be used without explicit conversion in + string interpolations and in most functions that expect a string. + */ + isStringLike = x: + isString x || + isPath x || + x ? outPath || + x ? __toString; + + /* Check whether a value is a store path. + + Example: + isStorePath "/nix/store/d945ibfx9x185xf04b890y4f9g3cbb63-python-2.7.11/bin/python" + => false + isStorePath "/nix/store/d945ibfx9x185xf04b890y4f9g3cbb63-python-2.7.11" + => true + isStorePath pkgs.python + => true + isStorePath [] || isStorePath 42 || isStorePath {} || … + => false + */ + isStorePath = x: + if isStringLike x then + let str = toString x; in + substring 0 1 str == "/" + && dirOf str == storeDir + else + false; + + /* Parse a string as an int. Does not support parsing of integers with preceding zero due to + ambiguity between zero-padded and octal numbers. See toIntBase10. + + Type: string -> int + + Example: + + toInt "1337" + => 1337 + + toInt "-4" + => -4 + + toInt " 123 " + => 123 + + toInt "00024" + => error: Ambiguity in interpretation of 00024 between octal and zero padded integer. + + toInt "3.14" + => error: floating point JSON numbers are not supported + */ + toInt = str: + let + # RegEx: Match any leading whitespace, possibly a '-', one or more digits, + # and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*(-?[[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match a leading '0' then one or more digits. + isLeadingZero = match "0[[:digit:]]+" (head strippedInput) == []; + + # Attempt to parse input + parsedInput = fromJSON (head strippedInput); + + generalError = "toInt: Could not convert ${escapeNixString str} to int."; + + octalAmbigError = "toInt: Ambiguity in interpretation of ${escapeNixString str}" + + " between octal and zero padded integer."; + + in + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # Error on presence of leading zero/octal ambiguity. + else if isLeadingZero + then throw octalAmbigError + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; + + + /* Parse a string as a base 10 int. This supports parsing of zero-padded integers. + + Type: string -> int + + Example: + toIntBase10 "1337" + => 1337 + + toIntBase10 "-4" + => -4 + + toIntBase10 " 123 " + => 123 + + toIntBase10 "00024" + => 24 + + toIntBase10 "3.14" + => error: floating point JSON numbers are not supported + */ + toIntBase10 = str: + let + # RegEx: Match any leading whitespace, then match any zero padding, + # capture possibly a '-' followed by one or more digits, + # and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*0*(-?[[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match at least one '0'. + isZero = match "0+" (head strippedInput) == []; + + # Attempt to parse input + parsedInput = fromJSON (head strippedInput); + + generalError = "toIntBase10: Could not convert ${escapeNixString str} to int."; + + in + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # In the special case zero-padded zero (00000), return early. + else if isZero + then 0 + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; + + /* Read a list of paths from `file`, relative to the `rootPath`. + Lines beginning with `#` are treated as comments and ignored. + Whitespace is significant. + + NOTE: This function is not performant and should be avoided. + + Example: + readPathsFromFile /prefix + ./pkgs/development/libraries/qt-5/5.4/qtbase/series + => [ "/prefix/dlopen-resolv.patch" "/prefix/tzdir.patch" + "/prefix/dlopen-libXcursor.patch" "/prefix/dlopen-openssl.patch" + "/prefix/dlopen-dbus.patch" "/prefix/xdg-config-dirs.patch" + "/prefix/nix-profiles-library-paths.patch" + "/prefix/compose-search-path.patch" ] + */ + readPathsFromFile = lib.warn "lib.readPathsFromFile is deprecated, use a list instead" + (rootPath: file: + let + lines = lib.splitString "\n" (readFile file); + removeComments = lib.filter (line: line != "" && !(lib.hasPrefix "#" line)); + relativePaths = removeComments lines; + absolutePaths = map (path: rootPath + "/${path}") relativePaths; + in + absolutePaths); + + /* Read the contents of a file removing the trailing \n + + Type: fileContents :: path -> string + + Example: + $ echo "1.0" > ./version + + fileContents ./version + => "1.0" + */ + fileContents = file: removeSuffix "\n" (readFile file); + + + /* Creates a valid derivation name from a potentially invalid one. + + Type: sanitizeDerivationName :: String -> String + + Example: + sanitizeDerivationName "../hello.bar # foo" + => "-hello.bar-foo" + sanitizeDerivationName "" + => "unknown" + sanitizeDerivationName pkgs.hello + => "-nix-store-2g75chlbpxlrqn15zlby2dfh8hr9qwbk-hello-2.10" + */ + sanitizeDerivationName = + let okRegex = match "[[:alnum:]+_?=-][[:alnum:]+._?=-]*"; + in + string: + # First detect the common case of already valid strings, to speed those up + if stringLength string <= 207 && okRegex string != null + then unsafeDiscardStringContext string + else lib.pipe string [ + # Get rid of string context. This is safe under the assumption that the + # resulting string is only used as a derivation name + unsafeDiscardStringContext + # Strip all leading "." + (x: elemAt (match "\\.*(.*)" x) 0) + # Split out all invalid characters + # https://github.com/NixOS/nix/blob/2.3.2/src/libstore/store-api.cc#L85-L112 + # https://github.com/NixOS/nix/blob/2242be83c61788b9c0736a92bb0b5c7bbfc40803/nix-rust/src/store/path.rs#L100-L125 + (split "[^[:alnum:]+._?=-]+") + # Replace invalid character ranges with a "-" + (concatMapStrings (s: if lib.isList s then "-" else s)) + # Limit to 211 characters (minus 4 chars for ".drv") + (x: substring (lib.max (stringLength x - 207) 0) (-1) x) + # If the result is empty, replace it with "unknown" + (x: if stringLength x == 0 then "unknown" else x) + ]; + + /* Computes the Levenshtein distance between two strings. + Complexity O(n*m) where n and m are the lengths of the strings. + Algorithm adjusted from https://stackoverflow.com/a/9750974/6605742 + + Type: levenshtein :: string -> string -> int + + Example: + levenshtein "foo" "foo" + => 0 + levenshtein "book" "hook" + => 1 + levenshtein "hello" "Heyo" + => 3 + */ + levenshtein = a: b: let + # Two dimensional array with dimensions (stringLength a + 1, stringLength b + 1) + arr = lib.genList (i: + lib.genList (j: + dist i j + ) (stringLength b + 1) + ) (stringLength a + 1); + d = x: y: lib.elemAt (lib.elemAt arr x) y; + dist = i: j: + let c = if substring (i - 1) 1 a == substring (j - 1) 1 b + then 0 else 1; + in + if j == 0 then i + else if i == 0 then j + else lib.min + ( lib.min (d (i - 1) j + 1) (d i (j - 1) + 1)) + ( d (i - 1) (j - 1) + c ); + in d (stringLength a) (stringLength b); + + /* Returns the length of the prefix common to both strings. + */ + commonPrefixLength = a: b: + let + m = lib.min (stringLength a) (stringLength b); + go = i: if i >= m then m else if substring i 1 a == substring i 1 b then go (i + 1) else i; + in go 0; + + /* Returns the length of the suffix common to both strings. + */ + commonSuffixLength = a: b: + let + m = lib.min (stringLength a) (stringLength b); + go = i: if i >= m then m else if substring (stringLength a - i - 1) 1 a == substring (stringLength b - i - 1) 1 b then go (i + 1) else i; + in go 0; + + /* Returns whether the levenshtein distance between two strings is at most some value + Complexity is O(min(n,m)) for k <= 2 and O(n*m) otherwise + + Type: levenshteinAtMost :: int -> string -> string -> bool + + Example: + levenshteinAtMost 0 "foo" "foo" + => true + levenshteinAtMost 1 "foo" "boa" + => false + levenshteinAtMost 2 "foo" "boa" + => true + levenshteinAtMost 2 "This is a sentence" "this is a sentense." + => false + levenshteinAtMost 3 "This is a sentence" "this is a sentense." + => true + + */ + levenshteinAtMost = let + infixDifferAtMost1 = x: y: stringLength x <= 1 && stringLength y <= 1; + + # This function takes two strings stripped by their common pre and suffix, + # and returns whether they differ by at most two by Levenshtein distance. + # Because of this stripping, if they do indeed differ by at most two edits, + # we know that those edits were (if at all) done at the start or the end, + # while the middle has to have stayed the same. This fact is used in the + # implementation. + infixDifferAtMost2 = x: y: + let + xlen = stringLength x; + ylen = stringLength y; + # This function is only called with |x| >= |y| and |x| - |y| <= 2, so + # diff is one of 0, 1 or 2 + diff = xlen - ylen; + + # Infix of x and y, stripped by the left and right most character + xinfix = substring 1 (xlen - 2) x; + yinfix = substring 1 (ylen - 2) y; + + # x and y but a character deleted at the left or right + xdelr = substring 0 (xlen - 1) x; + xdell = substring 1 (xlen - 1) x; + ydelr = substring 0 (ylen - 1) y; + ydell = substring 1 (ylen - 1) y; + in + # A length difference of 2 can only be gotten with 2 delete edits, + # which have to have happened at the start and end of x + # Example: "abcdef" -> "bcde" + if diff == 2 then xinfix == y + # A length difference of 1 can only be gotten with a deletion on the + # right and a replacement on the left or vice versa. + # Example: "abcdef" -> "bcdez" or "zbcde" + else if diff == 1 then xinfix == ydelr || xinfix == ydell + # No length difference can either happen through replacements on both + # sides, or a deletion on the left and an insertion on the right or + # vice versa + # Example: "abcdef" -> "zbcdez" or "bcdefz" or "zabcde" + else xinfix == yinfix || xdelr == ydell || xdell == ydelr; + + in k: if k <= 0 then a: b: a == b else + let f = a: b: + let + alen = stringLength a; + blen = stringLength b; + prelen = commonPrefixLength a b; + suflen = commonSuffixLength a b; + presuflen = prelen + suflen; + ainfix = substring prelen (alen - presuflen) a; + binfix = substring prelen (blen - presuflen) b; + in + # Make a be the bigger string + if alen < blen then f b a + # If a has over k more characters than b, even with k deletes on a, b can't be reached + else if alen - blen > k then false + else if k == 1 then infixDifferAtMost1 ainfix binfix + else if k == 2 then infixDifferAtMost2 ainfix binfix + else levenshtein ainfix binfix <= k; + in f; +} diff --git a/systems/architectures.nix b/systems/architectures.nix new file mode 100644 index 000000000..57b9184ca --- /dev/null +++ b/systems/architectures.nix @@ -0,0 +1,114 @@ +{ lib }: + +rec { + # gcc.arch to its features (as in /proc/cpuinfo) + features = { + default = [ ]; + # x86_64 Intel + westmere = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" ]; + sandybridge = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" ]; + ivybridge = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" ]; + haswell = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "fma" ]; + broadwell = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "fma" ]; + skylake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "fma" ]; + skylake-avx512 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + cannonlake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + icelake-client = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + icelake-server = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + cascadelake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + cooperlake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + tigerlake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + # x86_64 AMD + btver1 = [ "sse3" "ssse3" "sse4_1" "sse4_2" ]; + btver2 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" ]; + bdver1 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "fma" "fma4" ]; + bdver2 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "fma" "fma4" ]; + bdver3 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "fma" "fma4" ]; + bdver4 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" "fma4" ]; + znver1 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" ]; + znver2 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" ]; + znver3 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" ]; + # other + armv5te = [ ]; + armv6 = [ ]; + armv7-a = [ ]; + armv8-a = [ ]; + mips32 = [ ]; + loongson2f = [ ]; + }; + + # a superior CPU has all the features of an inferior and is able to build and test code for it + inferiors = { + # x86_64 Intel + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html + default = [ ]; + westmere = [ ]; + sandybridge = [ "westmere" ] ++ inferiors.westmere; + ivybridge = [ "sandybridge" ] ++ inferiors.sandybridge; + haswell = [ "ivybridge" ] ++ inferiors.ivybridge; + broadwell = [ "haswell" ] ++ inferiors.haswell; + skylake = [ "broadwell" ] ++ inferiors.broadwell; + skylake-avx512 = [ "skylake" ] ++ inferiors.skylake; + cannonlake = [ "skylake-avx512" ] ++ inferiors.skylake-avx512; + icelake-client = [ "cannonlake" ] ++ inferiors.cannonlake; + icelake-server = [ "icelake-client" ] ++ inferiors.icelake-client; + cascadelake = [ "skylake-avx512" ] ++ inferiors.cannonlake; + cooperlake = [ "cascadelake" ] ++ inferiors.cascadelake; + tigerlake = [ "icelake-server" ] ++ inferiors.icelake-server; + + # x86_64 AMD + # TODO: fill this (need testing) + btver1 = [ ]; + btver2 = [ ]; + bdver1 = [ ]; + bdver2 = [ ]; + bdver3 = [ ]; + bdver4 = [ ]; + # Regarding `skylake` as inferior of `znver1`, there are reports of + # successful usage by Gentoo users and Phoronix benchmarking of different + # `-march` targets. + # + # The GCC documentation on extensions used and wikichip documentation + # regarding supperted extensions on znver1 and skylake was used to create + # this partial order. + # + # Note: + # + # - The successors of `skylake` (`cannonlake`, `icelake`, etc) use `avx512` + # which no current AMD Zen michroarch support. + # - `znver1` uses `ABM`, `CLZERO`, `CX16`, `MWAITX`, and `SSE4A` which no + # current Intel microarch support. + # + # https://www.phoronix.com/scan.php?page=article&item=amd-znver3-gcc11&num=1 + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html + # https://en.wikichip.org/wiki/amd/microarchitectures/zen + # https://en.wikichip.org/wiki/intel/microarchitectures/skylake + znver1 = [ "skylake" ] ++ inferiors.skylake; + znver2 = [ "znver1" ] ++ inferiors.znver1; + znver3 = [ "znver2" ] ++ inferiors.znver2; + + # other + armv5te = [ ]; + armv6 = [ ]; + armv7-a = [ ]; + armv8-a = [ ]; + mips32 = [ ]; + loongson2f = [ ]; + }; + + predicates = let + featureSupport = feature: x: builtins.elem feature features.${x} or []; + in { + sse3Support = featureSupport "sse3"; + ssse3Support = featureSupport "ssse3"; + sse4_1Support = featureSupport "sse4_1"; + sse4_2Support = featureSupport "sse4_2"; + sse4_aSupport = featureSupport "sse4a"; + avxSupport = featureSupport "avx"; + avx2Support = featureSupport "avx2"; + avx512Support = featureSupport "avx512"; + aesSupport = featureSupport "aes"; + fmaSupport = featureSupport "fma"; + fma4Support = featureSupport "fma4"; + }; +} diff --git a/systems/default.nix b/systems/default.nix new file mode 100644 index 000000000..f4784c61c --- /dev/null +++ b/systems/default.nix @@ -0,0 +1,228 @@ +{ lib }: + let inherit (lib.attrsets) mapAttrs; in + +rec { + doubles = import ./doubles.nix { inherit lib; }; + parse = import ./parse.nix { inherit lib; }; + inspect = import ./inspect.nix { inherit lib; }; + platforms = import ./platforms.nix { inherit lib; }; + examples = import ./examples.nix { inherit lib; }; + architectures = import ./architectures.nix { inherit lib; }; + + /* List of all Nix system doubles the nixpkgs flake will expose the package set + for. All systems listed here must be supported by nixpkgs as `localSystem`. + + **Warning**: This attribute is considered experimental and is subject to change. + */ + flakeExposed = import ./flake-systems.nix { }; + + # Elaborate a `localSystem` or `crossSystem` so that it contains everything + # necessary. + # + # `parsed` is inferred from args, both because there are two options with one + # clearly preferred, and to prevent cycles. A simpler fixed point where the RHS + # always just used `final.*` would fail on both counts. + elaborate = args': let + args = if lib.isString args' then { system = args'; } + else args'; + final = { + # Prefer to parse `config` as it is strictly more informative. + parsed = parse.mkSystemFromString (if args ? config then args.config else args.system); + # Either of these can be losslessly-extracted from `parsed` iff parsing succeeds. + system = parse.doubleFromSystem final.parsed; + config = parse.tripleFromSystem final.parsed; + # Determine whether we can execute binaries built for the provided platform. + canExecute = platform: + final.isAndroid == platform.isAndroid && + parse.isCompatible final.parsed.cpu platform.parsed.cpu + && final.parsed.kernel == platform.parsed.kernel; + isCompatible = _: throw "2022-05-23: isCompatible has been removed in favor of canExecute, refer to the 22.11 changelog for details"; + # Derived meta-data + libc = + /**/ if final.isDarwin then "libSystem" + else if final.isMinGW then "msvcrt" + else if final.isWasi then "wasilibc" + else if final.isRedox then "relibc" + else if final.isMusl then "musl" + else if final.isUClibc then "uclibc" + else if final.isAndroid then "bionic" + else if final.isLinux /* default */ then "glibc" + else if final.isFreeBSD then "fblibc" + else if final.isNetBSD then "nblibc" + else if final.isAvr then "avrlibc" + else if final.isGhcjs then null + else if final.isNone then "newlib" + # TODO(@Ericson2314) think more about other operating systems + else "native/impure"; + # Choose what linker we wish to use by default. Someday we might also + # choose the C compiler, runtime library, C++ standard library, etc. in + # this way, nice and orthogonally, and deprecate `useLLVM`. But due to + # the monolithic GCC build we cannot actually make those choices + # independently, so we are just doing `linker` and keeping `useLLVM` for + # now. + linker = + /**/ if final.useLLVM or false then "lld" + else if final.isDarwin then "cctools" + # "bfd" and "gold" both come from GNU binutils. The existence of Gold + # is why we use the more obscure "bfd" and not "binutils" for this + # choice. + else "bfd"; + extensions = rec { + sharedLibrary = + /**/ if final.isDarwin then ".dylib" + else if final.isWindows then ".dll" + else ".so"; + staticLibrary = + /**/ if final.isWindows then ".lib" + else ".a"; + library = + /**/ if final.isStatic then staticLibrary + else sharedLibrary; + executable = + /**/ if final.isWindows then ".exe" + else ""; + }; + # Misc boolean options + useAndroidPrebuilt = false; + useiOSPrebuilt = false; + + # Output from uname + uname = { + # uname -s + system = { + linux = "Linux"; + windows = "Windows"; + darwin = "Darwin"; + netbsd = "NetBSD"; + freebsd = "FreeBSD"; + openbsd = "OpenBSD"; + wasi = "Wasi"; + redox = "Redox"; + genode = "Genode"; + }.${final.parsed.kernel.name} or null; + + # uname -m + processor = + if final.isPower64 + then "ppc64${lib.optionalString final.isLittleEndian "le"}" + else if final.isPower + then "ppc${lib.optionalString final.isLittleEndian "le"}" + else if final.isMips64 + then "mips64" # endianness is *not* included on mips64 + else final.parsed.cpu.name; + + # uname -r + release = null; + }; + isStatic = final.isWasm || final.isRedox; + + # Just a guess, based on `system` + inherit + ({ + linux-kernel = args.linux-kernel or {}; + gcc = args.gcc or {}; + rustc = args.rustc or {}; + } // platforms.select final) + linux-kernel gcc rustc; + + linuxArch = + if final.isAarch32 then "arm" + else if final.isAarch64 then "arm64" + else if final.isx86_32 then "i386" + else if final.isx86_64 then "x86_64" + # linux kernel does not distinguish microblaze/microblazeel + else if final.isMicroBlaze then "microblaze" + else if final.isMips32 then "mips" + else if final.isMips64 then "mips" # linux kernel does not distinguish mips32/mips64 + else if final.isPower then "powerpc" + else if final.isRiscV then "riscv" + else if final.isS390 then "s390" + else if final.isLoongArch64 then "loongarch" + else final.parsed.cpu.name; + + qemuArch = + if final.isAarch32 then "arm" + else if final.isS390 && !final.isS390x then null + else if final.isx86_64 then "x86_64" + else if final.isx86 then "i386" + else if final.isMips64 then "mips64${lib.optionalString final.isLittleEndian "el"}" + else final.uname.processor; + + # Name used by UEFI for architectures. + efiArch = + if final.isx86_32 then "ia32" + else if final.isx86_64 then "x64" + else if final.isAarch32 then "arm" + else if final.isAarch64 then "aa64" + else final.parsed.cpu.name; + + darwinArch = { + armv7a = "armv7"; + aarch64 = "arm64"; + }.${final.parsed.cpu.name} or final.parsed.cpu.name; + + darwinPlatform = + if final.isMacOS then "macos" + else if final.isiOS then "ios" + else null; + # The canonical name for this attribute is darwinSdkVersion, but some + # platforms define the old name "sdkVer". + darwinSdkVersion = final.sdkVer or (if final.isAarch64 then "11.0" else "10.12"); + darwinMinVersion = final.darwinSdkVersion; + darwinMinVersionVariable = + if final.isMacOS then "MACOSX_DEPLOYMENT_TARGET" + else if final.isiOS then "IPHONEOS_DEPLOYMENT_TARGET" + else null; + } // ( + let + selectEmulator = pkgs: + let + qemu-user = pkgs.qemu.override { + smartcardSupport = false; + spiceSupport = false; + openGLSupport = false; + virglSupport = false; + vncSupport = false; + gtkSupport = false; + sdlSupport = false; + pulseSupport = false; + smbdSupport = false; + seccompSupport = false; + enableDocs = false; + hostCpuTargets = [ "${final.qemuArch}-linux-user" ]; + }; + wine = (pkgs.winePackagesFor "wine${toString final.parsed.cpu.bits}").minimal; + in + if final.parsed.kernel.name == pkgs.stdenv.hostPlatform.parsed.kernel.name && + pkgs.stdenv.hostPlatform.canExecute final + then "${pkgs.runtimeShell} -c '\"$@\"' --" + else if final.isWindows + then "${wine}/bin/wine${lib.optionalString (final.parsed.cpu.bits == 64) "64"}" + else if final.isLinux && pkgs.stdenv.hostPlatform.isLinux && final.qemuArch != null + then "${qemu-user}/bin/qemu-${final.qemuArch}" + else if final.isWasi + then "${pkgs.wasmtime}/bin/wasmtime" + else if final.isMmix + then "${pkgs.mmixware}/bin/mmix" + else null; + in { + emulatorAvailable = pkgs: (selectEmulator pkgs) != null; + + emulator = pkgs: + if (final.emulatorAvailable pkgs) + then selectEmulator pkgs + else throw "Don't know how to run ${final.config} executables."; + + }) // mapAttrs (n: v: v final.parsed) inspect.predicates + // mapAttrs (n: v: v final.gcc.arch or "default") architectures.predicates + // args; + in assert final.useAndroidPrebuilt -> final.isAndroid; + assert lib.foldl + (pass: { assertion, message }: + if assertion final + then pass + else throw message) + true + (final.parsed.abi.assertions or []); + final; +} diff --git a/systems/doubles.nix b/systems/doubles.nix new file mode 100644 index 000000000..66bf3d872 --- /dev/null +++ b/systems/doubles.nix @@ -0,0 +1,119 @@ +{ lib }: +let + inherit (lib) lists; + inherit (lib.systems) parse; + inherit (lib.systems.inspect) predicates; + inherit (lib.attrsets) matchAttrs; + + all = [ + # Cygwin + "i686-cygwin" "x86_64-cygwin" + + # Darwin + "x86_64-darwin" "i686-darwin" "aarch64-darwin" "armv7a-darwin" + + # FreeBSD + "i686-freebsd13" "x86_64-freebsd13" + + # Genode + "aarch64-genode" "i686-genode" "x86_64-genode" + + # illumos + "x86_64-solaris" + + # JS + "javascript-ghcjs" + + # Linux + "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" + "armv7l-linux" "i686-linux" "loongarch64-linux" "m68k-linux" "microblaze-linux" + "microblazeel-linux" "mips-linux" "mips64-linux" "mips64el-linux" + "mipsel-linux" "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" + "riscv64-linux" "s390-linux" "s390x-linux" "x86_64-linux" + + # MMIXware + "mmix-mmixware" + + # NetBSD + "aarch64-netbsd" "armv6l-netbsd" "armv7a-netbsd" "armv7l-netbsd" + "i686-netbsd" "m68k-netbsd" "mipsel-netbsd" "powerpc-netbsd" + "riscv32-netbsd" "riscv64-netbsd" "x86_64-netbsd" + + # none + "aarch64_be-none" "aarch64-none" "arm-none" "armv6l-none" "avr-none" "i686-none" + "microblaze-none" "microblazeel-none" "msp430-none" "or1k-none" "m68k-none" + "powerpc-none" "powerpcle-none" "riscv32-none" "riscv64-none" "rx-none" + "s390-none" "s390x-none" "vc4-none" "x86_64-none" + + # OpenBSD + "i686-openbsd" "x86_64-openbsd" + + # Redox + "x86_64-redox" + + # WASI + "wasm64-wasi" "wasm32-wasi" + + # Windows + "x86_64-windows" "i686-windows" + ]; + + allParsed = map parse.mkSystemFromString all; + + filterDoubles = f: map parse.doubleFromSystem (lists.filter f allParsed); + +in { + inherit all; + + none = []; + + arm = filterDoubles predicates.isAarch32; + armv7 = filterDoubles predicates.isArmv7; + aarch64 = filterDoubles predicates.isAarch64; + x86 = filterDoubles predicates.isx86; + i686 = filterDoubles predicates.isi686; + x86_64 = filterDoubles predicates.isx86_64; + microblaze = filterDoubles predicates.isMicroBlaze; + mips = filterDoubles predicates.isMips; + mmix = filterDoubles predicates.isMmix; + power = filterDoubles predicates.isPower; + riscv = filterDoubles predicates.isRiscV; + riscv32 = filterDoubles predicates.isRiscV32; + riscv64 = filterDoubles predicates.isRiscV64; + rx = filterDoubles predicates.isRx; + vc4 = filterDoubles predicates.isVc4; + or1k = filterDoubles predicates.isOr1k; + m68k = filterDoubles predicates.isM68k; + s390 = filterDoubles predicates.isS390; + s390x = filterDoubles predicates.isS390x; + loongarch64 = filterDoubles predicates.isLoongArch64; + js = filterDoubles predicates.isJavaScript; + + bigEndian = filterDoubles predicates.isBigEndian; + littleEndian = filterDoubles predicates.isLittleEndian; + + cygwin = filterDoubles predicates.isCygwin; + darwin = filterDoubles predicates.isDarwin; + freebsd = filterDoubles predicates.isFreeBSD; + # Should be better, but MinGW is unclear. + gnu = filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnu; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnueabi; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnueabihf; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabin32; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabi64; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabielfv1; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabielfv2; }); + illumos = filterDoubles predicates.isSunOS; + linux = filterDoubles predicates.isLinux; + netbsd = filterDoubles predicates.isNetBSD; + openbsd = filterDoubles predicates.isOpenBSD; + unix = filterDoubles predicates.isUnix; + wasi = filterDoubles predicates.isWasi; + redox = filterDoubles predicates.isRedox; + windows = filterDoubles predicates.isWindows; + genode = filterDoubles predicates.isGenode; + + embedded = filterDoubles predicates.isNone; + + mesaPlatforms = ["i686-linux" "x86_64-linux" "x86_64-darwin" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "armv7a-linux" "aarch64-linux" "powerpc64-linux" "powerpc64le-linux" "aarch64-darwin" "riscv64-linux"]; +} diff --git a/systems/examples.nix b/systems/examples.nix new file mode 100644 index 000000000..45b95c150 --- /dev/null +++ b/systems/examples.nix @@ -0,0 +1,335 @@ +# These can be passed to nixpkgs as either the `localSystem` or +# `crossSystem`. They are put here for user convenience, but also used by cross +# tests and linux cross stdenv building, so handle with care! +{ lib }: +let + platforms = import ./platforms.nix { inherit lib; }; + + riscv = bits: { + config = "riscv${bits}-unknown-linux-gnu"; + }; +in + +rec { + # + # Linux + # + powernv = { + config = "powerpc64le-unknown-linux-gnu"; + }; + musl-power = { + config = "powerpc64le-unknown-linux-musl"; + }; + + ppc64 = { + config = "powerpc64-unknown-linux-gnuabielfv2"; + }; + ppc64-musl = { + config = "powerpc64-unknown-linux-musl"; + gcc = { abi = "elfv2"; }; + }; + + sheevaplug = { + config = "armv5tel-unknown-linux-gnueabi"; + } // platforms.sheevaplug; + + raspberryPi = { + config = "armv6l-unknown-linux-gnueabihf"; + } // platforms.raspberrypi; + + remarkable1 = { + config = "armv7l-unknown-linux-gnueabihf"; + } // platforms.zero-gravitas; + + remarkable2 = { + config = "armv7l-unknown-linux-gnueabihf"; + } // platforms.zero-sugar; + + armv7l-hf-multiplatform = { + config = "armv7l-unknown-linux-gnueabihf"; + }; + + aarch64-multiplatform = { + config = "aarch64-unknown-linux-gnu"; + }; + + armv7a-android-prebuilt = { + config = "armv7a-unknown-linux-androideabi"; + rustc.config = "armv7-linux-androideabi"; + sdkVer = "28"; + ndkVer = "24"; + useAndroidPrebuilt = true; + } // platforms.armv7a-android; + + aarch64-android-prebuilt = { + config = "aarch64-unknown-linux-android"; + rustc.config = "aarch64-linux-android"; + sdkVer = "28"; + ndkVer = "24"; + useAndroidPrebuilt = true; + }; + + aarch64-android = { + config = "aarch64-unknown-linux-android"; + sdkVer = "30"; + ndkVer = "24"; + libc = "bionic"; + useAndroidPrebuilt = false; + useLLVM = true; + }; + + pogoplug4 = { + config = "armv5tel-unknown-linux-gnueabi"; + } // platforms.pogoplug4; + + ben-nanonote = { + config = "mipsel-unknown-linux-uclibc"; + } // platforms.ben_nanonote; + + fuloongminipc = { + config = "mipsel-unknown-linux-gnu"; + } // platforms.fuloong2f_n32; + + # can execute on 32bit chip + mips-linux-gnu = { config = "mips-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; + mipsel-linux-gnu = { config = "mipsel-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; + + # require 64bit chip (for more registers, 64-bit floating point, 64-bit "long long") but use 32bit pointers + mips64-linux-gnuabin32 = { config = "mips64-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; + mips64el-linux-gnuabin32 = { config = "mips64el-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; + + # 64bit pointers + mips64-linux-gnuabi64 = { config = "mips64-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; + mips64el-linux-gnuabi64 = { config = "mips64el-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; + + muslpi = raspberryPi // { + config = "armv6l-unknown-linux-musleabihf"; + }; + + aarch64-multiplatform-musl = { + config = "aarch64-unknown-linux-musl"; + }; + + gnu64 = { config = "x86_64-unknown-linux-gnu"; }; + gnu32 = { config = "i686-unknown-linux-gnu"; }; + + musl64 = { config = "x86_64-unknown-linux-musl"; }; + musl32 = { config = "i686-unknown-linux-musl"; }; + + riscv64 = riscv "64"; + riscv32 = riscv "32"; + + riscv64-embedded = { + config = "riscv64-none-elf"; + libc = "newlib"; + }; + + riscv32-embedded = { + config = "riscv32-none-elf"; + libc = "newlib"; + }; + + loongarch64-linux = { + config = "loongarch64-unknown-linux-gnu"; + }; + + mmix = { + config = "mmix-unknown-mmixware"; + libc = "newlib"; + }; + + rx-embedded = { + config = "rx-none-elf"; + libc = "newlib"; + }; + + msp430 = { + config = "msp430-elf"; + libc = "newlib"; + }; + + avr = { + config = "avr"; + }; + + vc4 = { + config = "vc4-elf"; + libc = "newlib"; + }; + + or1k = { + config = "or1k-elf"; + libc = "newlib"; + }; + + m68k = { + config = "m68k-unknown-linux-gnu"; + }; + + s390 = { + config = "s390-unknown-linux-gnu"; + }; + + s390x = { + config = "s390x-unknown-linux-gnu"; + }; + + arm-embedded = { + config = "arm-none-eabi"; + libc = "newlib"; + }; + armhf-embedded = { + config = "arm-none-eabihf"; + libc = "newlib"; + # GCC8+ does not build without this + # (https://www.mail-archive.com/gcc-bugs@gcc.gnu.org/msg552339.html): + gcc = { + arch = "armv5t"; + fpu = "vfp"; + }; + }; + + aarch64-embedded = { + config = "aarch64-none-elf"; + libc = "newlib"; + }; + + aarch64be-embedded = { + config = "aarch64_be-none-elf"; + libc = "newlib"; + }; + + ppc-embedded = { + config = "powerpc-none-eabi"; + libc = "newlib"; + }; + + ppcle-embedded = { + config = "powerpcle-none-eabi"; + libc = "newlib"; + }; + + i686-embedded = { + config = "i686-elf"; + libc = "newlib"; + }; + + x86_64-embedded = { + config = "x86_64-elf"; + libc = "newlib"; + }; + + # + # Redox + # + + x86_64-unknown-redox = { + config = "x86_64-unknown-redox"; + libc = "relibc"; + }; + + # + # Darwin + # + + iphone64 = { + config = "aarch64-apple-ios"; + # config = "aarch64-apple-darwin14"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneOS"; + useiOSPrebuilt = true; + }; + + iphone32 = { + config = "armv7a-apple-ios"; + # config = "arm-apple-darwin10"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneOS"; + useiOSPrebuilt = true; + }; + + iphone64-simulator = { + config = "x86_64-apple-ios"; + # config = "x86_64-apple-darwin14"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneSimulator"; + darwinPlatform = "ios-simulator"; + useiOSPrebuilt = true; + }; + + iphone32-simulator = { + config = "i686-apple-ios"; + # config = "i386-apple-darwin11"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneSimulator"; + darwinPlatform = "ios-simulator"; + useiOSPrebuilt = true; + }; + + aarch64-darwin = { + config = "aarch64-apple-darwin"; + xcodePlatform = "MacOSX"; + platform = {}; + }; + + x86_64-darwin = { + config = "x86_64-apple-darwin"; + xcodePlatform = "MacOSX"; + platform = {}; + }; + + # + # Windows + # + + # 32 bit mingw-w64 + mingw32 = { + config = "i686-w64-mingw32"; + libc = "msvcrt"; # This distinguishes the mingw (non posix) toolchain + }; + + # 64 bit mingw-w64 + mingwW64 = { + # That's the triplet they use in the mingw-w64 docs. + config = "x86_64-w64-mingw32"; + libc = "msvcrt"; # This distinguishes the mingw (non posix) toolchain + }; + + # BSDs + + x86_64-freebsd = { + config = "x86_64-unknown-freebsd13"; + useLLVM = true; + }; + + x86_64-netbsd = { + config = "x86_64-unknown-netbsd"; + }; + + # this is broken and never worked fully + x86_64-netbsd-llvm = { + config = "x86_64-unknown-netbsd"; + useLLVM = true; + }; + + # + # WASM + # + + wasi32 = { + config = "wasm32-unknown-wasi"; + useLLVM = true; + }; + + # Ghcjs + ghcjs = { + # This triple is special to GHC/Cabal/GHCJS and not recognized by autotools + # See: https://gitlab.haskell.org/ghc/ghc/-/commit/6636b670233522f01d002c9b97827d00289dbf5c + # https://github.com/ghcjs/ghcjs/issues/53 + config = "javascript-unknown-ghcjs"; + }; +} diff --git a/systems/flake-systems.nix b/systems/flake-systems.nix new file mode 100644 index 000000000..b1988c6a4 --- /dev/null +++ b/systems/flake-systems.nix @@ -0,0 +1,29 @@ +# See [RFC 46] for mandated platform support and ../../pkgs/stdenv for +# implemented platform support. This list is mainly descriptive, i.e. all +# system doubles for platforms where nixpkgs can do native compilation +# reasonably well are included. +# +# [RFC 46]: https://github.com/NixOS/rfcs/blob/master/rfcs/0046-platform-support-tiers.md +{ }: + +[ + # Tier 1 + "x86_64-linux" + # Tier 2 + "aarch64-linux" + "x86_64-darwin" + # Tier 3 + "armv6l-linux" + "armv7l-linux" + "i686-linux" + "mipsel-linux" + + # Other platforms with sufficient support in stdenv which is not formally + # mandated by their platform tier. + "aarch64-darwin" + "armv5tel-linux" + "powerpc64le-linux" + "riscv64-linux" + + # "x86_64-freebsd" is excluded because it is mostly broken +] diff --git a/systems/inspect.nix b/systems/inspect.nix new file mode 100644 index 000000000..89e9f4231 --- /dev/null +++ b/systems/inspect.nix @@ -0,0 +1,117 @@ +{ lib }: +with import ./parse.nix { inherit lib; }; +with lib.attrsets; +with lib.lists; + +let abis_ = abis; in +let abis = lib.mapAttrs (_: abi: builtins.removeAttrs abi [ "assertions" ]) abis_; in + +rec { + # these patterns are to be matched against {host,build,target}Platform.parsed + patterns = rec { + # The patterns below are lists in sum-of-products form. + # + # Each attribute is list of product conditions; non-list values are treated + # as a singleton list. If *any* product condition in the list matches then + # the predicate matches. Each product condition is tested by + # `lib.attrsets.matchAttrs`, which requires a match on *all* attributes of + # the product. + + isi686 = { cpu = cpuTypes.i686; }; + isx86_32 = { cpu = { family = "x86"; bits = 32; }; }; + isx86_64 = { cpu = { family = "x86"; bits = 64; }; }; + isPower = { cpu = { family = "power"; }; }; + isPower64 = { cpu = { family = "power"; bits = 64; }; }; + # This ABI is the default in NixOS PowerPC64 BE, but not on mainline GCC, + # so it sometimes causes issues in certain packages that makes the wrong + # assumption on the used ABI. + isAbiElfv2 = [ + { abi = { abi = "elfv2"; }; } + { abi = { name = "musl"; }; cpu = { family = "power"; bits = 64; }; } + ]; + isx86 = { cpu = { family = "x86"; }; }; + isAarch32 = { cpu = { family = "arm"; bits = 32; }; }; + isArmv7 = map ({ arch, ... }: { cpu = { inherit arch; }; }) + (lib.filter (cpu: lib.hasPrefix "armv7" cpu.arch or "") + (lib.attrValues cpuTypes)); + isAarch64 = { cpu = { family = "arm"; bits = 64; }; }; + isAarch = { cpu = { family = "arm"; }; }; + isMicroBlaze = { cpu = { family = "microblaze"; }; }; + isMips = { cpu = { family = "mips"; }; }; + isMips32 = { cpu = { family = "mips"; bits = 32; }; }; + isMips64 = { cpu = { family = "mips"; bits = 64; }; }; + isMips64n32 = { cpu = { family = "mips"; bits = 64; }; abi = { abi = "n32"; }; }; + isMips64n64 = { cpu = { family = "mips"; bits = 64; }; abi = { abi = "64"; }; }; + isMmix = { cpu = { family = "mmix"; }; }; + isRiscV = { cpu = { family = "riscv"; }; }; + isRiscV32 = { cpu = { family = "riscv"; bits = 32; }; }; + isRiscV64 = { cpu = { family = "riscv"; bits = 64; }; }; + isRx = { cpu = { family = "rx"; }; }; + isSparc = { cpu = { family = "sparc"; }; }; + isWasm = { cpu = { family = "wasm"; }; }; + isMsp430 = { cpu = { family = "msp430"; }; }; + isVc4 = { cpu = { family = "vc4"; }; }; + isAvr = { cpu = { family = "avr"; }; }; + isAlpha = { cpu = { family = "alpha"; }; }; + isOr1k = { cpu = { family = "or1k"; }; }; + isM68k = { cpu = { family = "m68k"; }; }; + isS390 = { cpu = { family = "s390"; }; }; + isS390x = { cpu = { family = "s390"; bits = 64; }; }; + isLoongArch64 = { cpu = { family = "loongarch"; bits = 64; }; }; + isJavaScript = { cpu = cpuTypes.javascript; }; + + is32bit = { cpu = { bits = 32; }; }; + is64bit = { cpu = { bits = 64; }; }; + isILP32 = map (a: { abi = { abi = a; }; }) [ "n32" "ilp32" "x32" ]; + isBigEndian = { cpu = { significantByte = significantBytes.bigEndian; }; }; + isLittleEndian = { cpu = { significantByte = significantBytes.littleEndian; }; }; + + isBSD = { kernel = { families = { inherit (kernelFamilies) bsd; }; }; }; + isDarwin = { kernel = { families = { inherit (kernelFamilies) darwin; }; }; }; + isUnix = [ isBSD isDarwin isLinux isSunOS isCygwin isRedox ]; + + isMacOS = { kernel = kernels.macos; }; + isiOS = { kernel = kernels.ios; }; + isLinux = { kernel = kernels.linux; }; + isSunOS = { kernel = kernels.solaris; }; + isFreeBSD = { kernel = { name = "freebsd"; }; }; + isNetBSD = { kernel = kernels.netbsd; }; + isOpenBSD = { kernel = kernels.openbsd; }; + isWindows = { kernel = kernels.windows; }; + isCygwin = { kernel = kernels.windows; abi = abis.cygnus; }; + isMinGW = { kernel = kernels.windows; abi = abis.gnu; }; + isWasi = { kernel = kernels.wasi; }; + isRedox = { kernel = kernels.redox; }; + isGhcjs = { kernel = kernels.ghcjs; }; + isGenode = { kernel = kernels.genode; }; + isNone = { kernel = kernels.none; }; + + isAndroid = [ { abi = abis.android; } { abi = abis.androideabi; } ]; + isGnu = with abis; map (a: { abi = a; }) [ gnuabi64 gnu gnueabi gnueabihf gnuabielfv1 gnuabielfv2 ]; + isMusl = with abis; map (a: { abi = a; }) [ musl musleabi musleabihf muslabin32 muslabi64 ]; + isUClibc = with abis; map (a: { abi = a; }) [ uclibc uclibceabi uclibceabihf ]; + + isEfi = [ + { cpu = { family = "arm"; version = "6"; }; } + { cpu = { family = "arm"; version = "7"; }; } + { cpu = { family = "arm"; version = "8"; }; } + { cpu = { family = "riscv"; }; } + { cpu = { family = "x86"; }; } + ]; + }; + + matchAnyAttrs = patterns: + if builtins.isList patterns then attrs: any (pattern: matchAttrs pattern attrs) patterns + else matchAttrs patterns; + + predicates = mapAttrs (_: matchAnyAttrs) patterns; + + # these patterns are to be matched against the entire + # {host,build,target}Platform structure; they include a `parsed={}` marker so + # that `lib.meta.availableOn` can distinguish them from the patterns which + # apply only to the `parsed` field. + + platformPatterns = mapAttrs (_: p: { parsed = {}; } // p) { + isStatic = { isStatic = true; }; + }; +} diff --git a/systems/parse.nix b/systems/parse.nix new file mode 100644 index 000000000..6eb4f27cc --- /dev/null +++ b/systems/parse.nix @@ -0,0 +1,503 @@ +# Define the list of system with their properties. +# +# See https://clang.llvm.org/docs/CrossCompilation.html and +# http://llvm.org/docs/doxygen/html/Triple_8cpp_source.html especially +# Triple::normalize. Parsing should essentially act as a more conservative +# version of that last function. +# +# Most of the types below come in "open" and "closed" pairs. The open ones +# specify what information we need to know about systems in general, and the +# closed ones are sub-types representing the whitelist of systems we support in +# practice. +# +# Code in the remainder of nixpkgs shouldn't rely on the closed ones in +# e.g. exhaustive cases. Its more a sanity check to make sure nobody defines +# systems that overlap with existing ones and won't notice something amiss. +# +{ lib }: +with lib.lists; +with lib.types; +with lib.attrsets; +with lib.strings; +with (import ./inspect.nix { inherit lib; }).predicates; + +let + inherit (lib.options) mergeOneOption; + + setTypes = type: + mapAttrs (name: value: + assert type.check value; + setType type.name ({ inherit name; } // value)); + +in + +rec { + + ################################################################################ + + types.openSignificantByte = mkOptionType { + name = "significant-byte"; + description = "Endianness"; + merge = mergeOneOption; + }; + + types.significantByte = enum (attrValues significantBytes); + + significantBytes = setTypes types.openSignificantByte { + bigEndian = {}; + littleEndian = {}; + }; + + ################################################################################ + + # Reasonable power of 2 + types.bitWidth = enum [ 8 16 32 64 128 ]; + + ################################################################################ + + types.openCpuType = mkOptionType { + name = "cpu-type"; + description = "instruction set architecture name and information"; + merge = mergeOneOption; + check = x: types.bitWidth.check x.bits + && (if 8 < x.bits + then types.significantByte.check x.significantByte + else !(x ? significantByte)); + }; + + types.cpuType = enum (attrValues cpuTypes); + + cpuTypes = with significantBytes; setTypes types.openCpuType { + arm = { bits = 32; significantByte = littleEndian; family = "arm"; }; + armv5tel = { bits = 32; significantByte = littleEndian; family = "arm"; version = "5"; arch = "armv5t"; }; + armv6m = { bits = 32; significantByte = littleEndian; family = "arm"; version = "6"; arch = "armv6-m"; }; + armv6l = { bits = 32; significantByte = littleEndian; family = "arm"; version = "6"; arch = "armv6"; }; + armv7a = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7-a"; }; + armv7r = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7-r"; }; + armv7m = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7-m"; }; + armv7l = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7"; }; + armv8a = { bits = 32; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + armv8r = { bits = 32; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + armv8m = { bits = 32; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-m"; }; + aarch64 = { bits = 64; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + aarch64_be = { bits = 64; significantByte = bigEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + + i386 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i386"; }; + i486 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i486"; }; + i586 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i586"; }; + i686 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i686"; }; + x86_64 = { bits = 64; significantByte = littleEndian; family = "x86"; arch = "x86-64"; }; + + microblaze = { bits = 32; significantByte = bigEndian; family = "microblaze"; }; + microblazeel = { bits = 32; significantByte = littleEndian; family = "microblaze"; }; + + mips = { bits = 32; significantByte = bigEndian; family = "mips"; }; + mipsel = { bits = 32; significantByte = littleEndian; family = "mips"; }; + mips64 = { bits = 64; significantByte = bigEndian; family = "mips"; }; + mips64el = { bits = 64; significantByte = littleEndian; family = "mips"; }; + + mmix = { bits = 64; significantByte = bigEndian; family = "mmix"; }; + + m68k = { bits = 32; significantByte = bigEndian; family = "m68k"; }; + + powerpc = { bits = 32; significantByte = bigEndian; family = "power"; }; + powerpc64 = { bits = 64; significantByte = bigEndian; family = "power"; }; + powerpc64le = { bits = 64; significantByte = littleEndian; family = "power"; }; + powerpcle = { bits = 32; significantByte = littleEndian; family = "power"; }; + + riscv32 = { bits = 32; significantByte = littleEndian; family = "riscv"; }; + riscv64 = { bits = 64; significantByte = littleEndian; family = "riscv"; }; + + s390 = { bits = 32; significantByte = bigEndian; family = "s390"; }; + s390x = { bits = 64; significantByte = bigEndian; family = "s390"; }; + + sparc = { bits = 32; significantByte = bigEndian; family = "sparc"; }; + sparc64 = { bits = 64; significantByte = bigEndian; family = "sparc"; }; + + wasm32 = { bits = 32; significantByte = littleEndian; family = "wasm"; }; + wasm64 = { bits = 64; significantByte = littleEndian; family = "wasm"; }; + + alpha = { bits = 64; significantByte = littleEndian; family = "alpha"; }; + + rx = { bits = 32; significantByte = littleEndian; family = "rx"; }; + msp430 = { bits = 16; significantByte = littleEndian; family = "msp430"; }; + avr = { bits = 8; family = "avr"; }; + + vc4 = { bits = 32; significantByte = littleEndian; family = "vc4"; }; + + or1k = { bits = 32; significantByte = bigEndian; family = "or1k"; }; + + loongarch64 = { bits = 64; significantByte = littleEndian; family = "loongarch"; }; + + javascript = { bits = 32; significantByte = littleEndian; family = "javascript"; }; + }; + + # GNU build systems assume that older NetBSD architectures are using a.out. + gnuNetBSDDefaultExecFormat = cpu: + if (cpu.family == "arm" && cpu.bits == 32) || + (cpu.family == "sparc" && cpu.bits == 32) || + (cpu.family == "m68k" && cpu.bits == 32) || + (cpu.family == "x86" && cpu.bits == 32) + then execFormats.aout + else execFormats.elf; + + # Determine when two CPUs are compatible with each other. That is, + # can code built for system B run on system A? For that to happen, + # the programs that system B accepts must be a subset of the + # programs that system A accepts. + # + # We have the following properties of the compatibility relation, + # which must be preserved when adding compatibility information for + # additional CPUs. + # - (reflexivity) + # Every CPU is compatible with itself. + # - (transitivity) + # If A is compatible with B and B is compatible with C then A is compatible with C. + # + # Note: Since 22.11 the archs of a mode switching CPU are no longer considered + # pairwise compatible. Mode switching implies that binaries built for A + # and B respectively can't be executed at the same time. + isCompatible = a: b: with cpuTypes; lib.any lib.id [ + # x86 + (b == i386 && isCompatible a i486) + (b == i486 && isCompatible a i586) + (b == i586 && isCompatible a i686) + + # XXX: Not true in some cases. Like in WSL mode. + (b == i686 && isCompatible a x86_64) + + # ARMv4 + (b == arm && isCompatible a armv5tel) + + # ARMv5 + (b == armv5tel && isCompatible a armv6l) + + # ARMv6 + (b == armv6l && isCompatible a armv6m) + (b == armv6m && isCompatible a armv7l) + + # ARMv7 + (b == armv7l && isCompatible a armv7a) + (b == armv7l && isCompatible a armv7r) + (b == armv7l && isCompatible a armv7m) + + # ARMv8 + (b == aarch64 && a == armv8a) + (b == armv8a && isCompatible a aarch64) + (b == armv8r && isCompatible a armv8a) + (b == armv8m && isCompatible a armv8a) + + # PowerPC + (b == powerpc && isCompatible a powerpc64) + (b == powerpcle && isCompatible a powerpc64le) + + # MIPS + (b == mips && isCompatible a mips64) + (b == mipsel && isCompatible a mips64el) + + # RISCV + (b == riscv32 && isCompatible a riscv64) + + # SPARC + (b == sparc && isCompatible a sparc64) + + # WASM + (b == wasm32 && isCompatible a wasm64) + + # identity + (b == a) + ]; + + ################################################################################ + + types.openVendor = mkOptionType { + name = "vendor"; + description = "vendor for the platform"; + merge = mergeOneOption; + }; + + types.vendor = enum (attrValues vendors); + + vendors = setTypes types.openVendor { + apple = {}; + pc = {}; + # Actually matters, unlocking some MinGW-w64-specific options in GCC. See + # bottom of https://sourceforge.net/p/mingw-w64/wiki2/Unicode%20apps/ + w64 = {}; + + none = {}; + unknown = {}; + }; + + ################################################################################ + + types.openExecFormat = mkOptionType { + name = "exec-format"; + description = "executable container used by the kernel"; + merge = mergeOneOption; + }; + + types.execFormat = enum (attrValues execFormats); + + execFormats = setTypes types.openExecFormat { + aout = {}; # a.out + elf = {}; + macho = {}; + pe = {}; + wasm = {}; + + unknown = {}; + }; + + ################################################################################ + + types.openKernelFamily = mkOptionType { + name = "exec-format"; + description = "executable container used by the kernel"; + merge = mergeOneOption; + }; + + types.kernelFamily = enum (attrValues kernelFamilies); + + kernelFamilies = setTypes types.openKernelFamily { + bsd = {}; + darwin = {}; + }; + + ################################################################################ + + types.openKernel = mkOptionType { + name = "kernel"; + description = "kernel name and information"; + merge = mergeOneOption; + check = x: types.execFormat.check x.execFormat + && all types.kernelFamily.check (attrValues x.families); + }; + + types.kernel = enum (attrValues kernels); + + kernels = with execFormats; with kernelFamilies; setTypes types.openKernel { + # TODO(@Ericson2314): Don't want to mass-rebuild yet to keeping 'darwin' as + # the normalized name for macOS. + macos = { execFormat = macho; families = { inherit darwin; }; name = "darwin"; }; + ios = { execFormat = macho; families = { inherit darwin; }; }; + # A tricky thing about FreeBSD is that there is no stable ABI across + # versions. That means that putting in the version as part of the + # config string is paramount. + freebsd12 = { execFormat = elf; families = { inherit bsd; }; name = "freebsd"; version = 12; }; + freebsd13 = { execFormat = elf; families = { inherit bsd; }; name = "freebsd"; version = 13; }; + linux = { execFormat = elf; families = { }; }; + netbsd = { execFormat = elf; families = { inherit bsd; }; }; + none = { execFormat = unknown; families = { }; }; + openbsd = { execFormat = elf; families = { inherit bsd; }; }; + solaris = { execFormat = elf; families = { }; }; + wasi = { execFormat = wasm; families = { }; }; + redox = { execFormat = elf; families = { }; }; + windows = { execFormat = pe; families = { }; }; + ghcjs = { execFormat = unknown; families = { }; }; + genode = { execFormat = elf; families = { }; }; + mmixware = { execFormat = unknown; families = { }; }; + } // { # aliases + # 'darwin' is the kernel for all of them. We choose macOS by default. + darwin = kernels.macos; + watchos = kernels.ios; + tvos = kernels.ios; + win32 = kernels.windows; + }; + + ################################################################################ + + types.openAbi = mkOptionType { + name = "abi"; + description = "binary interface for compiled code and syscalls"; + merge = mergeOneOption; + }; + + types.abi = enum (attrValues abis); + + abis = setTypes types.openAbi { + cygnus = {}; + msvc = {}; + + # Note: eabi is specific to ARM and PowerPC. + # On PowerPC, this corresponds to PPCEABI. + # On ARM, this corresponds to ARMEABI. + eabi = { float = "soft"; }; + eabihf = { float = "hard"; }; + + # Other architectures should use ELF in embedded situations. + elf = {}; + + androideabi = {}; + android = { + assertions = [ + { assertion = platform: !platform.isAarch32; + message = '' + The "android" ABI is not for 32-bit ARM. Use "androideabi" instead. + ''; + } + ]; + }; + + gnueabi = { float = "soft"; }; + gnueabihf = { float = "hard"; }; + gnu = { + assertions = [ + { assertion = platform: !platform.isAarch32; + message = '' + The "gnu" ABI is ambiguous on 32-bit ARM. Use "gnueabi" or "gnueabihf" instead. + ''; + } + { assertion = platform: with platform; !(isPower64 && isBigEndian); + message = '' + The "gnu" ABI is ambiguous on big-endian 64-bit PowerPC. Use "gnuabielfv2" or "gnuabielfv1" instead. + ''; + } + ]; + }; + gnuabi64 = { abi = "64"; }; + muslabi64 = { abi = "64"; }; + + # NOTE: abi=n32 requires a 64-bit MIPS chip! That is not a typo. + # It is basically the 64-bit abi with 32-bit pointers. Details: + # https://www.linux-mips.org/pub/linux/mips/doc/ABI/MIPS-N32-ABI-Handbook.pdf + gnuabin32 = { abi = "n32"; }; + muslabin32 = { abi = "n32"; }; + + gnuabielfv2 = { abi = "elfv2"; }; + gnuabielfv1 = { abi = "elfv1"; }; + + musleabi = { float = "soft"; }; + musleabihf = { float = "hard"; }; + musl = {}; + + uclibceabi = { float = "soft"; }; + uclibceabihf = { float = "hard"; }; + uclibc = {}; + + unknown = {}; + }; + + ################################################################################ + + types.parsedPlatform = mkOptionType { + name = "system"; + description = "fully parsed representation of llvm- or nix-style platform tuple"; + merge = mergeOneOption; + check = { cpu, vendor, kernel, abi }: + types.cpuType.check cpu + && types.vendor.check vendor + && types.kernel.check kernel + && types.abi.check abi; + }; + + isSystem = isType "system"; + + mkSystem = components: + assert types.parsedPlatform.check components; + setType "system" components; + + mkSkeletonFromList = l: { + "1" = if elemAt l 0 == "avr" + then { cpu = elemAt l 0; kernel = "none"; abi = "unknown"; } + else throw "Target specification with 1 components is ambiguous"; + "2" = # We only do 2-part hacks for things Nix already supports + if elemAt l 1 == "cygwin" + then { cpu = elemAt l 0; kernel = "windows"; abi = "cygnus"; } + # MSVC ought to be the default ABI so this case isn't needed. But then it + # becomes difficult to handle the gnu* variants for Aarch32 correctly for + # minGW. So it's easier to make gnu* the default for the MinGW, but + # hack-in MSVC for the non-MinGW case right here. + else if elemAt l 1 == "windows" + then { cpu = elemAt l 0; kernel = "windows"; abi = "msvc"; } + else if (elemAt l 1) == "elf" + then { cpu = elemAt l 0; vendor = "unknown"; kernel = "none"; abi = elemAt l 1; } + else { cpu = elemAt l 0; kernel = elemAt l 1; }; + "3" = + # cpu-kernel-environment + if elemAt l 1 == "linux" || + elem (elemAt l 2) ["eabi" "eabihf" "elf" "gnu"] + then { + cpu = elemAt l 0; + kernel = elemAt l 1; + abi = elemAt l 2; + vendor = "unknown"; + } + # cpu-vendor-os + else if elemAt l 1 == "apple" || + elem (elemAt l 2) [ "wasi" "redox" "mmixware" "ghcjs" "mingw32" ] || + hasPrefix "freebsd" (elemAt l 2) || + hasPrefix "netbsd" (elemAt l 2) || + hasPrefix "genode" (elemAt l 2) + then { + cpu = elemAt l 0; + vendor = elemAt l 1; + kernel = if elemAt l 2 == "mingw32" + then "windows" # autotools breaks on -gnu for window + else elemAt l 2; + } + else throw "Target specification with 3 components is ambiguous"; + "4" = { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; abi = elemAt l 3; }; + }.${toString (length l)} + or (throw "system string has invalid number of hyphen-separated components"); + + # This should revert the job done by config.guess from the gcc compiler. + mkSystemFromSkeleton = { cpu + , # Optional, but fallback too complex for here. + # Inferred below instead. + vendor ? assert false; null + , kernel + , # Also inferred below + abi ? assert false; null + } @ args: let + getCpu = name: cpuTypes.${name} or (throw "Unknown CPU type: ${name}"); + getVendor = name: vendors.${name} or (throw "Unknown vendor: ${name}"); + getKernel = name: kernels.${name} or (throw "Unknown kernel: ${name}"); + getAbi = name: abis.${name} or (throw "Unknown ABI: ${name}"); + + parsed = { + cpu = getCpu args.cpu; + vendor = + /**/ if args ? vendor then getVendor args.vendor + else if isDarwin parsed then vendors.apple + else if isWindows parsed then vendors.pc + else vendors.unknown; + kernel = if hasPrefix "darwin" args.kernel then getKernel "darwin" + else if hasPrefix "netbsd" args.kernel then getKernel "netbsd" + else getKernel args.kernel; + abi = + /**/ if args ? abi then getAbi args.abi + else if isLinux parsed || isWindows parsed then + if isAarch32 parsed then + if lib.versionAtLeast (parsed.cpu.version or "0") "6" + then abis.gnueabihf + else abis.gnueabi + # Default ppc64 BE to ELFv2 + else if isPower64 parsed && isBigEndian parsed then abis.gnuabielfv2 + else abis.gnu + else abis.unknown; + }; + + in mkSystem parsed; + + mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (lib.splitString "-" s)); + + kernelName = kernel: + kernel.name + toString (kernel.version or ""); + + doubleFromSystem = { cpu, kernel, abi, ... }: + /**/ if abi == abis.cygnus then "${cpu.name}-cygwin" + else if kernel.families ? darwin then "${cpu.name}-darwin" + else "${cpu.name}-${kernelName kernel}"; + + tripleFromSystem = { cpu, vendor, kernel, abi, ... } @ sys: assert isSystem sys; let + optExecFormat = + lib.optionalString (kernel.name == "netbsd" && + gnuNetBSDDefaultExecFormat cpu != kernel.execFormat) + kernel.execFormat.name; + optAbi = lib.optionalString (abi != abis.unknown) "-${abi.name}"; + in "${cpu.name}-${vendor.name}-${kernelName kernel}${optExecFormat}${optAbi}"; + + ################################################################################ + +} diff --git a/systems/platforms.nix b/systems/platforms.nix new file mode 100644 index 000000000..d574943e4 --- /dev/null +++ b/systems/platforms.nix @@ -0,0 +1,565 @@ +# Note: lib/systems/default.nix takes care of producing valid, +# fully-formed "platform" values (e.g. hostPlatform, buildPlatform, +# targetPlatform, etc) containing at least the minimal set of attrs +# required (see types.parsedPlatform in lib/systems/parse.nix). This +# file takes an already-valid platform and further elaborates it with +# optional fields; currently these are: linux-kernel, gcc, and rustc. + +{ lib }: +rec { + pc = { + linux-kernel = { + name = "pc"; + + baseConfig = "defconfig"; + # Build whatever possible as a module, if not stated in the extra config. + autoModules = true; + target = "bzImage"; + }; + }; + + pc_simplekernel = lib.recursiveUpdate pc { + linux-kernel.autoModules = false; + }; + + powernv = { + linux-kernel = { + name = "PowerNV"; + + baseConfig = "powernv_defconfig"; + target = "vmlinux"; + autoModules = true; + # avoid driver/FS trouble arising from unusual page size + extraConfig = '' + PPC_64K_PAGES n + PPC_4K_PAGES y + IPV6 y + + ATA_BMDMA y + ATA_SFF y + VIRTIO_MENU y + ''; + }; + }; + + ## + ## ARM + ## + + pogoplug4 = { + linux-kernel = { + name = "pogoplug4"; + + baseConfig = "multi_v5_defconfig"; + autoModules = false; + extraConfig = '' + # Ubi for the mtd + MTD_UBI y + UBIFS_FS y + UBIFS_FS_XATTR y + UBIFS_FS_ADVANCED_COMPR y + UBIFS_FS_LZO y + UBIFS_FS_ZLIB y + UBIFS_FS_DEBUG n + ''; + makeFlags = [ "LOADADDR=0x8000" ]; + target = "uImage"; + # TODO reenable once manual-config's config actually builds a .dtb and this is checked to be working + #DTB = true; + }; + gcc = { + arch = "armv5te"; + }; + }; + + sheevaplug = { + linux-kernel = { + name = "sheevaplug"; + + baseConfig = "multi_v5_defconfig"; + autoModules = false; + extraConfig = '' + BLK_DEV_RAM y + BLK_DEV_INITRD y + BLK_DEV_CRYPTOLOOP m + BLK_DEV_DM m + DM_CRYPT m + MD y + REISERFS_FS m + BTRFS_FS m + XFS_FS m + JFS_FS m + EXT4_FS m + USB_STORAGE_CYPRESS_ATACB m + + # mv cesa requires this sw fallback, for mv-sha1 + CRYPTO_SHA1 y + # Fast crypto + CRYPTO_TWOFISH y + CRYPTO_TWOFISH_COMMON y + CRYPTO_BLOWFISH y + CRYPTO_BLOWFISH_COMMON y + + IP_PNP y + IP_PNP_DHCP y + NFS_FS y + ROOT_NFS y + TUN m + NFS_V4 y + NFS_V4_1 y + NFS_FSCACHE y + NFSD m + NFSD_V2_ACL y + NFSD_V3 y + NFSD_V3_ACL y + NFSD_V4 y + NETFILTER y + IP_NF_IPTABLES y + IP_NF_FILTER y + IP_NF_MATCH_ADDRTYPE y + IP_NF_TARGET_LOG y + IP_NF_MANGLE y + IPV6 m + VLAN_8021Q m + + CIFS y + CIFS_XATTR y + CIFS_POSIX y + CIFS_FSCACHE y + CIFS_ACL y + + WATCHDOG y + WATCHDOG_CORE y + ORION_WATCHDOG m + + ZRAM m + NETCONSOLE m + + # Disable OABI to have seccomp_filter (required for systemd) + # https://github.com/raspberrypi/firmware/issues/651 + OABI_COMPAT n + + # Fail to build + DRM n + SCSI_ADVANSYS n + USB_ISP1362_HCD n + SND_SOC n + SND_ALI5451 n + FB_SAVAGE n + SCSI_NSP32 n + ATA_SFF n + SUNGEM n + IRDA n + ATM_HE n + SCSI_ACARD n + BLK_DEV_CMD640_ENHANCED n + + FUSE_FS m + + # systemd uses cgroups + CGROUPS y + + # Latencytop + LATENCYTOP y + + # Ubi for the mtd + MTD_UBI y + UBIFS_FS y + UBIFS_FS_XATTR y + UBIFS_FS_ADVANCED_COMPR y + UBIFS_FS_LZO y + UBIFS_FS_ZLIB y + UBIFS_FS_DEBUG n + + # Kdb, for kernel troubles + KGDB y + KGDB_SERIAL_CONSOLE y + KGDB_KDB y + ''; + makeFlags = [ "LOADADDR=0x0200000" ]; + target = "uImage"; + DTB = true; # Beyond 3.10 + }; + gcc = { + arch = "armv5te"; + }; + }; + + raspberrypi = { + linux-kernel = { + name = "raspberrypi"; + + baseConfig = "bcm2835_defconfig"; + DTB = true; + autoModules = true; + preferBuiltin = true; + extraConfig = '' + # Disable OABI to have seccomp_filter (required for systemd) + # https://github.com/raspberrypi/firmware/issues/651 + OABI_COMPAT n + ''; + target = "zImage"; + }; + gcc = { + arch = "armv6"; + fpu = "vfp"; + }; + }; + + # Legacy attribute, for compatibility with existing configs only. + raspberrypi2 = armv7l-hf-multiplatform; + + zero-gravitas = { + linux-kernel = { + name = "zero-gravitas"; + + baseConfig = "zero-gravitas_defconfig"; + # Target verified by checking /boot on reMarkable 1 device + target = "zImage"; + autoModules = false; + DTB = true; + }; + gcc = { + fpu = "neon"; + cpu = "cortex-a9"; + }; + }; + + zero-sugar = { + linux-kernel = { + name = "zero-sugar"; + + baseConfig = "zero-sugar_defconfig"; + DTB = true; + autoModules = false; + preferBuiltin = true; + target = "zImage"; + }; + gcc = { + cpu = "cortex-a7"; + fpu = "neon-vfpv4"; + float-abi = "hard"; + }; + }; + + utilite = { + linux-kernel = { + name = "utilite"; + maseConfig = "multi_v7_defconfig"; + autoModules = false; + extraConfig = '' + # Ubi for the mtd + MTD_UBI y + UBIFS_FS y + UBIFS_FS_XATTR y + UBIFS_FS_ADVANCED_COMPR y + UBIFS_FS_LZO y + UBIFS_FS_ZLIB y + UBIFS_FS_DEBUG n + ''; + makeFlags = [ "LOADADDR=0x10800000" ]; + target = "uImage"; + DTB = true; + }; + gcc = { + cpu = "cortex-a9"; + fpu = "neon"; + }; + }; + + guruplug = lib.recursiveUpdate sheevaplug { + # Define `CONFIG_MACH_GURUPLUG' (see + # <http://kerneltrap.org/mailarchive/git-commits-head/2010/5/19/33618>) + # and other GuruPlug-specific things. Requires the `guruplug-defconfig' + # patch. + linux-kernel.baseConfig = "guruplug_defconfig"; + }; + + beaglebone = lib.recursiveUpdate armv7l-hf-multiplatform { + linux-kernel = { + name = "beaglebone"; + baseConfig = "bb.org_defconfig"; + autoModules = false; + extraConfig = ""; # TBD kernel config + target = "zImage"; + }; + }; + + # https://developer.android.com/ndk/guides/abis#v7a + armv7a-android = { + linux-kernel.name = "armeabi-v7a"; + gcc = { + arch = "armv7-a"; + float-abi = "softfp"; + fpu = "vfpv3-d16"; + }; + }; + + armv7l-hf-multiplatform = { + linux-kernel = { + name = "armv7l-hf-multiplatform"; + Major = "2.6"; # Using "2.6" enables 2.6 kernel syscalls in glibc. + baseConfig = "multi_v7_defconfig"; + DTB = true; + autoModules = true; + preferBuiltin = true; + target = "zImage"; + extraConfig = '' + # Serial port for Raspberry Pi 3. Wasn't included in ARMv7 defconfig + # until 4.17. + SERIAL_8250_BCM2835AUX y + SERIAL_8250_EXTENDED y + SERIAL_8250_SHARE_IRQ y + + # Hangs ODROID-XU4 + ARM_BIG_LITTLE_CPUIDLE n + + # Disable OABI to have seccomp_filter (required for systemd) + # https://github.com/raspberrypi/firmware/issues/651 + OABI_COMPAT n + + # >=5.12 fails with: + # drivers/net/ethernet/micrel/ks8851_common.o: in function `ks8851_probe_common': + # ks8851_common.c:(.text+0x179c): undefined reference to `__this_module' + # See: https://lore.kernel.org/netdev/20210116164828.40545-1-marex@denx.de/T/ + KS8851_MLL y + ''; + }; + gcc = { + # Some table about fpu flags: + # http://community.arm.com/servlet/JiveServlet/showImage/38-1981-3827/blogentry-103749-004812900+1365712953_thumb.png + # Cortex-A5: -mfpu=neon-fp16 + # Cortex-A7 (rpi2): -mfpu=neon-vfpv4 + # Cortex-A8 (beaglebone): -mfpu=neon + # Cortex-A9: -mfpu=neon-fp16 + # Cortex-A15: -mfpu=neon-vfpv4 + + # More about FPU: + # https://wiki.debian.org/ArmHardFloatPort/VfpComparison + + # vfpv3-d16 is what Debian uses and seems to be the best compromise: NEON is not supported in e.g. Scaleway or Tegra 2, + # and the above page suggests NEON is only an improvement with hand-written assembly. + arch = "armv7-a"; + fpu = "vfpv3-d16"; + + # For Raspberry Pi the 2 the best would be: + # cpu = "cortex-a7"; + # fpu = "neon-vfpv4"; + }; + }; + + aarch64-multiplatform = { + linux-kernel = { + name = "aarch64-multiplatform"; + baseConfig = "defconfig"; + DTB = true; + autoModules = true; + preferBuiltin = true; + extraConfig = '' + # Raspberry Pi 3 stuff. Not needed for s >= 4.10. + ARCH_BCM2835 y + BCM2835_MBOX y + BCM2835_WDT y + RASPBERRYPI_FIRMWARE y + RASPBERRYPI_POWER y + SERIAL_8250_BCM2835AUX y + SERIAL_8250_EXTENDED y + SERIAL_8250_SHARE_IRQ y + + # Cavium ThunderX stuff. + PCI_HOST_THUNDER_ECAM y + + # Nvidia Tegra stuff. + PCI_TEGRA y + + # The default (=y) forces us to have the XHCI firmware available in initrd, + # which our initrd builder can't currently do easily. + USB_XHCI_TEGRA m + ''; + target = "Image"; + }; + gcc = { + arch = "armv8-a"; + }; + }; + + apple-m1 = { + gcc = { + arch = "armv8.3-a+crypto+sha2+aes+crc+fp16+lse+simd+ras+rdm+rcpc"; + cpu = "apple-a13"; + }; + }; + + ## + ## MIPS + ## + + ben_nanonote = { + linux-kernel = { + name = "ben_nanonote"; + }; + gcc = { + arch = "mips32"; + float = "soft"; + }; + }; + + fuloong2f_n32 = { + linux-kernel = { + name = "fuloong2f_n32"; + baseConfig = "lemote2f_defconfig"; + autoModules = false; + extraConfig = '' + MIGRATION n + COMPACTION n + + # nixos mounts some cgroup + CGROUPS y + + BLK_DEV_RAM y + BLK_DEV_INITRD y + BLK_DEV_CRYPTOLOOP m + BLK_DEV_DM m + DM_CRYPT m + MD y + REISERFS_FS m + EXT4_FS m + USB_STORAGE_CYPRESS_ATACB m + + IP_PNP y + IP_PNP_DHCP y + IP_PNP_BOOTP y + NFS_FS y + ROOT_NFS y + TUN m + NFS_V4 y + NFS_V4_1 y + NFS_FSCACHE y + NFSD m + NFSD_V2_ACL y + NFSD_V3 y + NFSD_V3_ACL y + NFSD_V4 y + + # Fail to build + DRM n + SCSI_ADVANSYS n + USB_ISP1362_HCD n + SND_SOC n + SND_ALI5451 n + FB_SAVAGE n + SCSI_NSP32 n + ATA_SFF n + SUNGEM n + IRDA n + ATM_HE n + SCSI_ACARD n + BLK_DEV_CMD640_ENHANCED n + + FUSE_FS m + + # Needed for udev >= 150 + SYSFS_DEPRECATED_V2 n + + VGA_CONSOLE n + VT_HW_CONSOLE_BINDING y + SERIAL_8250_CONSOLE y + FRAMEBUFFER_CONSOLE y + EXT2_FS y + EXT3_FS y + REISERFS_FS y + MAGIC_SYSRQ y + + # The kernel doesn't boot at all, with FTRACE + FTRACE n + ''; + target = "vmlinux"; + }; + gcc = { + arch = "loongson2f"; + float = "hard"; + abi = "n32"; + }; + }; + + # can execute on 32bit chip + gcc_mips32r2_o32 = { gcc = { arch = "mips32r2"; abi = "32"; }; }; + gcc_mips32r6_o32 = { gcc = { arch = "mips32r6"; abi = "32"; }; }; + gcc_mips64r2_n32 = { gcc = { arch = "mips64r2"; abi = "n32"; }; }; + gcc_mips64r6_n32 = { gcc = { arch = "mips64r6"; abi = "n32"; }; }; + gcc_mips64r2_64 = { gcc = { arch = "mips64r2"; abi = "64"; }; }; + gcc_mips64r6_64 = { gcc = { arch = "mips64r6"; abi = "64"; }; }; + + # based on: + # https://www.mail-archive.com/qemu-discuss@nongnu.org/msg05179.html + # https://gmplib.org/~tege/qemu.html#mips64-debian + mips64el-qemu-linux-gnuabi64 = { + linux-kernel = { + name = "mips64el"; + baseConfig = "64r2el_defconfig"; + target = "vmlinuz"; + autoModules = false; + DTB = true; + # for qemu 9p passthrough filesystem + extraConfig = '' + MIPS_MALTA y + PAGE_SIZE_4KB y + CPU_LITTLE_ENDIAN y + CPU_MIPS64_R2 y + 64BIT y + CPU_MIPS64_R2 y + + NET_9P y + NET_9P_VIRTIO y + 9P_FS y + 9P_FS_POSIX_ACL y + PCI y + VIRTIO_PCI y + ''; + }; + }; + + ## + ## Other + ## + + riscv-multiplatform = { + linux-kernel = { + name = "riscv-multiplatform"; + target = "Image"; + autoModules = true; + baseConfig = "defconfig"; + DTB = true; + extraConfig = '' + SERIAL_OF_PLATFORM y + ''; + }; + }; + + # This function takes a minimally-valid "platform" and returns an + # attrset containing zero or more additional attrs which should be + # included in the platform in order to further elaborate it. + select = platform: + # x86 + /**/ if platform.isx86 then pc + + # ARM + else if platform.isAarch32 then let + version = platform.parsed.cpu.version or null; + in if version == null then pc + else if lib.versionOlder version "6" then sheevaplug + else if lib.versionOlder version "7" then raspberrypi + else armv7l-hf-multiplatform + + else if platform.isAarch64 then + if platform.isDarwin then apple-m1 + else aarch64-multiplatform + + else if platform.isRiscV then riscv-multiplatform + + else if platform.parsed.cpu == lib.systems.parse.cpuTypes.mipsel then (import ./examples.nix { inherit lib; }).mipsel-linux-gnu + + else if platform.parsed.cpu == lib.systems.parse.cpuTypes.powerpc64le then powernv + + else { }; +} diff --git a/tests/check-eval.nix b/tests/check-eval.nix new file mode 100644 index 000000000..8bd7b605a --- /dev/null +++ b/tests/check-eval.nix @@ -0,0 +1,7 @@ +# Throws an error if any of our lib tests fail. + +let tests = [ "misc" "systems" ]; + all = builtins.concatLists (map (f: import (./. + "/${f}.nix")) tests); +in if all == [] + then null + else throw (builtins.toJSON all) diff --git a/tests/filesystem.sh b/tests/filesystem.sh new file mode 100755 index 000000000..cfd333d00 --- /dev/null +++ b/tests/filesystem.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Tests lib/filesystem.nix +# Run: +# [nixpkgs]$ lib/tests/filesystem.sh +# or: +# [nixpkgs]$ nix-build lib/tests/release.nix + +set -euo pipefail +shopt -s inherit_errexit + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + NIX_PATH=nixpkgs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" +fi +export NIX_PATH + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd "$work" + +mkdir directory +touch regular +ln -s target symlink +mkfifo fifo + +expectSuccess() { + local expr=$1 + local expectedResultRegex=$2 + if ! result=$(nix-instantiate --eval --strict --json \ + --expr "with (import <nixpkgs/lib>).filesystem; $expr"); then + die "$expr failed to evaluate, but it was expected to succeed" + fi + if [[ ! "$result" =~ $expectedResultRegex ]]; then + die "$expr == $result, but $expectedResultRegex was expected" + fi +} + +expectFailure() { + local expr=$1 + local expectedErrorRegex=$2 + if result=$(nix-instantiate --eval --strict --json 2>"$work/stderr" \ + --expr "with (import <nixpkgs/lib>).filesystem; $expr"); then + die "$expr evaluated successfully to $result, but it was expected to fail" + fi + if [[ ! "$(<"$work/stderr")" =~ $expectedErrorRegex ]]; then + die "Error was $(<"$work/stderr"), but $expectedErrorRegex was expected" + fi +} + +expectSuccess "pathType /." '"directory"' +expectSuccess "pathType $PWD/directory" '"directory"' +expectSuccess "pathType $PWD/regular" '"regular"' +expectSuccess "pathType $PWD/symlink" '"symlink"' +expectSuccess "pathType $PWD/fifo" '"unknown"' +# Different errors depending on whether the builtins.readFilePath primop is available or not +expectFailure "pathType $PWD/non-existent" "error: (evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'|getting status of '$PWD/non-existent': No such file or directory)" + +expectSuccess "pathIsDirectory /." "true" +expectSuccess "pathIsDirectory $PWD/directory" "true" +expectSuccess "pathIsDirectory $PWD/regular" "false" +expectSuccess "pathIsDirectory $PWD/symlink" "false" +expectSuccess "pathIsDirectory $PWD/fifo" "false" +expectSuccess "pathIsDirectory $PWD/non-existent" "false" + +expectSuccess "pathIsRegularFile /." "false" +expectSuccess "pathIsRegularFile $PWD/directory" "false" +expectSuccess "pathIsRegularFile $PWD/regular" "true" +expectSuccess "pathIsRegularFile $PWD/symlink" "false" +expectSuccess "pathIsRegularFile $PWD/fifo" "false" +expectSuccess "pathIsRegularFile $PWD/non-existent" "false" + +echo >&2 tests ok diff --git a/tests/maintainer-module.nix b/tests/maintainer-module.nix new file mode 100644 index 000000000..afa12587a --- /dev/null +++ b/tests/maintainer-module.nix @@ -0,0 +1,32 @@ +{ lib, ... }: +let + inherit (lib) types; +in { + options = { + name = lib.mkOption { + type = types.str; + }; + email = lib.mkOption { + type = types.nullOr types.str; + default = null; + }; + matrix = lib.mkOption { + type = types.nullOr types.str; + default = null; + }; + github = lib.mkOption { + type = types.nullOr types.str; + default = null; + }; + githubId = lib.mkOption { + type = types.nullOr types.ints.unsigned; + default = null; + }; + keys = lib.mkOption { + type = types.listOf (types.submodule { + options.fingerprint = lib.mkOption { type = types.str; }; + }); + default = []; + }; + }; +} diff --git a/tests/maintainers.nix b/tests/maintainers.nix new file mode 100644 index 000000000..be1c8aaa8 --- /dev/null +++ b/tests/maintainers.nix @@ -0,0 +1,53 @@ +# to run these tests (and the others) +# nix-build nixpkgs/lib/tests/release.nix +# These tests should stay in sync with the comment in maintainers/maintainers-list.nix +{ # The pkgs used for dependencies for the testing itself + pkgs ? import ../.. {} +, lib ? pkgs.lib +}: + +let + checkMaintainer = handle: uncheckedAttrs: + let + prefix = [ "lib" "maintainers" handle ]; + checkedAttrs = (lib.modules.evalModules { + inherit prefix; + modules = [ + ./maintainer-module.nix + { + _file = toString ../../maintainers/maintainer-list.nix; + config = uncheckedAttrs; + } + ]; + }).config; + + checks = lib.optional (checkedAttrs.github != null && checkedAttrs.githubId == null) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': If `github` is specified, `githubId` must be too.' + # Calling this too often would hit non-authenticated API limits, but this + # shouldn't happen since such errors will get fixed rather quickly + info=$(curl -sS https://api.github.com/users/${checkedAttrs.github}) + id=$(jq -r '.id' <<< "$info") + echo "The GitHub ID for GitHub user ${checkedAttrs.github} is $id:" + echo -e " githubId = $id;\n" + '' ++ lib.optional (checkedAttrs.email == null && checkedAttrs.github == null && checkedAttrs.matrix == null) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': At least one of `email`, `github` or `matrix` must be specified, so that users know how to reach you.' + '' ++ lib.optional (checkedAttrs.email != null && lib.hasSuffix "noreply.github.com" checkedAttrs.email) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': If an email address is given, it should allow people to reach you. If you do not want that, you can just provide `github` or `matrix` instead.' + ''; + in lib.deepSeq checkedAttrs checks; + + missingGithubIds = lib.concatLists (lib.mapAttrsToList checkMaintainer lib.maintainers); + + success = pkgs.runCommand "checked-maintainers-success" {} ">$out"; + + failure = pkgs.runCommand "checked-maintainers-failure" { + nativeBuildInputs = [ pkgs.curl pkgs.jq ]; + outputHash = "sha256:${lib.fakeSha256}"; + outputHAlgo = "sha256"; + outputHashMode = "flat"; + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + } '' + ${lib.concatStringsSep "\n" missingGithubIds} + exit 1 + ''; +in if missingGithubIds == [] then success else failure diff --git a/tests/misc.nix b/tests/misc.nix new file mode 100644 index 000000000..ce980436c --- /dev/null +++ b/tests/misc.nix @@ -0,0 +1,1657 @@ +# to run these tests: +# nix-instantiate --eval --strict nixpkgs/lib/tests/misc.nix +# if the resulting list is empty, all tests passed +with import ../default.nix; + +let + testingThrow = expr: { + expr = (builtins.tryEval (builtins.seq expr "didn't throw")); + expected = { success = false; value = false; }; + }; + testingDeepThrow = expr: testingThrow (builtins.deepSeq expr expr); + + testSanitizeDerivationName = { name, expected }: + let + drv = derivation { + name = strings.sanitizeDerivationName name; + builder = "x"; + system = "x"; + }; + in { + # Evaluate the derivation so an invalid name would be caught + expr = builtins.seq drv.drvPath drv.name; + inherit expected; + }; + +in + +runTests { + +# TRIVIAL + + testId = { + expr = id 1; + expected = 1; + }; + + testConst = { + expr = const 2 3; + expected = 2; + }; + + testPipe = { + expr = pipe 2 [ + (x: x + 2) # 2 + 2 = 4 + (x: x * 2) # 4 * 2 = 8 + ]; + expected = 8; + }; + + testPipeEmpty = { + expr = pipe 2 []; + expected = 2; + }; + + testPipeStrings = { + expr = pipe [ 3 4 ] [ + (map toString) + (map (s: s + "\n")) + concatStrings + ]; + expected = '' + 3 + 4 + ''; + }; + + /* + testOr = { + expr = or true false; + expected = true; + }; + */ + + testAnd = { + expr = and true false; + expected = false; + }; + + testFix = { + expr = fix (x: {a = if x ? a then "a" else "b";}); + expected = {a = "a";}; + }; + + testComposeExtensions = { + expr = let obj = makeExtensible (self: { foo = self.bar; }); + f = self: super: { bar = false; baz = true; }; + g = self: super: { bar = super.baz or false; }; + f_o_g = composeExtensions f g; + composed = obj.extend f_o_g; + in composed.foo; + expected = true; + }; + + testComposeManyExtensions0 = { + expr = let obj = makeExtensible (self: { foo = true; }); + emptyComposition = composeManyExtensions []; + composed = obj.extend emptyComposition; + in composed.foo; + expected = true; + }; + + testComposeManyExtensions = + let f = self: super: { bar = false; baz = true; }; + g = self: super: { bar = super.baz or false; }; + h = self: super: { qux = super.bar or false; }; + obj = makeExtensible (self: { foo = self.qux; }); + in { + expr = let composition = composeManyExtensions [f g h]; + composed = obj.extend composition; + in composed.foo; + expected = (obj.extend (composeExtensions f (composeExtensions g h))).foo; + }; + + testBitAnd = { + expr = (bitAnd 3 10); + expected = 2; + }; + + testBitOr = { + expr = (bitOr 3 10); + expected = 11; + }; + + testBitXor = { + expr = (bitXor 3 10); + expected = 9; + }; + + testToHexString = { + expr = toHexString 250; + expected = "FA"; + }; + + testToBaseDigits = { + expr = toBaseDigits 2 6; + expected = [ 1 1 0 ]; + }; + + testFunctionArgsFunctor = { + expr = functionArgs { __functor = self: { a, b }: null; }; + expected = { a = false; b = false; }; + }; + + testFunctionArgsSetFunctionArgs = { + expr = functionArgs (setFunctionArgs (args: args.x) { x = false; }); + expected = { x = false; }; + }; + +# STRINGS + + testConcatMapStrings = { + expr = concatMapStrings (x: x + ";") ["a" "b" "c"]; + expected = "a;b;c;"; + }; + + testConcatStringsSep = { + expr = concatStringsSep "," ["a" "b" "c"]; + expected = "a,b,c"; + }; + + testConcatLines = { + expr = concatLines ["a" "b" "c"]; + expected = "a\nb\nc\n"; + }; + + testSplitStringsSimple = { + expr = strings.splitString "." "a.b.c.d"; + expected = [ "a" "b" "c" "d" ]; + }; + + testSplitStringsEmpty = { + expr = strings.splitString "." "a..b"; + expected = [ "a" "" "b" ]; + }; + + testSplitStringsOne = { + expr = strings.splitString ":" "a.b"; + expected = [ "a.b" ]; + }; + + testSplitStringsNone = { + expr = strings.splitString "." ""; + expected = [ "" ]; + }; + + testSplitStringsFirstEmpty = { + expr = strings.splitString "/" "/a/b/c"; + expected = [ "" "a" "b" "c" ]; + }; + + testSplitStringsLastEmpty = { + expr = strings.splitString ":" "2001:db8:0:0042::8a2e:370:"; + expected = [ "2001" "db8" "0" "0042" "" "8a2e" "370" "" ]; + }; + + testSplitStringsRegex = { + expr = strings.splitString "\\[{}]()^$?*+|." "A\\[{}]()^$?*+|.B"; + expected = [ "A" "B" ]; + }; + + testSplitStringsDerivation = { + expr = take 3 (strings.splitString "/" (derivation { + name = "name"; + builder = "builder"; + system = "system"; + })); + expected = ["" "nix" "store"]; + }; + + testSplitVersionSingle = { + expr = versions.splitVersion "1"; + expected = [ "1" ]; + }; + + testSplitVersionDouble = { + expr = versions.splitVersion "1.2"; + expected = [ "1" "2" ]; + }; + + testSplitVersionTriple = { + expr = versions.splitVersion "1.2.3"; + expected = [ "1" "2" "3" ]; + }; + + testPadVersionLess = { + expr = versions.pad 3 "1.2"; + expected = "1.2.0"; + }; + + testPadVersionLessExtra = { + expr = versions.pad 3 "1.3-rc1"; + expected = "1.3.0-rc1"; + }; + + testPadVersionMore = { + expr = versions.pad 3 "1.2.3.4"; + expected = "1.2.3"; + }; + + testIsStorePath = { + expr = + let goodPath = + "${builtins.storeDir}/d945ibfx9x185xf04b890y4f9g3cbb63-python-2.7.11"; + in { + storePath = isStorePath goodPath; + storePathDerivation = isStorePath (import ../.. { system = "x86_64-linux"; }).hello; + storePathAppendix = isStorePath + "${goodPath}/bin/python"; + nonAbsolute = isStorePath (concatStrings (tail (stringToCharacters goodPath))); + asPath = isStorePath (/. + goodPath); + otherPath = isStorePath "/something/else"; + otherVals = { + attrset = isStorePath {}; + list = isStorePath []; + int = isStorePath 42; + }; + }; + expected = { + storePath = true; + storePathDerivation = true; + storePathAppendix = false; + nonAbsolute = false; + asPath = true; + otherPath = false; + otherVals = { + attrset = false; + list = false; + int = false; + }; + }; + }; + + testEscapeXML = { + expr = escapeXML ''"test" 'test' < & >''; + expected = ""test" 'test' < & >"; + }; + + testToShellVars = { + expr = '' + ${toShellVars { + STRing01 = "just a 'string'"; + _array_ = [ "with" "more strings" ]; + assoc."with some" = '' + strings + possibly newlines + ''; + drv = { + outPath = "/drv"; + foo = "ignored attribute"; + }; + path = /path; + stringable = { + __toString = _: "hello toString"; + bar = "ignored attribute"; + }; + }} + ''; + expected = '' + STRing01='just a '\'''string'\'''' + declare -a _array_=('with' 'more strings') + declare -A assoc=(['with some']='strings + possibly newlines + ') + drv='/drv' + path='/path' + stringable='hello toString' + ''; + }; + + testHasInfixFalse = { + expr = hasInfix "c" "abde"; + expected = false; + }; + + testHasInfixTrue = { + expr = hasInfix "c" "abcde"; + expected = true; + }; + + testHasInfixDerivation = { + expr = hasInfix "hello" (import ../.. { system = "x86_64-linux"; }).hello; + expected = true; + }; + + testHasInfixPath = { + expr = hasInfix "tests" ./.; + expected = true; + }; + + testHasInfixPathStoreDir = { + expr = hasInfix builtins.storeDir ./.; + expected = true; + }; + + testHasInfixToString = { + expr = hasInfix "a" { __toString = _: "a"; }; + expected = true; + }; + + testNormalizePath = { + expr = strings.normalizePath "//a/b//c////d/"; + expected = "/a/b/c/d/"; + }; + + testCharToInt = { + expr = strings.charToInt "A"; + expected = 65; + }; + + testEscapeC = { + expr = strings.escapeC [ " " ] "Hello World"; + expected = "Hello\\x20World"; + }; + + testEscapeURL = testAllTrue [ + ("" == strings.escapeURL "") + ("Hello" == strings.escapeURL "Hello") + ("Hello%20World" == strings.escapeURL "Hello World") + ("Hello%2FWorld" == strings.escapeURL "Hello/World") + ("42%25" == strings.escapeURL "42%") + ("%20%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%09%3A%2F%40%24%27%28%29%2A%2C%3B" == strings.escapeURL " ?&=#+%!<>#\"{}|\\^[]`\t:/@$'()*,;") + ]; + + testToInt = testAllTrue [ + # Naive + (123 == toInt "123") + (0 == toInt "0") + # Whitespace Padding + (123 == toInt " 123") + (123 == toInt "123 ") + (123 == toInt " 123 ") + (123 == toInt " 123 ") + (0 == toInt " 0") + (0 == toInt "0 ") + (0 == toInt " 0 ") + (-1 == toInt "-1") + (-1 == toInt " -1 ") + ]; + + testToIntFails = testAllTrue [ + ( builtins.tryEval (toInt "") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " d0 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "00") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "01") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "002") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 002 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo123 ") == { success = false; value = false; } ) + ]; + + testToIntBase10 = testAllTrue [ + # Naive + (123 == toIntBase10 "123") + (0 == toIntBase10 "0") + # Whitespace Padding + (123 == toIntBase10 " 123") + (123 == toIntBase10 "123 ") + (123 == toIntBase10 " 123 ") + (123 == toIntBase10 " 123 ") + (0 == toIntBase10 " 0") + (0 == toIntBase10 "0 ") + (0 == toIntBase10 " 0 ") + # Zero Padding + (123 == toIntBase10 "0123") + (123 == toIntBase10 "0000123") + (0 == toIntBase10 "000000") + # Whitespace and Zero Padding + (123 == toIntBase10 " 0123") + (123 == toIntBase10 "0123 ") + (123 == toIntBase10 " 0123 ") + (123 == toIntBase10 " 0000123") + (123 == toIntBase10 "0000123 ") + (123 == toIntBase10 " 0000123 ") + (0 == toIntBase10 " 000000") + (0 == toIntBase10 "000000 ") + (0 == toIntBase10 " 000000 ") + (-1 == toIntBase10 "-1") + (-1 == toIntBase10 " -1 ") + ]; + + testToIntBase10Fails = testAllTrue [ + ( builtins.tryEval (toIntBase10 "") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " d0 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 00123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo00123 ") == { success = false; value = false; } ) + ]; + +# LISTS + + testFilter = { + expr = filter (x: x != "a") ["a" "b" "c" "a"]; + expected = ["b" "c"]; + }; + + testFold = + let + f = op: fold: fold op 0 (range 0 100); + # fold with associative operator + assoc = f builtins.add; + # fold with non-associative operator + nonAssoc = f builtins.sub; + in { + expr = { + assocRight = assoc foldr; + # right fold with assoc operator is same as left fold + assocRightIsLeft = assoc foldr == assoc foldl; + nonAssocRight = nonAssoc foldr; + nonAssocLeft = nonAssoc foldl; + # with non-assoc operator the fold results are not the same + nonAssocRightIsNotLeft = nonAssoc foldl != nonAssoc foldr; + # fold is an alias for foldr + foldIsRight = nonAssoc fold == nonAssoc foldr; + }; + expected = { + assocRight = 5050; + assocRightIsLeft = true; + nonAssocRight = 50; + nonAssocLeft = (-5050); + nonAssocRightIsNotLeft = true; + foldIsRight = true; + }; + }; + + testTake = testAllTrue [ + ([] == (take 0 [ 1 2 3 ])) + ([1] == (take 1 [ 1 2 3 ])) + ([ 1 2 ] == (take 2 [ 1 2 3 ])) + ([ 1 2 3 ] == (take 3 [ 1 2 3 ])) + ([ 1 2 3 ] == (take 4 [ 1 2 3 ])) + ]; + + testFoldAttrs = { + expr = foldAttrs (n: a: [n] ++ a) [] [ + { a = 2; b = 7; } + { a = 3; c = 8; } + ]; + expected = { a = [ 2 3 ]; b = [7]; c = [8];}; + }; + + testSort = { + expr = sort builtins.lessThan [ 40 2 30 42 ]; + expected = [2 30 40 42]; + }; + + testReplicate = { + expr = replicate 3 "a"; + expected = ["a" "a" "a"]; + }; + + testToIntShouldConvertStringToInt = { + expr = toInt "27"; + expected = 27; + }; + + testToIntShouldThrowErrorIfItCouldNotConvertToInt = { + expr = builtins.tryEval (toInt "\"foo\""); + expected = { success = false; value = false; }; + }; + + testHasAttrByPathTrue = { + expr = hasAttrByPath ["a" "b"] { a = { b = "yey"; }; }; + expected = true; + }; + + testHasAttrByPathFalse = { + expr = hasAttrByPath ["a" "b"] { a = { c = "yey"; }; }; + expected = false; + }; + + testFindFirstExample1 = { + expr = findFirst (x: x > 3) 7 [ 1 6 4 ]; + expected = 6; + }; + + testFindFirstExample2 = { + expr = findFirst (x: x > 9) 7 [ 1 6 4 ]; + expected = 7; + }; + + testFindFirstEmpty = { + expr = findFirst (abort "when the list is empty, the predicate is not needed") null []; + expected = null; + }; + + testFindFirstSingleMatch = { + expr = findFirst (x: x == 5) null [ 5 ]; + expected = 5; + }; + + testFindFirstSingleDefault = { + expr = findFirst (x: false) null [ (abort "if the predicate doesn't access the value, it must not be evaluated") ]; + expected = null; + }; + + testFindFirstNone = { + expr = builtins.tryEval (findFirst (x: x == 2) null [ 1 (throw "the last element must be evaluated when there's no match") ]); + expected = { success = false; value = false; }; + }; + + # Makes sure that the implementation doesn't cause a stack overflow + testFindFirstBig = { + expr = findFirst (x: x == 1000000) null (range 0 1000000); + expected = 1000000; + }; + + testFindFirstLazy = { + expr = findFirst (x: x == 1) 7 [ 1 (abort "list elements after the match must not be evaluated") ]; + expected = 1; + }; + +# ATTRSETS + + testConcatMapAttrs = { + expr = concatMapAttrs + (name: value: { + ${name} = value; + ${name + value} = value; + }) + { + foo = "bar"; + foobar = "baz"; + }; + expected = { + foo = "bar"; + foobar = "baz"; + foobarbaz = "baz"; + }; + }; + + # code from example + testFoldlAttrs = { + expr = { + example = foldlAttrs + (acc: name: value: { + sum = acc.sum + value; + names = acc.names ++ [ name ]; + }) + { sum = 0; names = [ ]; } + { + foo = 1; + bar = 10; + }; + # should just return the initial value + emptySet = foldlAttrs (throw "function not needed") 123 { }; + # should just evaluate to the last value + accNotNeeded = foldlAttrs (_acc: _name: v: v) (throw "accumulator not needed") { z = 3; a = 2; }; + # the accumulator doesnt have to be an attrset it can be as trivial as being just a number or string + trivialAcc = foldlAttrs (acc: _name: v: acc * 10 + v) 1 { z = 1; a = 2; }; + }; + expected = { + example = { + sum = 11; + names = [ "bar" "foo" ]; + }; + emptySet = 123; + accNotNeeded = 3; + trivialAcc = 121; + }; + }; + + # code from the example + testRecursiveUpdateUntil = { + expr = recursiveUpdateUntil (path: l: r: path == ["foo"]) { + # first attribute set + foo.bar = 1; + foo.baz = 2; + bar = 3; + } { + #second attribute set + foo.bar = 1; + foo.quz = 2; + baz = 4; + }; + expected = { + foo.bar = 1; # 'foo.*' from the second set + foo.quz = 2; # + bar = 3; # 'bar' from the first set + baz = 4; # 'baz' from the second set + }; + }; + + testOverrideExistingEmpty = { + expr = overrideExisting {} { a = 1; }; + expected = {}; + }; + + testOverrideExistingDisjoint = { + expr = overrideExisting { b = 2; } { a = 1; }; + expected = { b = 2; }; + }; + + testOverrideExistingOverride = { + expr = overrideExisting { a = 3; b = 2; } { a = 1; }; + expected = { a = 1; b = 2; }; + }; + +# GENERATORS +# these tests assume attributes are converted to lists +# in alphabetical order + + testMkKeyValueDefault = { + expr = generators.mkKeyValueDefault {} ":" "f:oo" "bar"; + expected = ''f\:oo:bar''; + }; + + testMkValueString = { + expr = let + vals = { + int = 42; + string = ''fo"o''; + bool = true; + bool2 = false; + null = null; + # float = 42.23; # floats are strange + }; + in mapAttrs + (const (generators.mkValueStringDefault {})) + vals; + expected = { + int = "42"; + string = ''fo"o''; + bool = "true"; + bool2 = "false"; + null = "null"; + # float = "42.23" true false [ "bar" ] ]''; + }; + }; + + testToKeyValue = { + expr = generators.toKeyValue {} { + key = "value"; + "other=key" = "baz"; + }; + expected = '' + key=value + other\=key=baz + ''; + }; + + testToINIEmpty = { + expr = generators.toINI {} {}; + expected = ""; + }; + + testToINIEmptySection = { + expr = generators.toINI {} { foo = {}; bar = {}; }; + expected = '' + [bar] + + [foo] + ''; + }; + + testToINIDuplicateKeys = { + expr = generators.toINI { listsAsDuplicateKeys = true; } { foo.bar = true; baz.qux = [ 1 false ]; }; + expected = '' + [baz] + qux=1 + qux=false + + [foo] + bar=true + ''; + }; + + testToINIDefaultEscapes = { + expr = generators.toINI {} { + "no [ and ] allowed unescaped" = { + "and also no = in keys" = 42; + }; + }; + expected = '' + [no \[ and \] allowed unescaped] + and also no \= in keys=42 + ''; + }; + + testToINIDefaultFull = { + expr = generators.toINI {} { + "section 1" = { + attribute1 = 5; + x = "Me-se JarJar Binx"; + # booleans are converted verbatim by default + boolean = false; + }; + "foo[]" = { + "he\\h=he" = "this is okay"; + }; + }; + expected = '' + [foo\[\]] + he\h\=he=this is okay + + [section 1] + attribute1=5 + boolean=false + x=Me-se JarJar Binx + ''; + }; + + testToINIWithGlobalSectionEmpty = { + expr = generators.toINIWithGlobalSection {} { + globalSection = { + }; + sections = { + }; + }; + expected = '' + ''; + }; + + testToINIWithGlobalSectionGlobalEmptyIsTheSameAsToINI = + let + sections = { + "section 1" = { + attribute1 = 5; + x = "Me-se JarJar Binx"; + }; + "foo" = { + "he\\h=he" = "this is okay"; + }; + }; + in { + expr = + generators.toINIWithGlobalSection {} { + globalSection = {}; + sections = sections; + }; + expected = generators.toINI {} sections; + }; + + testToINIWithGlobalSectionFull = { + expr = generators.toINIWithGlobalSection {} { + globalSection = { + foo = "bar"; + test = false; + }; + sections = { + "section 1" = { + attribute1 = 5; + x = "Me-se JarJar Binx"; + }; + "foo" = { + "he\\h=he" = "this is okay"; + }; + }; + }; + expected = '' + foo=bar + test=false + + [foo] + he\h\=he=this is okay + + [section 1] + attribute1=5 + x=Me-se JarJar Binx + ''; + }; + + /* right now only invocation check */ + testToJSONSimple = + let val = { + foobar = [ "baz" 1 2 3 ]; + }; + in { + expr = generators.toJSON {} val; + # trivial implementation + expected = builtins.toJSON val; + }; + + /* right now only invocation check */ + testToYAMLSimple = + let val = { + list = [ { one = 1; } { two = 2; } ]; + all = 42; + }; + in { + expr = generators.toYAML {} val; + # trivial implementation + expected = builtins.toJSON val; + }; + + testToPretty = + let + deriv = derivation { name = "test"; builder = "/bin/sh"; system = "aarch64-linux"; }; + in { + expr = mapAttrs (const (generators.toPretty { multiline = false; })) rec { + int = 42; + float = 0.1337; + bool = true; + emptystring = ""; + string = "fn\${o}\"r\\d"; + newlinestring = "\n"; + path = /. + "/foo"; + null_ = null; + function = x: x; + functionArgs = { arg ? 4, foo }: arg; + list = [ 3 4 function [ false ] ]; + emptylist = []; + attrs = { foo = null; "foo b/ar" = "baz"; }; + emptyattrs = {}; + drv = deriv; + }; + expected = rec { + int = "42"; + float = "0.1337"; + bool = "true"; + emptystring = ''""''; + string = ''"fn\''${o}\"r\\d"''; + newlinestring = "\"\\n\""; + path = "/foo"; + null_ = "null"; + function = "<function>"; + functionArgs = "<function, args: {arg?, foo}>"; + list = "[ 3 4 ${function} [ false ] ]"; + emptylist = "[ ]"; + attrs = "{ foo = null; \"foo b/ar\" = \"baz\"; }"; + emptyattrs = "{ }"; + drv = "<derivation ${deriv.name}>"; + }; + }; + + testToPrettyLimit = + let + a.b = 1; + a.c = a; + in { + expr = generators.toPretty { } (generators.withRecursion { throwOnDepthLimit = false; depthLimit = 2; } a); + expected = "{\n b = 1;\n c = {\n b = \"<unevaluated>\";\n c = {\n b = \"<unevaluated>\";\n c = \"<unevaluated>\";\n };\n };\n}"; + }; + + testToPrettyLimitThrow = + let + a.b = 1; + a.c = a; + in { + expr = (builtins.tryEval + (generators.toPretty { } (generators.withRecursion { depthLimit = 2; } a))).success; + expected = false; + }; + + testWithRecursionDealsWithFunctors = + let + functor = { + __functor = self: { a, b, }: null; + }; + a = { + value = "1234"; + b = functor; + c.d = functor; + }; + in { + expr = generators.toPretty { } (generators.withRecursion { depthLimit = 1; throwOnDepthLimit = false; } a); + expected = "{\n b = <function, args: {a, b}>;\n c = {\n d = \"<unevaluated>\";\n };\n value = \"<unevaluated>\";\n}"; + }; + + testToPrettyMultiline = { + expr = mapAttrs (const (generators.toPretty { })) rec { + list = [ 3 4 [ false ] ]; + attrs = { foo = null; bar.foo = "baz"; }; + newlinestring = "\n"; + multilinestring = '' + hello + ''${there} + te'''st + ''; + multilinestring' = '' + hello + there + test''; + }; + expected = rec { + list = '' + [ + 3 + 4 + [ + false + ] + ]''; + attrs = '' + { + bar = { + foo = "baz"; + }; + foo = null; + }''; + newlinestring = "''\n \n''"; + multilinestring = '' + ''' + hello + '''''${there} + te''''st + '''''; + multilinestring' = '' + ''' + hello + there + test'''''; + + }; + }; + + testToPrettyAllowPrettyValues = { + expr = generators.toPretty { allowPrettyValues = true; } + { __pretty = v: "«" + v + "»"; val = "foo"; }; + expected = "«foo»"; + }; + + testToPlist = + let + deriv = derivation { name = "test"; builder = "/bin/sh"; system = "aarch64-linux"; }; + in { + expr = mapAttrs (const (generators.toPlist { })) { + value = { + nested.values = rec { + int = 42; + float = 0.1337; + bool = true; + emptystring = ""; + string = "fn\${o}\"r\\d"; + newlinestring = "\n"; + path = /. + "/foo"; + null_ = null; + list = [ 3 4 "test" ]; + emptylist = []; + attrs = { foo = null; "foo b/ar" = "baz"; }; + emptyattrs = {}; + }; + }; + }; + expected = { value = builtins.readFile ./test-to-plist-expected.plist; }; + }; + + testToLuaEmptyAttrSet = { + expr = generators.toLua {} {}; + expected = ''{}''; + }; + + testToLuaEmptyList = { + expr = generators.toLua {} []; + expected = ''{}''; + }; + + testToLuaListOfVariousTypes = { + expr = generators.toLua {} [ null 43 3.14159 true ]; + expected = '' + { + nil, + 43, + 3.14159, + true + }''; + }; + + testToLuaString = { + expr = generators.toLua {} ''double-quote (") and single quotes (')''; + expected = ''"double-quote (\") and single quotes (')"''; + }; + + testToLuaAttrsetWithLuaInline = { + expr = generators.toLua {} { x = generators.mkLuaInline ''"abc" .. "def"''; }; + expected = '' + { + ["x"] = ("abc" .. "def") + }''; + }; + + testToLuaAttrsetWithSpaceInKey = { + expr = generators.toLua {} { "some space and double-quote (\")" = 42; }; + expected = '' + { + ["some space and double-quote (\")"] = 42 + }''; + }; + + testToLuaWithoutMultiline = { + expr = generators.toLua { multiline = false; } [ 41 43 ]; + expected = ''{ 41, 43 }''; + }; + + testToLuaEmptyBindings = { + expr = generators.toLua { asBindings = true; } {}; + expected = ""; + }; + + testToLuaBindings = { + expr = generators.toLua { asBindings = true; } { x1 = 41; _y = { a = 43; }; }; + expected = '' + _y = { + ["a"] = 43 + } + x1 = 41 + ''; + }; + + testToLuaPartialTableBindings = { + expr = generators.toLua { asBindings = true; } { "x.y" = 42; }; + expected = '' + x.y = 42 + ''; + }; + + testToLuaIndentedBindings = { + expr = generators.toLua { asBindings = true; indent = " "; } { x = { y = 42; }; }; + expected = " x = {\n [\"y\"] = 42\n }\n"; + }; + + testToLuaBindingsWithSpace = testingThrow ( + generators.toLua { asBindings = true; } { "with space" = 42; } + ); + + testToLuaBindingsWithLeadingDigit = testingThrow ( + generators.toLua { asBindings = true; } { "11eleven" = 42; } + ); + + testToLuaBasicExample = { + expr = generators.toLua {} { + cmd = [ "typescript-language-server" "--stdio" ]; + settings.workspace.library = generators.mkLuaInline ''vim.api.nvim_get_runtime_file("", true)''; + }; + expected = '' + { + ["cmd"] = { + "typescript-language-server", + "--stdio" + }, + ["settings"] = { + ["workspace"] = { + ["library"] = (vim.api.nvim_get_runtime_file("", true)) + } + } + }''; + }; + +# CLI + + testToGNUCommandLine = { + expr = cli.toGNUCommandLine {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + }; + + expected = [ + "-X" "PUT" + "--data" "{\"id\":0}" + "--retry" "3" + "--url" "https://example.com/foo" + "--url" "https://example.com/bar" + "--verbose" + ]; + }; + + testToGNUCommandLineShell = { + expr = cli.toGNUCommandLineShell {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + }; + + expected = "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'"; + }; + + testSanitizeDerivationNameLeadingDots = testSanitizeDerivationName { + name = "..foo"; + expected = "foo"; + }; + + testSanitizeDerivationNameUnicode = testSanitizeDerivationName { + name = "fö"; + expected = "f-"; + }; + + testSanitizeDerivationNameAscii = testSanitizeDerivationName { + name = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"; + expected = "-+--.-0123456789-=-?-ABCDEFGHIJKLMNOPQRSTUVWXYZ-_-abcdefghijklmnopqrstuvwxyz-"; + }; + + testSanitizeDerivationNameTooLong = testSanitizeDerivationName { + name = "This string is loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"; + expected = "loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"; + }; + + testSanitizeDerivationNameTooLongWithInvalid = testSanitizeDerivationName { + name = "Hello there aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa &&&&&&&&"; + expected = "there-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-"; + }; + + testSanitizeDerivationNameEmpty = testSanitizeDerivationName { + name = ""; + expected = "unknown"; + }; + + testFreeformOptions = { + expr = + let + submodule = { lib, ... }: { + freeformType = lib.types.attrsOf (lib.types.submodule { + options.bar = lib.mkOption {}; + }); + options.bar = lib.mkOption {}; + }; + + module = { lib, ... }: { + options.foo = lib.mkOption { + type = lib.types.submodule submodule; + }; + }; + + options = (evalModules { + modules = [ module ]; + }).options; + + locs = filter (o: ! o.internal) (optionAttrSetToDocList options); + in map (o: o.loc) locs; + expected = [ [ "_module" "args" ] [ "foo" ] [ "foo" "<name>" "bar" ] [ "foo" "bar" ] ]; + }; + + testCartesianProductOfEmptySet = { + expr = cartesianProductOfSets {}; + expected = [ {} ]; + }; + + testCartesianProductOfOneSet = { + expr = cartesianProductOfSets { a = [ 1 2 3 ]; }; + expected = [ { a = 1; } { a = 2; } { a = 3; } ]; + }; + + testCartesianProductOfTwoSets = { + expr = cartesianProductOfSets { a = [ 1 ]; b = [ 10 20 ]; }; + expected = [ + { a = 1; b = 10; } + { a = 1; b = 20; } + ]; + }; + + testCartesianProductOfTwoSetsWithOneEmpty = { + expr = cartesianProductOfSets { a = [ ]; b = [ 10 20 ]; }; + expected = [ ]; + }; + + testCartesianProductOfThreeSets = { + expr = cartesianProductOfSets { + a = [ 1 2 3 ]; + b = [ 10 20 30 ]; + c = [ 100 200 300 ]; + }; + expected = [ + { a = 1; b = 10; c = 100; } + { a = 1; b = 10; c = 200; } + { a = 1; b = 10; c = 300; } + + { a = 1; b = 20; c = 100; } + { a = 1; b = 20; c = 200; } + { a = 1; b = 20; c = 300; } + + { a = 1; b = 30; c = 100; } + { a = 1; b = 30; c = 200; } + { a = 1; b = 30; c = 300; } + + { a = 2; b = 10; c = 100; } + { a = 2; b = 10; c = 200; } + { a = 2; b = 10; c = 300; } + + { a = 2; b = 20; c = 100; } + { a = 2; b = 20; c = 200; } + { a = 2; b = 20; c = 300; } + + { a = 2; b = 30; c = 100; } + { a = 2; b = 30; c = 200; } + { a = 2; b = 30; c = 300; } + + { a = 3; b = 10; c = 100; } + { a = 3; b = 10; c = 200; } + { a = 3; b = 10; c = 300; } + + { a = 3; b = 20; c = 100; } + { a = 3; b = 20; c = 200; } + { a = 3; b = 20; c = 300; } + + { a = 3; b = 30; c = 100; } + { a = 3; b = 30; c = 200; } + { a = 3; b = 30; c = 300; } + ]; + }; + + # The example from the showAttrPath documentation + testShowAttrPathExample = { + expr = showAttrPath [ "foo" "10" "bar" ]; + expected = "foo.\"10\".bar"; + }; + + testShowAttrPathEmpty = { + expr = showAttrPath []; + expected = "<root attribute path>"; + }; + + testShowAttrPathVarious = { + expr = showAttrPath [ + "." + "foo" + "2" + "a2-b" + "_bc'de" + ]; + expected = ''".".foo."2".a2-b._bc'de''; + }; + + testGroupBy = { + expr = groupBy (n: toString (mod n 5)) (range 0 16); + expected = { + "0" = [ 0 5 10 15 ]; + "1" = [ 1 6 11 16 ]; + "2" = [ 2 7 12 ]; + "3" = [ 3 8 13 ]; + "4" = [ 4 9 14 ]; + }; + }; + + testGroupBy' = { + expr = groupBy' builtins.add 0 (x: boolToString (x > 2)) [ 5 1 2 3 4 ]; + expected = { false = 3; true = 12; }; + }; + + # The example from the updateManyAttrsByPath documentation + testUpdateManyAttrsByPathExample = { + expr = updateManyAttrsByPath [ + { + path = [ "a" "b" ]; + update = old: { d = old.c; }; + } + { + path = [ "a" "b" "c" ]; + update = old: old + 1; + } + { + path = [ "x" "y" ]; + update = old: "xy"; + } + ] { a.b.c = 0; }; + expected = { a = { b = { d = 1; }; }; x = { y = "xy"; }; }; + }; + + # If there are no updates, the value is passed through + testUpdateManyAttrsByPathNone = { + expr = updateManyAttrsByPath [] "something"; + expected = "something"; + }; + + # A single update to the root path is just like applying the function directly + testUpdateManyAttrsByPathSingleIncrement = { + expr = updateManyAttrsByPath [ + { + path = [ ]; + update = old: old + 1; + } + ] 0; + expected = 1; + }; + + # Multiple updates can be applied are done in order + testUpdateManyAttrsByPathMultipleIncrements = { + expr = updateManyAttrsByPath [ + { + path = [ ]; + update = old: old + "a"; + } + { + path = [ ]; + update = old: old + "b"; + } + { + path = [ ]; + update = old: old + "c"; + } + ] ""; + expected = "abc"; + }; + + # If an update doesn't use the value, all previous updates are not evaluated + testUpdateManyAttrsByPathLazy = { + expr = updateManyAttrsByPath [ + { + path = [ ]; + update = old: old + throw "nope"; + } + { + path = [ ]; + update = old: "untainted"; + } + ] (throw "start"); + expected = "untainted"; + }; + + # Deeply nested attributes can be updated without affecting others + testUpdateManyAttrsByPathDeep = { + expr = updateManyAttrsByPath [ + { + path = [ "a" "b" "c" ]; + update = old: old + 1; + } + ] { + a.b.c = 0; + + a.b.z = 0; + a.y.z = 0; + x.y.z = 0; + }; + expected = { + a.b.c = 1; + + a.b.z = 0; + a.y.z = 0; + x.y.z = 0; + }; + }; + + # Nested attributes are updated first + testUpdateManyAttrsByPathNestedBeforehand = { + expr = updateManyAttrsByPath [ + { + path = [ "a" ]; + update = old: old // { x = old.b; }; + } + { + path = [ "a" "b" ]; + update = old: old + 1; + } + ] { + a.b = 0; + }; + expected = { + a.b = 1; + a.x = 1; + }; + }; + + ## Levenshtein distance functions and co. + testCommonPrefixLengthEmpty = { + expr = strings.commonPrefixLength "" "hello"; + expected = 0; + }; + + testCommonPrefixLengthSame = { + expr = strings.commonPrefixLength "hello" "hello"; + expected = 5; + }; + + testCommonPrefixLengthDiffering = { + expr = strings.commonPrefixLength "hello" "hey"; + expected = 2; + }; + + testCommonSuffixLengthEmpty = { + expr = strings.commonSuffixLength "" "hello"; + expected = 0; + }; + + testCommonSuffixLengthSame = { + expr = strings.commonSuffixLength "hello" "hello"; + expected = 5; + }; + + testCommonSuffixLengthDiffering = { + expr = strings.commonSuffixLength "test" "rest"; + expected = 3; + }; + + testLevenshteinEmpty = { + expr = strings.levenshtein "" ""; + expected = 0; + }; + + testLevenshteinOnlyAdd = { + expr = strings.levenshtein "" "hello there"; + expected = 11; + }; + + testLevenshteinOnlyRemove = { + expr = strings.levenshtein "hello there" ""; + expected = 11; + }; + + testLevenshteinOnlyTransform = { + expr = strings.levenshtein "abcdef" "ghijkl"; + expected = 6; + }; + + testLevenshteinMixed = { + expr = strings.levenshtein "kitchen" "sitting"; + expected = 5; + }; + + testLevenshteinAtMostZeroFalse = { + expr = strings.levenshteinAtMost 0 "foo" "boo"; + expected = false; + }; + + testLevenshteinAtMostZeroTrue = { + expr = strings.levenshteinAtMost 0 "foo" "foo"; + expected = true; + }; + + testLevenshteinAtMostOneFalse = { + expr = strings.levenshteinAtMost 1 "car" "ct"; + expected = false; + }; + + testLevenshteinAtMostOneTrue = { + expr = strings.levenshteinAtMost 1 "car" "cr"; + expected = true; + }; + + # We test levenshteinAtMost 2 particularly well because it uses a complicated + # implementation + testLevenshteinAtMostTwoIsEmpty = { + expr = strings.levenshteinAtMost 2 "" ""; + expected = true; + }; + + testLevenshteinAtMostTwoIsZero = { + expr = strings.levenshteinAtMost 2 "abcdef" "abcdef"; + expected = true; + }; + + testLevenshteinAtMostTwoIsOne = { + expr = strings.levenshteinAtMost 2 "abcdef" "abddef"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff0False = { + expr = strings.levenshteinAtMost 2 "abcdef" "aczyef"; + expected = false; + }; + + testLevenshteinAtMostTwoDiff0Outer = { + expr = strings.levenshteinAtMost 2 "abcdef" "zbcdez"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff0DelLeft = { + expr = strings.levenshteinAtMost 2 "abcdef" "bcdefz"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff0DelRight = { + expr = strings.levenshteinAtMost 2 "abcdef" "zabcde"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff1False = { + expr = strings.levenshteinAtMost 2 "abcdef" "bddez"; + expected = false; + }; + + testLevenshteinAtMostTwoDiff1DelLeft = { + expr = strings.levenshteinAtMost 2 "abcdef" "bcdez"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff1DelRight = { + expr = strings.levenshteinAtMost 2 "abcdef" "zbcde"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff2False = { + expr = strings.levenshteinAtMost 2 "hello" "hxo"; + expected = false; + }; + + testLevenshteinAtMostTwoDiff2True = { + expr = strings.levenshteinAtMost 2 "hello" "heo"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff3 = { + expr = strings.levenshteinAtMost 2 "hello" "ho"; + expected = false; + }; + + testLevenshteinAtMostThreeFalse = { + expr = strings.levenshteinAtMost 3 "hello" "Holla!"; + expected = false; + }; + + testLevenshteinAtMostThreeTrue = { + expr = strings.levenshteinAtMost 3 "hello" "Holla"; + expected = true; + }; + + # lazyDerivation + + testLazyDerivationIsLazyInDerivationForAttrNames = { + expr = attrNames (lazyDerivation { + derivation = throw "not lazy enough"; + }); + # It's ok to add attribute names here when lazyDerivation is improved + # in accordance with its inline comments. + expected = [ "drvPath" "meta" "name" "out" "outPath" "outputName" "outputs" "system" "type" ]; + }; + + testLazyDerivationIsLazyInDerivationForPassthruAttr = { + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + passthru.tests = "whatever is in tests"; + }).tests; + expected = "whatever is in tests"; + }; + + testLazyDerivationIsLazyInDerivationForPassthruAttr2 = { + # passthru.tests is not a special case. It works for any attr. + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + passthru.foo = "whatever is in foo"; + }).foo; + expected = "whatever is in foo"; + }; + + testLazyDerivationIsLazyInDerivationForMeta = { + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + meta = "whatever is in meta"; + }).meta; + expected = "whatever is in meta"; + }; + + testLazyDerivationReturnsDerivationAttrs = let + derivation = { + type = "derivation"; + outputs = ["out"]; + out = "test out"; + outPath = "test outPath"; + outputName = "out"; + drvPath = "test drvPath"; + name = "test name"; + system = "test system"; + meta = "test meta"; + }; + in { + expr = lazyDerivation { inherit derivation; }; + expected = derivation; + }; + + testTypeDescriptionInt = { + expr = (with types; int).description; + expected = "signed integer"; + }; + testTypeDescriptionListOfInt = { + expr = (with types; listOf int).description; + expected = "list of signed integer"; + }; + testTypeDescriptionListOfListOfInt = { + expr = (with types; listOf (listOf int)).description; + expected = "list of list of signed integer"; + }; + testTypeDescriptionListOfEitherStrOrBool = { + expr = (with types; listOf (either str bool)).description; + expected = "list of (string or boolean)"; + }; + testTypeDescriptionEitherListOfStrOrBool = { + expr = (with types; either (listOf bool) str).description; + expected = "(list of boolean) or string"; + }; + testTypeDescriptionEitherStrOrListOfBool = { + expr = (with types; either str (listOf bool)).description; + expected = "string or list of boolean"; + }; + testTypeDescriptionOneOfListOfStrOrBool = { + expr = (with types; oneOf [ (listOf bool) str ]).description; + expected = "(list of boolean) or string"; + }; + testTypeDescriptionOneOfListOfStrOrBoolOrNumber = { + expr = (with types; oneOf [ (listOf bool) str number ]).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherListOfBoolOrEitherStringOrNumber = { + expr = (with types; either (listOf bool) (either str number)).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherEitherListOfBoolOrStringOrNumber = { + expr = (with types; either (either (listOf bool) str) number).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherNullOrBoolOrString = { + expr = (with types; either (nullOr bool) str).description; + expected = "null or boolean or string"; + }; + testTypeDescriptionEitherListOfEitherBoolOrStrOrInt = { + expr = (with types; either (listOf (either bool str)) int).description; + expected = "(list of (boolean or string)) or signed integer"; + }; + testTypeDescriptionEitherIntOrListOrEitherBoolOrStr = { + expr = (with types; either int (listOf (either bool str))).description; + expected = "signed integer or list of (boolean or string)"; + }; +} diff --git a/tests/modules.sh b/tests/modules.sh new file mode 100755 index 000000000..7aebba6b5 --- /dev/null +++ b/tests/modules.sh @@ -0,0 +1,408 @@ +#!/usr/bin/env bash +# +# This script is used to test that the module system is working as expected. +# By default it test the version of nixpkgs which is defined in the NIX_PATH. + +set -o errexit -o noclobber -o nounset -o pipefail +shopt -s failglob inherit_errexit + +# https://stackoverflow.com/a/246128/6605742 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +cd "$DIR"/modules + +pass=0 +fail=0 + +evalConfig() { + local attr=$1 + shift + local script="import ./default.nix { modules = [ $* ];}" + nix-instantiate --timeout 1 -E "$script" -A "$attr" --eval-only --show-trace --read-write-mode +} + +reportFailure() { + local attr=$1 + shift + local script="import ./default.nix { modules = [ $* ];}" + echo 2>&1 "$ nix-instantiate -E '$script' -A '$attr' --eval-only" + evalConfig "$attr" "$@" || true + ((++fail)) +} + +checkConfigOutput() { + local outputContains=$1 + shift + if evalConfig "$@" 2>/dev/null | grep --silent "$outputContains" ; then + ((++pass)) + else + echo 2>&1 "error: Expected result matching '$outputContains', while evaluating" + reportFailure "$@" + fi +} + +checkConfigError() { + local errorContains=$1 + local err="" + shift + if err="$(evalConfig "$@" 2>&1 >/dev/null)"; then + echo 2>&1 "error: Expected error code, got exit code 0, while evaluating" + reportFailure "$@" + else + if echo "$err" | grep -zP --silent "$errorContains" ; then + ((++pass)) + else + echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" + reportFailure "$@" + fi + fi +} + +# Shorthand meta attribute does not duplicate the config +checkConfigOutput '^"one two"$' config.result ./shorthand-meta.nix + +# Check boolean option. +checkConfigOutput '^false$' config.enable ./declare-enable.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*: true' config.enable ./define-enable.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*' config.enable ./define-enable-throw.nix +checkConfigError 'while evaluating a definition from `.*/define-enable-abort.nix' config.enable ./define-enable-abort.nix +checkConfigError 'while evaluating the error message for definitions for .enable., which is an option that does not exist' config.enable ./define-enable-abort.nix + +checkConfigOutput '^1$' config.bare-submodule.nested ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix +checkConfigOutput '^2$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix +checkConfigOutput '^42$' config.bare-submodule.nested ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix ./declare-bare-submodule-deep-option.nix ./define-bare-submodule-values.nix +checkConfigOutput '^420$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix ./declare-bare-submodule-deep-option.nix ./define-bare-submodule-values.nix +checkConfigOutput '^2$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix ./define-shorthandOnlyDefinesConfig-true.nix +checkConfigError 'The option .bare-submodule.deep. in .*/declare-bare-submodule-deep-option.nix. is already declared in .*/declare-bare-submodule-deep-option-duplicate.nix' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix ./declare-bare-submodule-deep-option-duplicate.nix + +# Check integer types. +# unsigned +checkConfigOutput '^42$' config.value ./declare-int-unsigned-value.nix ./define-value-int-positive.nix +checkConfigError 'A definition for option .* is not of type.*unsigned integer.*. Definition values:\n\s*- In .*: -23' config.value ./declare-int-unsigned-value.nix ./define-value-int-negative.nix +# positive +checkConfigError 'A definition for option .* is not of type.*positive integer.*. Definition values:\n\s*- In .*: 0' config.value ./declare-int-positive-value.nix ./define-value-int-zero.nix +# between +checkConfigOutput '^42$' config.value ./declare-int-between-value.nix ./define-value-int-positive.nix +checkConfigError 'A definition for option .* is not of type.*between.*-21 and 43.*inclusive.*. Definition values:\n\s*- In .*: -23' config.value ./declare-int-between-value.nix ./define-value-int-negative.nix + +# Check either types +# types.either +checkConfigOutput '^42$' config.value ./declare-either.nix ./define-value-int-positive.nix +checkConfigOutput '^"24"$' config.value ./declare-either.nix ./define-value-string.nix +# types.oneOf +checkConfigOutput '^42$' config.value ./declare-oneOf.nix ./define-value-int-positive.nix +checkConfigOutput '^\[ \]$' config.value ./declare-oneOf.nix ./define-value-list.nix +checkConfigOutput '^"24"$' config.value ./declare-oneOf.nix ./define-value-string.nix + +# Check mkForce without submodules. +set -- config.enable ./declare-enable.nix ./define-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigOutput '^false$' "$@" ./define-force-enable.nix +checkConfigOutput '^false$' "$@" ./define-enable-force.nix + +# Check mkForce with option and submodules. +checkConfigError 'attribute .*foo.* .* not found' config.attrsOfSub.foo.enable ./declare-attrsOfSub-any-enable.nix +checkConfigOutput '^false$' config.attrsOfSub.foo.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix +set -- config.attrsOfSub.foo.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigOutput '^false$' "$@" ./define-force-attrsOfSub-foo-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-force-foo-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-foo-force-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-foo-enable-force.nix + +# Check overriding effect of mkForce on submodule definitions. +checkConfigError 'attribute .*bar.* .* not found' config.attrsOfSub.bar.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix +checkConfigOutput '^false$' config.attrsOfSub.bar.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix ./define-attrsOfSub-bar.nix +set -- config.attrsOfSub.bar.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix ./define-attrsOfSub-bar-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigError 'attribute .*bar.* .* not found' "$@" ./define-force-attrsOfSub-foo-enable.nix +checkConfigError 'attribute .*bar.* .* not found' "$@" ./define-attrsOfSub-force-foo-enable.nix +checkConfigOutput '^true$' "$@" ./define-attrsOfSub-foo-force-enable.nix +checkConfigOutput '^true$' "$@" ./define-attrsOfSub-foo-enable-force.nix + +# Check mkIf with submodules. +checkConfigError 'attribute .*foo.* .* not found' config.attrsOfSub.foo.enable ./declare-enable.nix ./declare-attrsOfSub-any-enable.nix +set -- config.attrsOfSub.foo.enable ./declare-enable.nix ./declare-attrsOfSub-any-enable.nix +checkConfigError 'attribute .*foo.* .* not found' "$@" ./define-if-attrsOfSub-foo-enable.nix +checkConfigError 'attribute .*foo.* .* not found' "$@" ./define-attrsOfSub-if-foo-enable.nix +checkConfigError 'attribute .*foo.* .* not found' "$@" ./define-attrsOfSub-foo-if-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-foo-enable-if.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-if-attrsOfSub-foo-enable.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-if-foo-enable.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-if-enable.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-enable-if.nix + +# Check disabledModules with config definitions and option declarations. +set -- config.enable ./define-enable.nix ./declare-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigOutput '^false$' "$@" ./disable-define-enable.nix +checkConfigOutput '^false$' "$@" ./disable-define-enable-string-path.nix +checkConfigError "The option .*enable.* does not exist. Definition values:\n\s*- In .*: true" "$@" ./disable-declare-enable.nix +checkConfigError "attribute .*enable.* in selection path .*config.enable.* not found" "$@" ./disable-define-enable.nix ./disable-declare-enable.nix +checkConfigError "attribute .*enable.* in selection path .*config.enable.* not found" "$@" ./disable-enable-modules.nix + +checkConfigOutput '^true$' 'config.positive.enable' ./disable-module-with-key.nix +checkConfigOutput '^false$' 'config.negative.enable' ./disable-module-with-key.nix +checkConfigError 'Module ..*disable-module-bad-key.nix. contains a disabledModules item that is an attribute set, presumably a module, that does not have a .key. attribute. .*' 'config.enable' ./disable-module-bad-key.nix + +# Not sure if we want to keep supporting module keys that aren't strings, paths or v?key, but we shouldn't remove support accidentally. +checkConfigOutput '^true$' 'config.positive.enable' ./disable-module-with-toString-key.nix +checkConfigOutput '^false$' 'config.negative.enable' ./disable-module-with-toString-key.nix + +# Check _module.args. +set -- config.enable ./declare-enable.nix ./define-enable-with-custom-arg.nix +checkConfigError 'while evaluating the module argument .*custom.* in .*define-enable-with-custom-arg.nix.*:' "$@" +checkConfigOutput '^true$' "$@" ./define-_module-args-custom.nix + +# Check that using _module.args on imports cause infinite recursions, with +# the proper error context. +set -- "$@" ./define-_module-args-custom.nix ./import-custom-arg.nix +checkConfigError 'while evaluating the module argument .*custom.* in .*import-custom-arg.nix.*:' "$@" +checkConfigError 'infinite recursion encountered' "$@" + +# Check _module.check. +set -- config.enable ./declare-enable.nix ./define-enable.nix ./define-attrsOfSub-foo.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*' "$@" +checkConfigOutput '^true$' "$@" ./define-module-check.nix + +# Check coerced value. +set -- +checkConfigOutput '^"42"$' config.value ./declare-coerced-value.nix +checkConfigOutput '^"24"$' config.value ./declare-coerced-value.nix ./define-value-string.nix +checkConfigError 'A definition for option .* is not.*string or signed integer convertible to it.*. Definition values:\n\s*- In .*: \[ \]' config.value ./declare-coerced-value.nix ./define-value-list.nix + +# Check coerced value with unsound coercion +checkConfigOutput '^12$' config.value ./declare-coerced-value-unsound.nix +checkConfigError 'A definition for option .* is not of type .*. Definition values:\n\s*- In .*: "1000"' config.value ./declare-coerced-value-unsound.nix ./define-value-string-bigint.nix +checkConfigError 'toInt: Could not convert .* to int' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix + +# Check mkAliasOptionModule. +checkConfigOutput '^true$' config.enable ./alias-with-priority.nix +checkConfigOutput '^true$' config.enableAlias ./alias-with-priority.nix +checkConfigOutput '^false$' config.enable ./alias-with-priority-can-override.nix +checkConfigOutput '^false$' config.enableAlias ./alias-with-priority-can-override.nix + +# Check mkPackageOption +checkConfigOutput '^"hello"$' config.package.pname ./declare-mkPackageOption.nix +checkConfigError 'The option .undefinedPackage. is used but not defined' config.undefinedPackage ./declare-mkPackageOption.nix +checkConfigOutput '^null$' config.nullablePackage ./declare-mkPackageOption.nix + +# submoduleWith + +## specialArgs should work +checkConfigOutput '^"foo"$' config.submodule.foo ./declare-submoduleWith-special.nix + +## shorthandOnlyDefines config behaves as expected +checkConfigOutput '^true$' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-shorthand.nix +checkConfigError 'is not of type `boolean' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-noshorthand.nix +checkConfigError "You're trying to define a value of type \`bool'\n\s*rather than an attribute set for the option" config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-shorthand.nix +checkConfigOutput '^true$' config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-noshorthand.nix + +## submoduleWith should merge all modules in one swoop +checkConfigOutput '^true$' config.submodule.inner ./declare-submoduleWith-modules.nix +checkConfigOutput '^true$' config.submodule.outer ./declare-submoduleWith-modules.nix +# Should also be able to evaluate the type name (which evaluates freeformType, +# which evaluates all the modules defined by the type) +checkConfigOutput '^"submodule"$' options.submodule.type.description ./declare-submoduleWith-modules.nix + +## submodules can be declared using (evalModules {...}).type +checkConfigOutput '^true$' config.submodule.inner ./declare-submodule-via-evalModules.nix +checkConfigOutput '^true$' config.submodule.outer ./declare-submodule-via-evalModules.nix +# Should also be able to evaluate the type name (which evaluates freeformType, +# which evaluates all the modules defined by the type) +checkConfigOutput '^"submodule"$' options.submodule.type.description ./declare-submodule-via-evalModules.nix + +## Paths should be allowed as values and work as expected +checkConfigOutput '^true$' config.submodule.enable ./declare-submoduleWith-path.nix + +## deferredModule +# default module is merged into nodes.foo +checkConfigOutput '"beta"' config.nodes.foo.settingsDict.c ./deferred-module.nix +# errors from the default module are reported with accurate location +checkConfigError 'In `the-file-that-contains-the-bad-config.nix, via option default'\'': "bogus"' config.nodes.foo.bottom ./deferred-module.nix +checkConfigError '.*lib/tests/modules/deferred-module-error.nix, via option deferred [(]:anon-1:anon-1:anon-1[)] does not look like a module.' config.result ./deferred-module-error.nix + +# Check the file location information is propagated into submodules +checkConfigOutput the-file.nix config.submodule.internalFiles.0 ./submoduleFiles.nix + + +# Check that disabledModules works recursively and correctly +checkConfigOutput '^true$' config.enable ./disable-recursive/main.nix +checkConfigOutput '^true$' config.enable ./disable-recursive/{main.nix,disable-foo.nix} +checkConfigOutput '^true$' config.enable ./disable-recursive/{main.nix,disable-bar.nix} +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*: true' config.enable ./disable-recursive/{main.nix,disable-foo.nix,disable-bar.nix} + +# Check that imports can depend on derivations +checkConfigOutput '^true$' config.enable ./import-from-store.nix + +# Check that configs can be conditional on option existence +checkConfigOutput '^true$' config.enable ./define-option-dependently.nix ./declare-enable.nix ./declare-int-positive-value.nix +checkConfigOutput '^360$' config.value ./define-option-dependently.nix ./declare-enable.nix ./declare-int-positive-value.nix +checkConfigOutput '^7$' config.value ./define-option-dependently.nix ./declare-int-positive-value.nix +checkConfigOutput '^true$' config.set.enable ./define-option-dependently-nested.nix ./declare-enable-nested.nix ./declare-int-positive-value-nested.nix +checkConfigOutput '^360$' config.set.value ./define-option-dependently-nested.nix ./declare-enable-nested.nix ./declare-int-positive-value-nested.nix +checkConfigOutput '^7$' config.set.value ./define-option-dependently-nested.nix ./declare-int-positive-value-nested.nix + +# Check attrsOf and lazyAttrsOf. Only lazyAttrsOf should be lazy, and only +# attrsOf should work with conditional definitions +# In addition, lazyAttrsOf should honor an options emptyValue +checkConfigError "is not lazy" config.isLazy ./declare-attrsOf.nix ./attrsOf-lazy-check.nix +checkConfigOutput '^true$' config.isLazy ./declare-lazyAttrsOf.nix ./attrsOf-lazy-check.nix +checkConfigOutput '^true$' config.conditionalWorks ./declare-attrsOf.nix ./attrsOf-conditional-check.nix +checkConfigOutput '^false$' config.conditionalWorks ./declare-lazyAttrsOf.nix ./attrsOf-conditional-check.nix +checkConfigOutput '^"empty"$' config.value.foo ./declare-lazyAttrsOf.nix ./attrsOf-conditional-check.nix + + +# Even with multiple assignments, a type error should be thrown if any of them aren't valid +checkConfigError 'A definition for option .* is not of type .*' \ + config.value ./declare-int-unsigned-value.nix ./define-value-list.nix ./define-value-int-positive.nix + +## Freeform modules +# Assigning without a declared option should work +checkConfigOutput '^"24"$' config.value ./freeform-attrsOf.nix ./define-value-string.nix +# Shorthand modules interpret `meta` and `class` as config items +checkConfigOutput '^true$' options._module.args.value.result ./freeform-attrsOf.nix ./define-freeform-keywords-shorthand.nix +# No freeform assignments shouldn't make it error +checkConfigOutput '^{ }$' config ./freeform-attrsOf.nix +# but only if the type matches +checkConfigError 'A definition for option .* is not of type .*' config.value ./freeform-attrsOf.nix ./define-value-list.nix +# and properties should be applied +checkConfigOutput '^"yes"$' config.value ./freeform-attrsOf.nix ./define-value-string-properties.nix +# Options should still be declarable, and be able to have a type that doesn't match the freeform type +checkConfigOutput '^false$' config.enable ./freeform-attrsOf.nix ./define-value-string.nix ./declare-enable.nix +checkConfigOutput '^"24"$' config.value ./freeform-attrsOf.nix ./define-value-string.nix ./declare-enable.nix +# and this should work too with nested values +checkConfigOutput '^false$' config.nest.foo ./freeform-attrsOf.nix ./freeform-nested.nix +checkConfigOutput '^"bar"$' config.nest.bar ./freeform-attrsOf.nix ./freeform-nested.nix +# Check whether a declared option can depend on an freeform-typed one +checkConfigOutput '^null$' config.foo ./freeform-attrsOf.nix ./freeform-str-dep-unstr.nix +checkConfigOutput '^"24"$' config.foo ./freeform-attrsOf.nix ./freeform-str-dep-unstr.nix ./define-value-string.nix +# Check whether an freeform-typed value can depend on a declared option, this can only work with lazyAttrsOf +checkConfigError 'infinite recursion encountered' config.foo ./freeform-attrsOf.nix ./freeform-unstr-dep-str.nix +checkConfigError 'The option .* is used but not defined' config.foo ./freeform-lazyAttrsOf.nix ./freeform-unstr-dep-str.nix +checkConfigOutput '^"24"$' config.foo ./freeform-lazyAttrsOf.nix ./freeform-unstr-dep-str.nix ./define-value-string.nix +# submodules in freeformTypes should have their locations annotated +checkConfigOutput '/freeform-submodules.nix"$' config.fooDeclarations.0 ./freeform-submodules.nix +# freeformTypes can get merged using `types.type`, including submodules +checkConfigOutput '^10$' config.free.xxx.foo ./freeform-submodules.nix +checkConfigOutput '^10$' config.free.yyy.bar ./freeform-submodules.nix + +## types.anything +# Check that attribute sets are merged recursively +checkConfigOutput '^null$' config.value.foo ./types-anything/nested-attrs.nix +checkConfigOutput '^null$' config.value.l1.foo ./types-anything/nested-attrs.nix +checkConfigOutput '^null$' config.value.l1.l2.foo ./types-anything/nested-attrs.nix +checkConfigOutput '^null$' config.value.l1.l2.l3.foo ./types-anything/nested-attrs.nix +# Attribute sets that are coercible to strings shouldn't be recursed into +checkConfigOutput '^"foo"$' config.value.outPath ./types-anything/attrs-coercible.nix +# Multiple lists aren't concatenated together +checkConfigError 'The option .* has conflicting definitions' config.value ./types-anything/lists.nix +# Check that all equalizable atoms can be used as long as all definitions are equal +checkConfigOutput '^0$' config.value.int ./types-anything/equal-atoms.nix +checkConfigOutput '^false$' config.value.bool ./types-anything/equal-atoms.nix +checkConfigOutput '^""$' config.value.string ./types-anything/equal-atoms.nix +checkConfigOutput '^/$' config.value.path ./types-anything/equal-atoms.nix +checkConfigOutput '^null$' config.value.null ./types-anything/equal-atoms.nix +checkConfigOutput '^0.1$' config.value.float ./types-anything/equal-atoms.nix +# Functions can't be merged together +checkConfigError "The option .value.multiple-lambdas.<function body>. has conflicting option types" config.applied.multiple-lambdas ./types-anything/functions.nix +checkConfigOutput '^<LAMBDA>$' config.value.single-lambda ./types-anything/functions.nix +checkConfigOutput '^null$' config.applied.merging-lambdas.x ./types-anything/functions.nix +checkConfigOutput '^null$' config.applied.merging-lambdas.y ./types-anything/functions.nix +# Check that all mk* modifiers are applied +checkConfigError 'attribute .* not found' config.value.mkiffalse ./types-anything/mk-mods.nix +checkConfigOutput '^{ }$' config.value.mkiftrue ./types-anything/mk-mods.nix +checkConfigOutput '^1$' config.value.mkdefault ./types-anything/mk-mods.nix +checkConfigOutput '^{ }$' config.value.mkmerge ./types-anything/mk-mods.nix +checkConfigOutput '^true$' config.value.mkbefore ./types-anything/mk-mods.nix +checkConfigOutput '^1$' config.value.nested.foo ./types-anything/mk-mods.nix +checkConfigOutput '^"baz"$' config.value.nested.bar.baz ./types-anything/mk-mods.nix + +## types.functionTo +checkConfigOutput '^"input is input"$' config.result ./functionTo/trivial.nix +checkConfigOutput '^"a b"$' config.result ./functionTo/merging-list.nix +checkConfigError 'A definition for option .fun.<function body>. is not of type .string.. Definition values:\n\s*- In .*wrong-type.nix' config.result ./functionTo/wrong-type.nix +checkConfigOutput '^"b a"$' config.result ./functionTo/list-order.nix +checkConfigOutput '^"a c"$' config.result ./functionTo/merging-attrs.nix +checkConfigOutput '^"a bee"$' config.result ./functionTo/submodule-options.nix +checkConfigOutput '^"fun.<function body>.a fun.<function body>.b"$' config.optionsResult ./functionTo/submodule-options.nix + +# moduleType +checkConfigOutput '^"a b"$' config.resultFoo ./declare-variants.nix ./define-variant.nix +checkConfigOutput '^"a b y z"$' config.resultFooBar ./declare-variants.nix ./define-variant.nix +checkConfigOutput '^"a b c"$' config.resultFooFoo ./declare-variants.nix ./define-variant.nix + +## emptyValue's +checkConfigOutput "[ ]" config.list.a ./emptyValues.nix +checkConfigOutput "{ }" config.attrs.a ./emptyValues.nix +checkConfigOutput "null" config.null.a ./emptyValues.nix +checkConfigOutput "{ }" config.submodule.a ./emptyValues.nix +# These types don't have empty values +checkConfigError 'The option .int.a. is used but not defined' config.int.a ./emptyValues.nix +checkConfigError 'The option .nonEmptyList.a. is used but not defined' config.nonEmptyList.a ./emptyValues.nix + +## types.raw +checkConfigOutput "{ foo = <CODE>; }" config.unprocessedNesting ./raw.nix +checkConfigOutput "10" config.processedToplevel ./raw.nix +checkConfigError "The option .multiple. is defined multiple times" config.multiple ./raw.nix +checkConfigOutput "bar" config.priorities ./raw.nix + +## Option collision +checkConfigError \ + 'The option .set. in module .*/declare-set.nix. would be a parent of the following options, but its type .attribute set of signed integer. does not support nested options.\n\s*- option[(]s[)] with prefix .set.enable. in module .*/declare-enable-nested.nix.' \ + config.set \ + ./declare-set.nix ./declare-enable-nested.nix + +# Test that types.optionType merges types correctly +checkConfigOutput '^10$' config.theOption.int ./optionTypeMerging.nix +checkConfigOutput '^"hello"$' config.theOption.str ./optionTypeMerging.nix + +# Test that types.optionType correctly annotates option locations +checkConfigError 'The option .theOption.nested. in .other.nix. is already declared in .optionTypeFile.nix.' config.theOption.nested ./optionTypeFile.nix + +# Test that types.optionType leaves types untouched as long as they don't need to be merged +checkConfigOutput 'ok' config.freeformItems.foo.bar ./adhoc-freeformType-survives-type-merge.nix + +# Anonymous submodules don't get nixed by import resolution/deduplication +# because of an `extendModules` bug, issue 168767. +checkConfigOutput '^1$' config.sub.specialisation.value ./extendModules-168767-imports.nix + +# Class checks, evalModules +checkConfigOutput '^{ }$' config.ok.config ./class-check.nix +checkConfigOutput '"nixos"' config.ok.class ./class-check.nix +checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.fail.config ./class-check.nix +checkConfigError 'The module foo.nix#darwinModules.default was imported into nixos instead of darwin.' config.fail-anon.config ./class-check.nix + +# Class checks, submoduleWith +checkConfigOutput '^{ }$' config.sub.nixosOk ./class-check.nix +checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.sub.nixosFail.config ./class-check.nix + +# submoduleWith type merge with different class +checkConfigError 'A submoduleWith option is declared multiple times with conflicting class values "darwin" and "nixos".' config.sub.mergeFail.config ./class-check.nix + +# _type check +checkConfigError 'Could not load a value as a module, because it is of type "flake", in file .*/module-imports-_type-check.nix' config.ok.config ./module-imports-_type-check.nix +checkConfigOutput '^true$' "$@" config.enable ./declare-enable.nix ./define-enable-with-top-level-mkIf.nix +checkConfigError 'Could not load a value as a module, because it is of type "configuration", in file .*/import-configuration.nix.*please only import the modules that make up the configuration.*' config ./import-configuration.nix + +# doRename works when `warnings` does not exist. +checkConfigOutput '^1234$' config.c.d.e ./doRename-basic.nix +# doRename adds a warning. +checkConfigOutput '^"The option `a\.b. defined in `.*/doRename-warnings\.nix. has been renamed to `c\.d\.e.\."$' \ + config.result \ + ./doRename-warnings.nix + +# Anonymous modules get deduplicated by key +checkConfigOutput '^"pear"$' config.once.raw ./merge-module-with-key.nix +checkConfigOutput '^"pear\\npear"$' config.twice.raw ./merge-module-with-key.nix + +cat <<EOF +====== module tests ====== +$pass Pass +$fail Fail +EOF + +if [ "$fail" -ne 0 ]; then + exit 1 +fi +exit 0 diff --git a/tests/modules/adhoc-freeformType-survives-type-merge.nix b/tests/modules/adhoc-freeformType-survives-type-merge.nix new file mode 100644 index 000000000..3cefb543c --- /dev/null +++ b/tests/modules/adhoc-freeformType-survives-type-merge.nix @@ -0,0 +1,14 @@ +{ lib, ... }: { + options.dummy = lib.mkOption { type = lib.types.anything; default = {}; }; + freeformType = + let + a = lib.types.attrsOf (lib.types.submodule { options.bar = lib.mkOption { }; }); + in + # modifying types like this breaks type merging. + # This test makes sure that type merging is not performed when only a single declaration exists. + # Don't modify types in practice! + a // { + merge = loc: defs: { freeformItems = a.merge loc defs; }; + }; + config.foo.bar = "ok"; +} diff --git a/tests/modules/alias-with-priority-can-override.nix b/tests/modules/alias-with-priority-can-override.nix new file mode 100644 index 000000000..9a18c9d9f --- /dev/null +++ b/tests/modules/alias-with-priority-can-override.nix @@ -0,0 +1,55 @@ +# This is a test to show that mkAliasOptionModule sets the priority correctly +# for aliased options. +# +# This test shows that an alias with a high priority is able to override +# a non-aliased option. + +{ config, lib, ... }: + +with lib; + +{ + options = { + # A simple boolean option that can be enabled or disabled. + enable = lib.mkOption { + type = types.nullOr types.bool; + default = null; + example = true; + description = '' + Some descriptive text + ''; + }; + + # mkAliasOptionModule sets warnings, so this has to be defined. + warnings = mkOption { + internal = true; + default = []; + type = types.listOf types.str; + example = [ "The `foo' service is deprecated and will go away soon!" ]; + description = '' + This option allows modules to show warnings to users during + the evaluation of the system configuration. + ''; + }; + }; + + imports = [ + # Create an alias for the "enable" option. + (mkAliasOptionModule [ "enableAlias" ] [ "enable" ]) + + # Disable the aliased option with a high priority so it + # should override the next import. + ( { config, lib, ... }: + { + enableAlias = lib.mkForce false; + } + ) + + # Enable the normal (non-aliased) option. + ( { config, lib, ... }: + { + enable = true; + } + ) + ]; +} diff --git a/tests/modules/alias-with-priority.nix b/tests/modules/alias-with-priority.nix new file mode 100644 index 000000000..a35a06fc6 --- /dev/null +++ b/tests/modules/alias-with-priority.nix @@ -0,0 +1,55 @@ +# This is a test to show that mkAliasOptionModule sets the priority correctly +# for aliased options. +# +# This test shows that an alias with a low priority is able to be overridden +# with a non-aliased option. + +{ config, lib, ... }: + +with lib; + +{ + options = { + # A simple boolean option that can be enabled or disabled. + enable = lib.mkOption { + type = types.nullOr types.bool; + default = null; + example = true; + description = '' + Some descriptive text + ''; + }; + + # mkAliasOptionModule sets warnings, so this has to be defined. + warnings = mkOption { + internal = true; + default = []; + type = types.listOf types.str; + example = [ "The `foo' service is deprecated and will go away soon!" ]; + description = '' + This option allows modules to show warnings to users during + the evaluation of the system configuration. + ''; + }; + }; + + imports = [ + # Create an alias for the "enable" option. + (mkAliasOptionModule [ "enableAlias" ] [ "enable" ]) + + # Disable the aliased option, but with a default (low) priority so it + # should be able to be overridden by the next import. + ( { config, lib, ... }: + { + enableAlias = lib.mkDefault false; + } + ) + + # Enable the normal (non-aliased) option. + ( { config, lib, ... }: + { + enable = true; + } + ) + ]; +} diff --git a/tests/modules/attrsOf-conditional-check.nix b/tests/modules/attrsOf-conditional-check.nix new file mode 100644 index 000000000..0f00ebca1 --- /dev/null +++ b/tests/modules/attrsOf-conditional-check.nix @@ -0,0 +1,7 @@ +{ lib, config, ... }: { + options.conditionalWorks = lib.mkOption { + default = ! config.value ? foo; + }; + + config.value.foo = lib.mkIf false "should not be defined"; +} diff --git a/tests/modules/attrsOf-lazy-check.nix b/tests/modules/attrsOf-lazy-check.nix new file mode 100644 index 000000000..ec5b418b1 --- /dev/null +++ b/tests/modules/attrsOf-lazy-check.nix @@ -0,0 +1,7 @@ +{ lib, config, ... }: { + options.isLazy = lib.mkOption { + default = ! config.value ? foo; + }; + + config.value.bar = throw "is not lazy"; +} diff --git a/tests/modules/class-check.nix b/tests/modules/class-check.nix new file mode 100644 index 000000000..293fd4abd --- /dev/null +++ b/tests/modules/class-check.nix @@ -0,0 +1,76 @@ +{ lib, ... }: { + options = { + sub = { + nixosOk = lib.mkOption { + type = lib.types.submoduleWith { + class = "nixos"; + modules = [ ]; + }; + }; + # Same but will have bad definition + nixosFail = lib.mkOption { + type = lib.types.submoduleWith { + class = "nixos"; + modules = [ ]; + }; + }; + + mergeFail = lib.mkOption { + type = lib.types.submoduleWith { + class = "nixos"; + modules = [ ]; + }; + default = { }; + }; + }; + }; + imports = [ + { + options = { + sub = { + mergeFail = lib.mkOption { + type = lib.types.submoduleWith { + class = "darwin"; + modules = [ ]; + }; + }; + }; + }; + } + ]; + config = { + _module.freeformType = lib.types.anything; + ok = + lib.evalModules { + class = "nixos"; + modules = [ + ./module-class-is-nixos.nix + ]; + }; + + fail = + lib.evalModules { + class = "nixos"; + modules = [ + ./module-class-is-nixos.nix + ./module-class-is-darwin.nix + ]; + }; + + fail-anon = + lib.evalModules { + class = "nixos"; + modules = [ + ./module-class-is-nixos.nix + { _file = "foo.nix#darwinModules.default"; + _class = "darwin"; + config = {}; + imports = []; + } + ]; + }; + + sub.nixosOk = { _class = "nixos"; }; + sub.nixosFail = { imports = [ ./module-class-is-darwin.nix ]; }; + }; +} diff --git a/tests/modules/declare-attrsOf.nix b/tests/modules/declare-attrsOf.nix new file mode 100644 index 000000000..d19964064 --- /dev/null +++ b/tests/modules/declare-attrsOf.nix @@ -0,0 +1,13 @@ +{ lib, ... }: +let + deathtrapArgs = lib.mapAttrs + (k: _: throw "The module system is too strict, accessing an unused option's ${k} mkOption-attribute.") + (lib.functionArgs lib.mkOption); +in +{ + options.value = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = {}; + }; + options.testing-laziness-so-don't-read-me = lib.mkOption deathtrapArgs; +} diff --git a/tests/modules/declare-attrsOfSub-any-enable.nix b/tests/modules/declare-attrsOfSub-any-enable.nix new file mode 100644 index 000000000..986d07227 --- /dev/null +++ b/tests/modules/declare-attrsOfSub-any-enable.nix @@ -0,0 +1,29 @@ +{ lib, ... }: + +let + submod = { ... }: { + options = { + enable = lib.mkOption { + default = false; + example = true; + type = lib.types.bool; + description = '' + Some descriptive text + ''; + }; + }; + }; +in + +{ + options = { + attrsOfSub = lib.mkOption { + default = {}; + example = {}; + type = lib.types.attrsOf (lib.types.submodule [ submod ]); + description = '' + Some descriptive text + ''; + }; + }; +} diff --git a/tests/modules/declare-bare-submodule-deep-option-duplicate.nix b/tests/modules/declare-bare-submodule-deep-option-duplicate.nix new file mode 100644 index 000000000..06ad1f6e0 --- /dev/null +++ b/tests/modules/declare-bare-submodule-deep-option-duplicate.nix @@ -0,0 +1,10 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; +in +{ + options.bare-submodule.deep = mkOption { + type = types.int; + default = 2; + }; +} diff --git a/tests/modules/declare-bare-submodule-deep-option.nix b/tests/modules/declare-bare-submodule-deep-option.nix new file mode 100644 index 000000000..06ad1f6e0 --- /dev/null +++ b/tests/modules/declare-bare-submodule-deep-option.nix @@ -0,0 +1,10 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; +in +{ + options.bare-submodule.deep = mkOption { + type = types.int; + default = 2; + }; +} diff --git a/tests/modules/declare-bare-submodule-nested-option.nix b/tests/modules/declare-bare-submodule-nested-option.nix new file mode 100644 index 000000000..da125c84b --- /dev/null +++ b/tests/modules/declare-bare-submodule-nested-option.nix @@ -0,0 +1,19 @@ +{ config, lib, ... }: +let + inherit (lib) mkOption types; +in +{ + options.bare-submodule = mkOption { + type = types.submoduleWith { + shorthandOnlyDefinesConfig = config.shorthandOnlyDefinesConfig; + modules = [ + { + options.nested = mkOption { + type = types.int; + default = 1; + }; + } + ]; + }; + }; +} diff --git a/tests/modules/declare-bare-submodule.nix b/tests/modules/declare-bare-submodule.nix new file mode 100644 index 000000000..5402f4ff5 --- /dev/null +++ b/tests/modules/declare-bare-submodule.nix @@ -0,0 +1,18 @@ +{ config, lib, ... }: +let + inherit (lib) mkOption types; +in +{ + options.bare-submodule = mkOption { + type = types.submoduleWith { + modules = [ ]; + shorthandOnlyDefinesConfig = config.shorthandOnlyDefinesConfig; + }; + default = {}; + }; + + # config-dependent options: won't recommend, but useful for making this test parameterized + options.shorthandOnlyDefinesConfig = mkOption { + default = false; + }; +} diff --git a/tests/modules/declare-coerced-value-unsound.nix b/tests/modules/declare-coerced-value-unsound.nix new file mode 100644 index 000000000..7a017f24e --- /dev/null +++ b/tests/modules/declare-coerced-value-unsound.nix @@ -0,0 +1,10 @@ +{ lib, ... }: + +{ + options = { + value = lib.mkOption { + default = "12"; + type = lib.types.coercedTo lib.types.str lib.toInt lib.types.ints.s8; + }; + }; +} diff --git a/tests/modules/declare-coerced-value.nix b/tests/modules/declare-coerced-value.nix new file mode 100644 index 000000000..76b12ad53 --- /dev/null +++ b/tests/modules/declare-coerced-value.nix @@ -0,0 +1,10 @@ +{ lib, ... }: + +{ + options = { + value = lib.mkOption { + default = 42; + type = lib.types.coercedTo lib.types.int builtins.toString lib.types.str; + }; + }; +} diff --git a/tests/modules/declare-either.nix b/tests/modules/declare-either.nix new file mode 100644 index 000000000..5a0fa978a --- /dev/null +++ b/tests/modules/declare-either.nix @@ -0,0 +1,5 @@ +{ lib, ... }: { + options.value = lib.mkOption { + type = lib.types.either lib.types.int lib.types.str; + }; +} diff --git a/tests/modules/declare-enable-nested.nix b/tests/modules/declare-enable-nested.nix new file mode 100644 index 000000000..c8da8273c --- /dev/null +++ b/tests/modules/declare-enable-nested.nix @@ -0,0 +1,14 @@ +{ lib, ... }: + +{ + options.set = { + enable = lib.mkOption { + default = false; + example = true; + type = lib.types.bool; + description = '' + Some descriptive text + ''; + }; + }; +} diff --git a/tests/modules/declare-enable.nix b/tests/modules/declare-enable.nix new file mode 100644 index 000000000..ebee243c7 --- /dev/null +++ b/tests/modules/declare-enable.nix @@ -0,0 +1,14 @@ +{ lib, ... }: + +{ + options = { + enable = lib.mkOption { + default = false; + example = true; + type = lib.types.bool; + description = '' + Some descriptive text + ''; + }; + }; +} diff --git a/tests/modules/declare-int-between-value.nix b/tests/modules/declare-int-between-value.nix new file mode 100644 index 000000000..8b2624cc5 --- /dev/null +++ b/tests/modules/declare-int-between-value.nix @@ -0,0 +1,9 @@ +{ lib, ... }: + +{ + options = { + value = lib.mkOption { + type = lib.types.ints.between (-21) 43; + }; + }; +} diff --git a/tests/modules/declare-int-positive-value-nested.nix b/tests/modules/declare-int-positive-value-nested.nix new file mode 100644 index 000000000..72d2fb89f --- /dev/null +++ b/tests/modules/declare-int-positive-value-nested.nix @@ -0,0 +1,9 @@ +{ lib, ... }: + +{ + options.set = { + value = lib.mkOption { + type = lib.types.ints.positive; + }; + }; +} diff --git a/tests/modules/declare-int-positive-value.nix b/tests/modules/declare-int-positive-value.nix new file mode 100644 index 000000000..6e48c6ac8 --- /dev/null +++ b/tests/modules/declare-int-positive-value.nix @@ -0,0 +1,9 @@ +{ lib, ... }: + +{ + options = { + value = lib.mkOption { + type = lib.types.ints.positive; + }; + }; +} diff --git a/tests/modules/declare-int-unsigned-value.nix b/tests/modules/declare-int-unsigned-value.nix new file mode 100644 index 000000000..05d0eff01 --- /dev/null +++ b/tests/modules/declare-int-unsigned-value.nix @@ -0,0 +1,9 @@ +{ lib, ... }: + +{ + options = { + value = lib.mkOption { + type = lib.types.ints.unsigned; + }; + }; +} diff --git a/tests/modules/declare-lazyAttrsOf.nix b/tests/modules/declare-lazyAttrsOf.nix new file mode 100644 index 000000000..1d9fec25f --- /dev/null +++ b/tests/modules/declare-lazyAttrsOf.nix @@ -0,0 +1,6 @@ +{ lib, ... }: { + options.value = lib.mkOption { + type = lib.types.lazyAttrsOf (lib.types.str // { emptyValue.value = "empty"; }); + default = {}; + }; +} diff --git a/tests/modules/declare-mkPackageOption.nix b/tests/modules/declare-mkPackageOption.nix new file mode 100644 index 000000000..640b19a7b --- /dev/null +++ b/tests/modules/declare-mkPackageOption.nix @@ -0,0 +1,19 @@ +{ lib, ... }: let + pkgs.hello = { + type = "derivation"; + pname = "hello"; + }; +in { + options = { + package = lib.mkPackageOption pkgs "hello" { }; + + undefinedPackage = lib.mkPackageOption pkgs "hello" { + default = null; + }; + + nullablePackage = lib.mkPackageOption pkgs "hello" { + nullable = true; + default = null; + }; + }; +} diff --git a/tests/modules/declare-oneOf.nix b/tests/modules/declare-oneOf.nix new file mode 100644 index 000000000..df092a14f --- /dev/null +++ b/tests/modules/declare-oneOf.nix @@ -0,0 +1,9 @@ +{ lib, ... }: { + options.value = lib.mkOption { + type = lib.types.oneOf [ + lib.types.int + (lib.types.listOf lib.types.int) + lib.types.str + ]; + }; +} diff --git a/tests/modules/declare-set.nix b/tests/modules/declare-set.nix new file mode 100644 index 000000000..853418531 --- /dev/null +++ b/tests/modules/declare-set.nix @@ -0,0 +1,12 @@ +{ lib, ... }: + +{ + options.set = lib.mkOption { + default = { }; + example = { a = 1; }; + type = lib.types.attrsOf lib.types.int; + description = '' + Some descriptive text + ''; + }; +} diff --git a/tests/modules/declare-submodule-via-evalModules.nix b/tests/modules/declare-submodule-via-evalModules.nix new file mode 100644 index 000000000..2841c64a0 --- /dev/null +++ b/tests/modules/declare-submodule-via-evalModules.nix @@ -0,0 +1,28 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + inherit (lib.evalModules { + modules = [ + { + options.inner = lib.mkOption { + type = lib.types.bool; + default = false; + }; + } + ]; + }) type; + default = {}; + }; + + config.submodule = lib.mkMerge [ + ({ lib, ... }: { + options.outer = lib.mkOption { + type = lib.types.bool; + default = false; + }; + }) + { + inner = true; + outer = true; + } + ]; +} diff --git a/tests/modules/declare-submoduleWith-modules.nix b/tests/modules/declare-submoduleWith-modules.nix new file mode 100644 index 000000000..a8b82d176 --- /dev/null +++ b/tests/modules/declare-submoduleWith-modules.nix @@ -0,0 +1,28 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + type = lib.types.submoduleWith { + modules = [ + { + options.inner = lib.mkOption { + type = lib.types.bool; + default = false; + }; + } + ]; + }; + default = {}; + }; + + config.submodule = lib.mkMerge [ + ({ lib, ... }: { + options.outer = lib.mkOption { + type = lib.types.bool; + default = false; + }; + }) + { + inner = true; + outer = true; + } + ]; +} diff --git a/tests/modules/declare-submoduleWith-noshorthand.nix b/tests/modules/declare-submoduleWith-noshorthand.nix new file mode 100644 index 000000000..af3b4ba47 --- /dev/null +++ b/tests/modules/declare-submoduleWith-noshorthand.nix @@ -0,0 +1,13 @@ +{ lib, ... }: let + sub.options.config = lib.mkOption { + type = lib.types.bool; + default = false; + }; +in { + options.submodule = lib.mkOption { + type = lib.types.submoduleWith { + modules = [ sub ]; + }; + default = {}; + }; +} diff --git a/tests/modules/declare-submoduleWith-path.nix b/tests/modules/declare-submoduleWith-path.nix new file mode 100644 index 000000000..477647f32 --- /dev/null +++ b/tests/modules/declare-submoduleWith-path.nix @@ -0,0 +1,12 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + type = lib.types.submoduleWith { + modules = [ + ./declare-enable.nix + ]; + }; + default = {}; + }; + + config.submodule = ./define-enable.nix; +} diff --git a/tests/modules/declare-submoduleWith-shorthand.nix b/tests/modules/declare-submoduleWith-shorthand.nix new file mode 100644 index 000000000..63ac16293 --- /dev/null +++ b/tests/modules/declare-submoduleWith-shorthand.nix @@ -0,0 +1,14 @@ +{ lib, ... }: let + sub.options.config = lib.mkOption { + type = lib.types.bool; + default = false; + }; +in { + options.submodule = lib.mkOption { + type = lib.types.submoduleWith { + modules = [ sub ]; + shorthandOnlyDefinesConfig = true; + }; + default = {}; + }; +} diff --git a/tests/modules/declare-submoduleWith-special.nix b/tests/modules/declare-submoduleWith-special.nix new file mode 100644 index 000000000..6b15c5bde --- /dev/null +++ b/tests/modules/declare-submoduleWith-special.nix @@ -0,0 +1,17 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + type = lib.types.submoduleWith { + modules = [ + ({ lib, ... }: { + options.foo = lib.mkOption { + default = lib.foo; + }; + }) + ]; + specialArgs.lib = lib // { + foo = "foo"; + }; + }; + default = {}; + }; +} diff --git a/tests/modules/declare-variants.nix b/tests/modules/declare-variants.nix new file mode 100644 index 000000000..3ed6fa689 --- /dev/null +++ b/tests/modules/declare-variants.nix @@ -0,0 +1,9 @@ +{ lib, moduleType, ... }: +let inherit (lib) mkOption types; +in +{ + options.variants = mkOption { + type = types.lazyAttrsOf moduleType; + default = {}; + }; +} diff --git a/tests/modules/default.nix b/tests/modules/default.nix new file mode 100644 index 000000000..5b0947104 --- /dev/null +++ b/tests/modules/default.nix @@ -0,0 +1,8 @@ +{ lib ? import ../.., modules ? [] }: + +{ + inherit (lib.evalModules { + inherit modules; + specialArgs.modulesPath = ./.; + }) config options; +} diff --git a/tests/modules/deferred-module-error.nix b/tests/modules/deferred-module-error.nix new file mode 100644 index 000000000..d48ae092e --- /dev/null +++ b/tests/modules/deferred-module-error.nix @@ -0,0 +1,20 @@ +{ config, lib, ... }: +let + inherit (lib) types mkOption setDefaultModuleLocation evalModules; + inherit (types) deferredModule lazyAttrsOf submodule str raw enum; +in +{ + options = { + deferred = mkOption { + type = deferredModule; + }; + result = mkOption { + default = (evalModules { modules = [ config.deferred ]; }).config.result; + }; + }; + config = { + deferred = { ... }: + # this should be an attrset, so this fails + true; + }; +} diff --git a/tests/modules/deferred-module.nix b/tests/modules/deferred-module.nix new file mode 100644 index 000000000..d03c60b02 --- /dev/null +++ b/tests/modules/deferred-module.nix @@ -0,0 +1,58 @@ +{ lib, ... }: +let + inherit (lib) types mkOption setDefaultModuleLocation; + inherit (types) deferredModule lazyAttrsOf submodule str raw enum; +in +{ + imports = [ + # generic module, declaring submodules: + # - nodes.<name> + # - default + # where all nodes include the default + ({ config, ... }: { + _file = "generic.nix"; + options.nodes = mkOption { + type = lazyAttrsOf (submodule { imports = [ config.default ]; }); + default = {}; + }; + options.default = mkOption { + type = deferredModule; + default = { }; + description = '' + Module that is included in all nodes. + ''; + }; + }) + + { + _file = "default-1.nix"; + default = { config, ... }: { + options.settingsDict = lib.mkOption { type = lazyAttrsOf str; default = {}; }; + options.bottom = lib.mkOption { type = enum []; }; + }; + } + + { + _file = "default-a-is-b.nix"; + default = ./define-settingsDict-a-is-b.nix; + } + + { + _file = "nodes-foo.nix"; + nodes.foo.settingsDict.b = "beta"; + } + + { + _file = "the-file-that-contains-the-bad-config.nix"; + default.bottom = "bogus"; + } + + { + _file = "nodes-foo-c-is-a.nix"; + nodes.foo = { config, ... }: { + settingsDict.c = config.settingsDict.a; + }; + } + + ]; +} diff --git a/tests/modules/define-_module-args-custom.nix b/tests/modules/define-_module-args-custom.nix new file mode 100644 index 000000000..e565fd215 --- /dev/null +++ b/tests/modules/define-_module-args-custom.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + config = { + _module.args.custom = true; + }; +} diff --git a/tests/modules/define-attrsOfSub-bar-enable.nix b/tests/modules/define-attrsOfSub-bar-enable.nix new file mode 100644 index 000000000..99c55d8b3 --- /dev/null +++ b/tests/modules/define-attrsOfSub-bar-enable.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.bar.enable = true; +} diff --git a/tests/modules/define-attrsOfSub-bar.nix b/tests/modules/define-attrsOfSub-bar.nix new file mode 100644 index 000000000..2a33068a5 --- /dev/null +++ b/tests/modules/define-attrsOfSub-bar.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.bar = {}; +} diff --git a/tests/modules/define-attrsOfSub-foo-enable-force.nix b/tests/modules/define-attrsOfSub-foo-enable-force.nix new file mode 100644 index 000000000..c9ee36446 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-enable-force.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + attrsOfSub.foo.enable = lib.mkForce false; +} diff --git a/tests/modules/define-attrsOfSub-foo-enable-if.nix b/tests/modules/define-attrsOfSub-foo-enable-if.nix new file mode 100644 index 000000000..0b3baddb5 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-enable-if.nix @@ -0,0 +1,5 @@ +{ config, lib, ... }: + +{ + attrsOfSub.foo.enable = lib.mkIf config.enable true; +} diff --git a/tests/modules/define-attrsOfSub-foo-enable.nix b/tests/modules/define-attrsOfSub-foo-enable.nix new file mode 100644 index 000000000..39cd63cef --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-enable.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.foo.enable = true; +} diff --git a/tests/modules/define-attrsOfSub-foo-force-enable.nix b/tests/modules/define-attrsOfSub-foo-force-enable.nix new file mode 100644 index 000000000..009da7c77 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-force-enable.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + attrsOfSub.foo = lib.mkForce { + enable = false; + }; +} diff --git a/tests/modules/define-attrsOfSub-foo-if-enable.nix b/tests/modules/define-attrsOfSub-foo-if-enable.nix new file mode 100644 index 000000000..93702dfa8 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-if-enable.nix @@ -0,0 +1,7 @@ +{ config, lib, ... }: + +{ + attrsOfSub.foo = lib.mkIf config.enable { + enable = true; + }; +} diff --git a/tests/modules/define-attrsOfSub-foo.nix b/tests/modules/define-attrsOfSub-foo.nix new file mode 100644 index 000000000..e6bb531de --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.foo = {}; +} diff --git a/tests/modules/define-attrsOfSub-force-foo-enable.nix b/tests/modules/define-attrsOfSub-force-foo-enable.nix new file mode 100644 index 000000000..5c02dd343 --- /dev/null +++ b/tests/modules/define-attrsOfSub-force-foo-enable.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + attrsOfSub = lib.mkForce { + foo.enable = false; + }; +} diff --git a/tests/modules/define-attrsOfSub-if-foo-enable.nix b/tests/modules/define-attrsOfSub-if-foo-enable.nix new file mode 100644 index 000000000..a3fe6051d --- /dev/null +++ b/tests/modules/define-attrsOfSub-if-foo-enable.nix @@ -0,0 +1,7 @@ +{ config, lib, ... }: + +{ + attrsOfSub = lib.mkIf config.enable { + foo.enable = true; + }; +} diff --git a/tests/modules/define-bare-submodule-values.nix b/tests/modules/define-bare-submodule-values.nix new file mode 100644 index 000000000..00ede929e --- /dev/null +++ b/tests/modules/define-bare-submodule-values.nix @@ -0,0 +1,4 @@ +{ + bare-submodule.nested = 42; + bare-submodule.deep = 420; +} diff --git a/tests/modules/define-enable-abort.nix b/tests/modules/define-enable-abort.nix new file mode 100644 index 000000000..85b58a567 --- /dev/null +++ b/tests/modules/define-enable-abort.nix @@ -0,0 +1,3 @@ +{ + config.enable = abort "oops"; +} diff --git a/tests/modules/define-enable-force.nix b/tests/modules/define-enable-force.nix new file mode 100644 index 000000000..f4990a328 --- /dev/null +++ b/tests/modules/define-enable-force.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + enable = lib.mkForce false; +} diff --git a/tests/modules/define-enable-throw.nix b/tests/modules/define-enable-throw.nix new file mode 100644 index 000000000..16a59b781 --- /dev/null +++ b/tests/modules/define-enable-throw.nix @@ -0,0 +1,3 @@ +{ + config.enable = throw "oops"; +} diff --git a/tests/modules/define-enable-with-custom-arg.nix b/tests/modules/define-enable-with-custom-arg.nix new file mode 100644 index 000000000..7da74671d --- /dev/null +++ b/tests/modules/define-enable-with-custom-arg.nix @@ -0,0 +1,7 @@ +{ lib, custom, ... }: + +{ + config = { + enable = custom; + }; +} diff --git a/tests/modules/define-enable-with-top-level-mkIf.nix b/tests/modules/define-enable-with-top-level-mkIf.nix new file mode 100644 index 000000000..4909c16d8 --- /dev/null +++ b/tests/modules/define-enable-with-top-level-mkIf.nix @@ -0,0 +1,5 @@ +{ lib, ... }: +# I think this might occur more realistically in a submodule +{ + imports = [ (lib.mkIf true { enable = true; }) ]; +} diff --git a/tests/modules/define-enable.nix b/tests/modules/define-enable.nix new file mode 100644 index 000000000..7dc26010a --- /dev/null +++ b/tests/modules/define-enable.nix @@ -0,0 +1,3 @@ +{ + enable = true; +} diff --git a/tests/modules/define-force-attrsOfSub-foo-enable.nix b/tests/modules/define-force-attrsOfSub-foo-enable.nix new file mode 100644 index 000000000..dafb2360e --- /dev/null +++ b/tests/modules/define-force-attrsOfSub-foo-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +lib.mkForce { + attrsOfSub.foo.enable = false; +} diff --git a/tests/modules/define-force-enable.nix b/tests/modules/define-force-enable.nix new file mode 100644 index 000000000..978caa2a8 --- /dev/null +++ b/tests/modules/define-force-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +lib.mkForce { + enable = false; +} diff --git a/tests/modules/define-freeform-keywords-shorthand.nix b/tests/modules/define-freeform-keywords-shorthand.nix new file mode 100644 index 000000000..8de1ec6a7 --- /dev/null +++ b/tests/modules/define-freeform-keywords-shorthand.nix @@ -0,0 +1,15 @@ +{ config, ... }: { + class = { "just" = "data"; }; + a = "one"; + b = "two"; + meta = "meta"; + + _module.args.result = + let r = builtins.removeAttrs config [ "_module" ]; + in builtins.trace (builtins.deepSeq r r) (r == { + a = "one"; + b = "two"; + class = { "just" = "data"; }; + meta = "meta"; + }); +} diff --git a/tests/modules/define-if-attrsOfSub-foo-enable.nix b/tests/modules/define-if-attrsOfSub-foo-enable.nix new file mode 100644 index 000000000..6a8e32e80 --- /dev/null +++ b/tests/modules/define-if-attrsOfSub-foo-enable.nix @@ -0,0 +1,5 @@ +{ config, lib, ... }: + +lib.mkIf config.enable { + attrsOfSub.foo.enable = true; +} diff --git a/tests/modules/define-module-check.nix b/tests/modules/define-module-check.nix new file mode 100644 index 000000000..5a0707c97 --- /dev/null +++ b/tests/modules/define-module-check.nix @@ -0,0 +1,3 @@ +{ + _module.check = false; +} diff --git a/tests/modules/define-option-dependently-nested.nix b/tests/modules/define-option-dependently-nested.nix new file mode 100644 index 000000000..69ee42555 --- /dev/null +++ b/tests/modules/define-option-dependently-nested.nix @@ -0,0 +1,16 @@ +{ lib, options, ... }: + +# Some modules may be distributed separately and need to adapt to other modules +# that are distributed and versioned separately. +{ + + # Always defined, but the value depends on the presence of an option. + config.set = { + value = if options ? set.enable then 360 else 7; + } + # Only define if possible. + // lib.optionalAttrs (options ? set.enable) { + enable = true; + }; + +} diff --git a/tests/modules/define-option-dependently.nix b/tests/modules/define-option-dependently.nix new file mode 100644 index 000000000..ad85f99a9 --- /dev/null +++ b/tests/modules/define-option-dependently.nix @@ -0,0 +1,16 @@ +{ lib, options, ... }: + +# Some modules may be distributed separately and need to adapt to other modules +# that are distributed and versioned separately. +{ + + # Always defined, but the value depends on the presence of an option. + config = { + value = if options ? enable then 360 else 7; + } + # Only define if possible. + // lib.optionalAttrs (options ? enable) { + enable = true; + }; + +} diff --git a/tests/modules/define-settingsDict-a-is-b.nix b/tests/modules/define-settingsDict-a-is-b.nix new file mode 100644 index 000000000..42363f45f --- /dev/null +++ b/tests/modules/define-settingsDict-a-is-b.nix @@ -0,0 +1,3 @@ +{ config, ... }: { + settingsDict.a = config.settingsDict.b; +} diff --git a/tests/modules/define-shorthandOnlyDefinesConfig-true.nix b/tests/modules/define-shorthandOnlyDefinesConfig-true.nix new file mode 100644 index 000000000..bd3a73dce --- /dev/null +++ b/tests/modules/define-shorthandOnlyDefinesConfig-true.nix @@ -0,0 +1 @@ +{ shorthandOnlyDefinesConfig = true; } diff --git a/tests/modules/define-submoduleWith-noshorthand.nix b/tests/modules/define-submoduleWith-noshorthand.nix new file mode 100644 index 000000000..35e1607b6 --- /dev/null +++ b/tests/modules/define-submoduleWith-noshorthand.nix @@ -0,0 +1,3 @@ +{ + submodule.config.config = true; +} diff --git a/tests/modules/define-submoduleWith-shorthand.nix b/tests/modules/define-submoduleWith-shorthand.nix new file mode 100644 index 000000000..17df248db --- /dev/null +++ b/tests/modules/define-submoduleWith-shorthand.nix @@ -0,0 +1,3 @@ +{ + submodule.config = true; +} diff --git a/tests/modules/define-value-int-negative.nix b/tests/modules/define-value-int-negative.nix new file mode 100644 index 000000000..a04122298 --- /dev/null +++ b/tests/modules/define-value-int-negative.nix @@ -0,0 +1,3 @@ +{ + value = -23; +} diff --git a/tests/modules/define-value-int-positive.nix b/tests/modules/define-value-int-positive.nix new file mode 100644 index 000000000..5803de172 --- /dev/null +++ b/tests/modules/define-value-int-positive.nix @@ -0,0 +1,3 @@ +{ + value = 42; +} diff --git a/tests/modules/define-value-int-zero.nix b/tests/modules/define-value-int-zero.nix new file mode 100644 index 000000000..68bb9f415 --- /dev/null +++ b/tests/modules/define-value-int-zero.nix @@ -0,0 +1,3 @@ +{ + value = 0; +} diff --git a/tests/modules/define-value-list.nix b/tests/modules/define-value-list.nix new file mode 100644 index 000000000..4831c1cc0 --- /dev/null +++ b/tests/modules/define-value-list.nix @@ -0,0 +1,3 @@ +{ + value = []; +} diff --git a/tests/modules/define-value-string-arbitrary.nix b/tests/modules/define-value-string-arbitrary.nix new file mode 100644 index 000000000..8e3abaf53 --- /dev/null +++ b/tests/modules/define-value-string-arbitrary.nix @@ -0,0 +1,3 @@ +{ + value = "foobar"; +} diff --git a/tests/modules/define-value-string-bigint.nix b/tests/modules/define-value-string-bigint.nix new file mode 100644 index 000000000..f27e31985 --- /dev/null +++ b/tests/modules/define-value-string-bigint.nix @@ -0,0 +1,3 @@ +{ + value = "1000"; +} diff --git a/tests/modules/define-value-string-properties.nix b/tests/modules/define-value-string-properties.nix new file mode 100644 index 000000000..972304c01 --- /dev/null +++ b/tests/modules/define-value-string-properties.nix @@ -0,0 +1,12 @@ +{ lib, ... }: { + + imports = [{ + value = lib.mkDefault "def"; + }]; + + value = lib.mkMerge [ + (lib.mkIf false "nope") + "yes" + ]; + +} diff --git a/tests/modules/define-value-string.nix b/tests/modules/define-value-string.nix new file mode 100644 index 000000000..e7a166965 --- /dev/null +++ b/tests/modules/define-value-string.nix @@ -0,0 +1,3 @@ +{ + value = "24"; +} diff --git a/tests/modules/define-variant.nix b/tests/modules/define-variant.nix new file mode 100644 index 000000000..423cb0e37 --- /dev/null +++ b/tests/modules/define-variant.nix @@ -0,0 +1,22 @@ +{ config, lib, ... }: +let inherit (lib) types mkOption attrNames; +in +{ + options = { + attrs = mkOption { type = types.attrsOf lib.types.int; }; + result = mkOption { }; + resultFoo = mkOption { }; + resultFooBar = mkOption { }; + resultFooFoo = mkOption { }; + }; + config = { + attrs.a = 1; + variants.foo.attrs.b = 1; + variants.bar.attrs.y = 1; + variants.foo.variants.bar.attrs.z = 1; + variants.foo.variants.foo.attrs.c = 3; + resultFoo = lib.concatMapStringsSep " " toString (attrNames config.variants.foo.attrs); + resultFooBar = lib.concatMapStringsSep " " toString (attrNames config.variants.foo.variants.bar.attrs); + resultFooFoo = lib.concatMapStringsSep " " toString (attrNames config.variants.foo.variants.foo.attrs); + }; +} diff --git a/tests/modules/disable-declare-enable.nix b/tests/modules/disable-declare-enable.nix new file mode 100644 index 000000000..a373ee7e5 --- /dev/null +++ b/tests/modules/disable-declare-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ ./declare-enable.nix ]; +} diff --git a/tests/modules/disable-define-enable-string-path.nix b/tests/modules/disable-define-enable-string-path.nix new file mode 100644 index 000000000..6429a6d63 --- /dev/null +++ b/tests/modules/disable-define-enable-string-path.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ (toString ./define-enable.nix) ]; +} diff --git a/tests/modules/disable-define-enable.nix b/tests/modules/disable-define-enable.nix new file mode 100644 index 000000000..0d84a7c3c --- /dev/null +++ b/tests/modules/disable-define-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ ./define-enable.nix ]; +} diff --git a/tests/modules/disable-enable-modules.nix b/tests/modules/disable-enable-modules.nix new file mode 100644 index 000000000..c325f4e07 --- /dev/null +++ b/tests/modules/disable-enable-modules.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ "define-enable.nix" "declare-enable.nix" ]; +} diff --git a/tests/modules/disable-module-bad-key.nix b/tests/modules/disable-module-bad-key.nix new file mode 100644 index 000000000..f50d06f2f --- /dev/null +++ b/tests/modules/disable-module-bad-key.nix @@ -0,0 +1,16 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { config, ... }: { + config = { + enable = true; + }; + }; +in +{ + imports = [ + ./declare-enable.nix + ]; + disabledModules = [ { } ]; +} diff --git a/tests/modules/disable-module-with-key.nix b/tests/modules/disable-module-with-key.nix new file mode 100644 index 000000000..ea2a60aa8 --- /dev/null +++ b/tests/modules/disable-module-with-key.nix @@ -0,0 +1,34 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { + key = "disable-module-with-key.nix#moduleWithKey"; + config = { + enable = true; + }; + }; +in +{ + options = { + positive = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + }; + default = {}; + }; + negative = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + disabledModules = [ moduleWithKey ]; + }; + default = {}; + }; + }; +} diff --git a/tests/modules/disable-module-with-toString-key.nix b/tests/modules/disable-module-with-toString-key.nix new file mode 100644 index 000000000..3f8c81904 --- /dev/null +++ b/tests/modules/disable-module-with-toString-key.nix @@ -0,0 +1,34 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { + key = 123; + config = { + enable = true; + }; + }; +in +{ + options = { + positive = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + }; + default = {}; + }; + negative = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + disabledModules = [ 123 ]; + }; + default = {}; + }; + }; +} diff --git a/tests/modules/disable-recursive/bar.nix b/tests/modules/disable-recursive/bar.nix new file mode 100644 index 000000000..4d9240a43 --- /dev/null +++ b/tests/modules/disable-recursive/bar.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ../declare-enable.nix + ]; +} diff --git a/tests/modules/disable-recursive/disable-bar.nix b/tests/modules/disable-recursive/disable-bar.nix new file mode 100644 index 000000000..987b2802a --- /dev/null +++ b/tests/modules/disable-recursive/disable-bar.nix @@ -0,0 +1,7 @@ +{ + + disabledModules = [ + ./bar.nix + ]; + +} diff --git a/tests/modules/disable-recursive/disable-foo.nix b/tests/modules/disable-recursive/disable-foo.nix new file mode 100644 index 000000000..5b68a3c46 --- /dev/null +++ b/tests/modules/disable-recursive/disable-foo.nix @@ -0,0 +1,7 @@ +{ + + disabledModules = [ + ./foo.nix + ]; + +} diff --git a/tests/modules/disable-recursive/foo.nix b/tests/modules/disable-recursive/foo.nix new file mode 100644 index 000000000..4d9240a43 --- /dev/null +++ b/tests/modules/disable-recursive/foo.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ../declare-enable.nix + ]; +} diff --git a/tests/modules/disable-recursive/main.nix b/tests/modules/disable-recursive/main.nix new file mode 100644 index 000000000..48a3c6218 --- /dev/null +++ b/tests/modules/disable-recursive/main.nix @@ -0,0 +1,8 @@ +{ + imports = [ + ./foo.nix + ./bar.nix + ]; + + enable = true; +} diff --git a/tests/modules/doRename-basic.nix b/tests/modules/doRename-basic.nix new file mode 100644 index 000000000..9d79fa4f2 --- /dev/null +++ b/tests/modules/doRename-basic.nix @@ -0,0 +1,11 @@ +{ lib, ... }: { + imports = [ + (lib.doRename { from = ["a" "b"]; to = ["c" "d" "e"]; warn = true; use = x: x; visible = true; }) + ]; + options = { + c.d.e = lib.mkOption {}; + }; + config = { + a.b = 1234; + }; +} diff --git a/tests/modules/doRename-warnings.nix b/tests/modules/doRename-warnings.nix new file mode 100644 index 000000000..6f0f1e87e --- /dev/null +++ b/tests/modules/doRename-warnings.nix @@ -0,0 +1,14 @@ +{ lib, config, ... }: { + imports = [ + (lib.doRename { from = ["a" "b"]; to = ["c" "d" "e"]; warn = true; use = x: x; visible = true; }) + ]; + options = { + warnings = lib.mkOption { type = lib.types.listOf lib.types.str; }; + c.d.e = lib.mkOption {}; + result = lib.mkOption {}; + }; + config = { + a.b = 1234; + result = lib.concatStringsSep "%" config.warnings; + }; +} diff --git a/tests/modules/emptyValues.nix b/tests/modules/emptyValues.nix new file mode 100644 index 000000000..77825de32 --- /dev/null +++ b/tests/modules/emptyValues.nix @@ -0,0 +1,36 @@ +{ lib, ... }: +let + inherit (lib) types; +in { + + options = { + int = lib.mkOption { + type = types.lazyAttrsOf types.int; + }; + list = lib.mkOption { + type = types.lazyAttrsOf (types.listOf types.int); + }; + nonEmptyList = lib.mkOption { + type = types.lazyAttrsOf (types.nonEmptyListOf types.int); + }; + attrs = lib.mkOption { + type = types.lazyAttrsOf (types.attrsOf types.int); + }; + null = lib.mkOption { + type = types.lazyAttrsOf (types.nullOr types.int); + }; + submodule = lib.mkOption { + type = types.lazyAttrsOf (types.submodule {}); + }; + }; + + config = { + int.a = lib.mkIf false null; + list.a = lib.mkIf false null; + nonEmptyList.a = lib.mkIf false null; + attrs.a = lib.mkIf false null; + null.a = lib.mkIf false null; + submodule.a = lib.mkIf false null; + }; + +} diff --git a/tests/modules/extendModules-168767-imports.nix b/tests/modules/extendModules-168767-imports.nix new file mode 100644 index 000000000..489e6b5a5 --- /dev/null +++ b/tests/modules/extendModules-168767-imports.nix @@ -0,0 +1,41 @@ +{ lib +, extendModules +, ... +}: +with lib; +{ + imports = [ + + { + options.sub = mkOption { + default = { }; + type = types.submodule ( + { config + , extendModules + , ... + }: + { + options.value = mkOption { + type = types.int; + }; + + options.specialisation = mkOption { + default = { }; + inherit + (extendModules { + modules = [{ + specialisation = mkOverride 0 { }; + }]; + }) + type; + }; + } + ); + }; + } + + { config.sub.value = 1; } + + + ]; +} diff --git a/tests/modules/freeform-attrsOf.nix b/tests/modules/freeform-attrsOf.nix new file mode 100644 index 000000000..8cc577f38 --- /dev/null +++ b/tests/modules/freeform-attrsOf.nix @@ -0,0 +1,3 @@ +{ lib, ... }: { + freeformType = with lib.types; attrsOf (either str (attrsOf str)); +} diff --git a/tests/modules/freeform-lazyAttrsOf.nix b/tests/modules/freeform-lazyAttrsOf.nix new file mode 100644 index 000000000..36d6c0b13 --- /dev/null +++ b/tests/modules/freeform-lazyAttrsOf.nix @@ -0,0 +1,3 @@ +{ lib, ... }: { + freeformType = with lib.types; lazyAttrsOf (either str (lazyAttrsOf str)); +} diff --git a/tests/modules/freeform-nested.nix b/tests/modules/freeform-nested.nix new file mode 100644 index 000000000..b81fa7f0d --- /dev/null +++ b/tests/modules/freeform-nested.nix @@ -0,0 +1,14 @@ +{ lib, ... }: +let + deathtrapArgs = lib.mapAttrs + (k: _: throw "The module system is too strict, accessing an unused option's ${k} mkOption-attribute.") + (lib.functionArgs lib.mkOption); +in +{ + options.nest.foo = lib.mkOption { + type = lib.types.bool; + default = false; + }; + options.nest.unused = lib.mkOption deathtrapArgs; + config.nest.bar = "bar"; +} diff --git a/tests/modules/freeform-str-dep-unstr.nix b/tests/modules/freeform-str-dep-unstr.nix new file mode 100644 index 000000000..a2dfbc80c --- /dev/null +++ b/tests/modules/freeform-str-dep-unstr.nix @@ -0,0 +1,8 @@ +{ lib, config, ... }: { + options.foo = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + + config.foo = lib.mkIf (config ? value) config.value; +} diff --git a/tests/modules/freeform-submodules.nix b/tests/modules/freeform-submodules.nix new file mode 100644 index 000000000..3910435a7 --- /dev/null +++ b/tests/modules/freeform-submodules.nix @@ -0,0 +1,22 @@ +{ lib, options, ... }: with lib.types; { + + options.fooDeclarations = lib.mkOption { + default = (options.free.type.getSubOptions [])._freeformOptions.foo.declarations; + }; + + options.free = lib.mkOption { + type = submodule { + config._module.freeformType = lib.mkMerge [ + (attrsOf (submodule { + options.foo = lib.mkOption {}; + })) + (attrsOf (submodule { + options.bar = lib.mkOption {}; + })) + ]; + }; + }; + + config.free.xxx.foo = 10; + config.free.yyy.bar = 10; +} diff --git a/tests/modules/freeform-unstr-dep-str.nix b/tests/modules/freeform-unstr-dep-str.nix new file mode 100644 index 000000000..549d89afe --- /dev/null +++ b/tests/modules/freeform-unstr-dep-str.nix @@ -0,0 +1,8 @@ +{ lib, config, ... }: { + options.value = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + + config.foo = lib.mkIf (config.value != null) config.value; +} diff --git a/tests/modules/functionTo/list-order.nix b/tests/modules/functionTo/list-order.nix new file mode 100644 index 000000000..77a1a43a8 --- /dev/null +++ b/tests/modules/functionTo/list-order.nix @@ -0,0 +1,25 @@ + +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo (types.listOf types.str); + }; + + result = lib.mkOption { + type = types.str; + default = toString (config.fun { + a = "a"; + b = "b"; + c = "c"; + }); + }; + }; + + config.fun = lib.mkMerge [ + (input: lib.mkAfter [ input.a ]) + (input: [ input.b ]) + ]; +} diff --git a/tests/modules/functionTo/merging-attrs.nix b/tests/modules/functionTo/merging-attrs.nix new file mode 100644 index 000000000..97c015f92 --- /dev/null +++ b/tests/modules/functionTo/merging-attrs.nix @@ -0,0 +1,27 @@ +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo (types.attrsOf types.str); + }; + + result = lib.mkOption { + type = types.str; + default = toString (lib.attrValues (config.fun { + a = "a"; + b = "b"; + c = "c"; + })); + }; + }; + + config.fun = lib.mkMerge [ + (input: { inherit (input) a; }) + (input: { inherit (input) b; }) + (input: { + b = lib.mkForce input.c; + }) + ]; +} diff --git a/tests/modules/functionTo/merging-list.nix b/tests/modules/functionTo/merging-list.nix new file mode 100644 index 000000000..15fcd2bdc --- /dev/null +++ b/tests/modules/functionTo/merging-list.nix @@ -0,0 +1,24 @@ +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo (types.listOf types.str); + }; + + result = lib.mkOption { + type = types.str; + default = toString (config.fun { + a = "a"; + b = "b"; + c = "c"; + }); + }; + }; + + config.fun = lib.mkMerge [ + (input: [ input.a ]) + (input: [ input.b ]) + ]; +} diff --git a/tests/modules/functionTo/submodule-options.nix b/tests/modules/functionTo/submodule-options.nix new file mode 100644 index 000000000..b884892ef --- /dev/null +++ b/tests/modules/functionTo/submodule-options.nix @@ -0,0 +1,61 @@ +{ lib, config, options, ... }: +let + inherit (lib) types; +in +{ + imports = [ + + # fun.<function-body>.a + ({ ... }: { + options = { + fun = lib.mkOption { + type = types.functionTo (types.submodule { + options.a = lib.mkOption { default = "a"; }; + }); + }; + }; + }) + + # fun.<function-body>.b + ({ ... }: { + options = { + fun = lib.mkOption { + type = types.functionTo (types.submodule { + options.b = lib.mkOption { default = "b"; }; + }); + }; + }; + }) + ]; + + options = { + result = lib.mkOption + { + type = types.str; + default = lib.concatStringsSep " " (lib.attrValues (config.fun (throw "shouldn't use input param"))); + }; + + optionsResult = lib.mkOption + { + type = types.str; + default = lib.concatStringsSep " " + (lib.concatLists + (lib.mapAttrsToList + (k: v: + if k == "_module" + then [ ] + else [ (lib.showOption v.loc) ] + ) + ( + (options.fun.type.getSubOptions [ "fun" ]) + ) + ) + ); + }; + }; + + config.fun = lib.mkMerge + [ + (input: { b = "bee"; }) + ]; +} diff --git a/tests/modules/functionTo/trivial.nix b/tests/modules/functionTo/trivial.nix new file mode 100644 index 000000000..0962a0cf8 --- /dev/null +++ b/tests/modules/functionTo/trivial.nix @@ -0,0 +1,17 @@ +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo types.str; + }; + + result = lib.mkOption { + type = types.str; + default = config.fun "input"; + }; + }; + + config.fun = input: "input is ${input}"; +} diff --git a/tests/modules/functionTo/wrong-type.nix b/tests/modules/functionTo/wrong-type.nix new file mode 100644 index 000000000..fd65b7508 --- /dev/null +++ b/tests/modules/functionTo/wrong-type.nix @@ -0,0 +1,18 @@ + +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo types.str; + }; + + result = lib.mkOption { + type = types.str; + default = config.fun 0; + }; + }; + + config.fun = input: input + 1; +} diff --git a/tests/modules/import-configuration.nix b/tests/modules/import-configuration.nix new file mode 100644 index 000000000..a2a32bbee --- /dev/null +++ b/tests/modules/import-configuration.nix @@ -0,0 +1,12 @@ +{ lib, ... }: +let + myconf = lib.evalModules { modules = [ { } ]; }; +in +{ + imports = [ + # We can't do this. A configuration is not equal to its set of a modules. + # Equating those would lead to a mess, as specialArgs, anonymous modules + # that can't be deduplicated, and possibly more come into play. + myconf + ]; +} diff --git a/tests/modules/import-custom-arg.nix b/tests/modules/import-custom-arg.nix new file mode 100644 index 000000000..3e687b661 --- /dev/null +++ b/tests/modules/import-custom-arg.nix @@ -0,0 +1,6 @@ +{ lib, custom, ... }: + +{ + imports = [] + ++ lib.optional custom ./define-enable-force.nix; +} diff --git a/tests/modules/import-from-store.nix b/tests/modules/import-from-store.nix new file mode 100644 index 000000000..f5af22432 --- /dev/null +++ b/tests/modules/import-from-store.nix @@ -0,0 +1,11 @@ +{ lib, ... }: +{ + + imports = [ + "${builtins.toFile "drv" "{}"}" + ./declare-enable.nix + ./define-enable.nix + ]; + +} + diff --git a/tests/modules/merge-module-with-key.nix b/tests/modules/merge-module-with-key.nix new file mode 100644 index 000000000..21f00e6ef --- /dev/null +++ b/tests/modules/merge-module-with-key.nix @@ -0,0 +1,49 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithoutKey = { + config = { + raw = "pear"; + }; + }; + + moduleWithKey = { + key = __curPos.file + "#moduleWithKey"; + config = { + raw = "pear"; + }; + }; + + decl = { + options = { + raw = mkOption { + type = types.lines; + }; + }; + }; +in +{ + options = { + once = mkOption { + type = types.submodule { + imports = [ + decl + moduleWithKey + moduleWithKey + ]; + }; + default = {}; + }; + twice = mkOption { + type = types.submodule { + imports = [ + decl + moduleWithoutKey + moduleWithoutKey + ]; + }; + default = {}; + }; + }; +} diff --git a/tests/modules/module-class-is-darwin.nix b/tests/modules/module-class-is-darwin.nix new file mode 100644 index 000000000..bacf45626 --- /dev/null +++ b/tests/modules/module-class-is-darwin.nix @@ -0,0 +1,4 @@ +{ + _class = "darwin"; + config = {}; +} diff --git a/tests/modules/module-class-is-nixos.nix b/tests/modules/module-class-is-nixos.nix new file mode 100644 index 000000000..6d62feeda --- /dev/null +++ b/tests/modules/module-class-is-nixos.nix @@ -0,0 +1,4 @@ +{ + _class = "nixos"; + config = {}; +} diff --git a/tests/modules/module-imports-_type-check.nix b/tests/modules/module-imports-_type-check.nix new file mode 100644 index 000000000..1e29c469d --- /dev/null +++ b/tests/modules/module-imports-_type-check.nix @@ -0,0 +1,3 @@ +{ + imports = [ { _type = "flake"; } ]; +} diff --git a/tests/modules/optionTypeFile.nix b/tests/modules/optionTypeFile.nix new file mode 100644 index 000000000..6015d59a7 --- /dev/null +++ b/tests/modules/optionTypeFile.nix @@ -0,0 +1,28 @@ +{ config, lib, ... }: { + + _file = "optionTypeFile.nix"; + + options.theType = lib.mkOption { + type = lib.types.optionType; + }; + + options.theOption = lib.mkOption { + type = config.theType; + default = {}; + }; + + config.theType = lib.mkMerge [ + (lib.types.submodule { + options.nested = lib.mkOption { + type = lib.types.int; + }; + }) + (lib.types.submodule { + _file = "other.nix"; + options.nested = lib.mkOption { + type = lib.types.str; + }; + }) + ]; + +} diff --git a/tests/modules/optionTypeMerging.nix b/tests/modules/optionTypeMerging.nix new file mode 100644 index 000000000..74a620c46 --- /dev/null +++ b/tests/modules/optionTypeMerging.nix @@ -0,0 +1,27 @@ +{ config, lib, ... }: { + + options.theType = lib.mkOption { + type = lib.types.optionType; + }; + + options.theOption = lib.mkOption { + type = config.theType; + }; + + config.theType = lib.mkMerge [ + (lib.types.submodule { + options.int = lib.mkOption { + type = lib.types.int; + default = 10; + }; + }) + (lib.types.submodule { + options.str = lib.mkOption { + type = lib.types.str; + }; + }) + ]; + + config.theOption.str = "hello"; + +} diff --git a/tests/modules/raw.nix b/tests/modules/raw.nix new file mode 100644 index 000000000..418e671ed --- /dev/null +++ b/tests/modules/raw.nix @@ -0,0 +1,30 @@ +{ lib, ... }: { + + options = { + processedToplevel = lib.mkOption { + type = lib.types.raw; + }; + unprocessedNesting = lib.mkOption { + type = lib.types.raw; + }; + multiple = lib.mkOption { + type = lib.types.raw; + }; + priorities = lib.mkOption { + type = lib.types.raw; + }; + }; + + config = { + processedToplevel = lib.mkIf true 10; + unprocessedNesting.foo = throw "foo"; + multiple = lib.mkMerge [ + "foo" + "foo" + ]; + priorities = lib.mkMerge [ + "foo" + (lib.mkForce "bar") + ]; + }; +} diff --git a/tests/modules/shorthand-meta.nix b/tests/modules/shorthand-meta.nix new file mode 100644 index 000000000..8c9619e18 --- /dev/null +++ b/tests/modules/shorthand-meta.nix @@ -0,0 +1,19 @@ +{ lib, ... }: +let + inherit (lib) types mkOption; +in +{ + imports = [ + ({ config, ... }: { + options = { + meta.foo = mkOption { + type = types.listOf types.str; + }; + result = mkOption { default = lib.concatStringsSep " " config.meta.foo; }; + }; + }) + { + meta.foo = [ "one" "two" ]; + } + ]; +} diff --git a/tests/modules/submoduleFiles.nix b/tests/modules/submoduleFiles.nix new file mode 100644 index 000000000..c0d9b2cef --- /dev/null +++ b/tests/modules/submoduleFiles.nix @@ -0,0 +1,21 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + default = {}; + type = lib.types.submoduleWith { + modules = [ ({ options, ... }: { + options.value = lib.mkOption {}; + + options.internalFiles = lib.mkOption { + default = options.value.files; + }; + })]; + }; + }; + + imports = [ + { + _file = "the-file.nix"; + submodule.value = 10; + } + ]; +} diff --git a/tests/modules/types-anything/attrs-coercible.nix b/tests/modules/types-anything/attrs-coercible.nix new file mode 100644 index 000000000..085cbd638 --- /dev/null +++ b/tests/modules/types-anything/attrs-coercible.nix @@ -0,0 +1,12 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config.value = { + outPath = "foo"; + err = throw "err"; + }; + +} diff --git a/tests/modules/types-anything/equal-atoms.nix b/tests/modules/types-anything/equal-atoms.nix new file mode 100644 index 000000000..972711201 --- /dev/null +++ b/tests/modules/types-anything/equal-atoms.nix @@ -0,0 +1,26 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value.int = 0; + value.bool = false; + value.string = ""; + value.path = /.; + value.null = null; + value.float = 0.1; + } + { + value.int = 0; + value.bool = false; + value.string = ""; + value.path = /.; + value.null = null; + value.float = 0.1; + } + ]; + +} diff --git a/tests/modules/types-anything/functions.nix b/tests/modules/types-anything/functions.nix new file mode 100644 index 000000000..21edd4aff --- /dev/null +++ b/tests/modules/types-anything/functions.nix @@ -0,0 +1,23 @@ +{ lib, config, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + options.applied = lib.mkOption { + default = lib.mapAttrs (name: fun: fun null) config.value; + }; + + config = lib.mkMerge [ + { + value.single-lambda = x: x; + value.multiple-lambdas = x: { inherit x; }; + value.merging-lambdas = x: { inherit x; }; + } + { + value.multiple-lambdas = x: [ x ]; + value.merging-lambdas = y: { inherit y; }; + } + ]; + +} diff --git a/tests/modules/types-anything/lists.nix b/tests/modules/types-anything/lists.nix new file mode 100644 index 000000000..bd846afd3 --- /dev/null +++ b/tests/modules/types-anything/lists.nix @@ -0,0 +1,16 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value = [ null ]; + } + { + value = [ null ]; + } + ]; + +} diff --git a/tests/modules/types-anything/mk-mods.nix b/tests/modules/types-anything/mk-mods.nix new file mode 100644 index 000000000..f84ad01df --- /dev/null +++ b/tests/modules/types-anything/mk-mods.nix @@ -0,0 +1,44 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value.mkiffalse = lib.mkIf false {}; + } + { + value.mkiftrue = lib.mkIf true {}; + } + { + value.mkdefault = lib.mkDefault 0; + } + { + value.mkdefault = 1; + } + { + value.mkmerge = lib.mkMerge [ + {} + ]; + } + { + value.mkbefore = lib.mkBefore true; + } + { + value.nested = lib.mkMerge [ + { + foo = lib.mkDefault 0; + bar = lib.mkIf false 0; + } + (lib.mkIf true { + foo = lib.mkIf true (lib.mkForce 1); + bar = { + baz = lib.mkDefault "baz"; + }; + }) + ]; + } + ]; + +} diff --git a/tests/modules/types-anything/nested-attrs.nix b/tests/modules/types-anything/nested-attrs.nix new file mode 100644 index 000000000..e57d33ef8 --- /dev/null +++ b/tests/modules/types-anything/nested-attrs.nix @@ -0,0 +1,22 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value.foo = null; + } + { + value.l1.foo = null; + } + { + value.l1.l2.foo = null; + } + { + value.l1.l2.l3.foo = null; + } + ]; + +} diff --git a/tests/release.nix b/tests/release.nix new file mode 100644 index 000000000..c3bf58db2 --- /dev/null +++ b/tests/release.nix @@ -0,0 +1,64 @@ +{ # The pkgs used for dependencies for the testing itself + # Don't test properties of pkgs.lib, but rather the lib in the parent directory + pkgs ? import ../.. {} // { lib = throw "pkgs.lib accessed, but the lib tests should use nixpkgs' lib path directly!"; }, + nix ? pkgs.nix, + nixVersions ? [ pkgs.nixVersions.minimum nix pkgs.nixVersions.unstable ], +}: + +let + testWithNix = nix: + pkgs.runCommand "nixpkgs-lib-tests-nix-${nix.version}" { + buildInputs = [ + (import ./check-eval.nix) + (import ./maintainers.nix { + inherit pkgs; + lib = import ../.; + }) + (import ./teams.nix { + inherit pkgs; + lib = import ../.; + }) + (import ../path/tests { + inherit pkgs; + }) + ]; + nativeBuildInputs = [ + nix + ]; + strictDeps = true; + } '' + datadir="${nix}/share" + export TEST_ROOT=$(pwd)/test-tmp + export NIX_BUILD_HOOK= + export NIX_CONF_DIR=$TEST_ROOT/etc + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_STORE_DIR=$TEST_ROOT/store + export PAGER=cat + cacheDir=$TEST_ROOT/binary-cache + + mkdir -p $NIX_CONF_DIR + echo "experimental-features = nix-command" >> $NIX_CONF_DIR/nix.conf + + nix-store --init + + cp -r ${../.} lib + echo "Running lib/tests/modules.sh" + bash lib/tests/modules.sh + + echo "Running lib/tests/filesystem.sh" + TEST_LIB=$PWD/lib bash lib/tests/filesystem.sh + + echo "Running lib/tests/sources.sh" + TEST_LIB=$PWD/lib bash lib/tests/sources.sh + + mkdir $out + echo success > $out/${nix.version} + ''; + +in + pkgs.symlinkJoin { + name = "nixpkgs-lib-tests"; + paths = map testWithNix nixVersions; + } diff --git a/tests/sources.sh b/tests/sources.sh new file mode 100755 index 000000000..cda77aa96 --- /dev/null +++ b/tests/sources.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s inherit_errexit + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + NIX_PATH=nixpkgs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" +fi +export NIX_PATH + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd "$work" + +# Crudely unquotes a JSON string by just taking everything between the first and the second quote. +# We're only using this for resulting /nix/store paths, which can't contain " anyways, +# nor can they contain any other characters that would need to be escaped specially in JSON +# This way we don't need to add a dependency on e.g. jq +crudeUnquoteJSON() { + cut -d \" -f2 +} + +touch {README.md,module.o,foo.bar} + +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${ + cleanSource ./. +}")' | crudeUnquoteJSON)" +(cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF +. +./foo.bar +./README.md +EOF +) || die "cleanSource 1" + + +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${ + cleanSourceWith { src = '"$work"'; filter = path: type: ! hasSuffix ".bar" path; } +}")' | crudeUnquoteJSON)" +(cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF +. +./module.o +./README.md +EOF +) || die "cleanSourceWith 1" + +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${ + cleanSourceWith { src = cleanSource '"$work"'; filter = path: type: ! hasSuffix ".bar" path; } +}")' | crudeUnquoteJSON)" +(cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF +. +./README.md +EOF +) || die "cleanSourceWith + cleanSource" + +echo >&2 tests ok diff --git a/tests/systems.nix b/tests/systems.nix new file mode 100644 index 000000000..2afe128c4 --- /dev/null +++ b/tests/systems.nix @@ -0,0 +1,42 @@ +# We assert that the new algorithmic way of generating these lists matches the +# way they were hard-coded before. +# +# One might think "if we exhaustively test, what's the point of procedurally +# calculating the lists anyway?". The answer is one can mindlessly update these +# tests as new platforms become supported, and then just give the diff a quick +# sanity check before committing :). +let + lib = import ../default.nix; + mseteq = x: y: { + expr = lib.sort lib.lessThan x; + expected = lib.sort lib.lessThan y; + }; +in +with lib.systems.doubles; lib.runTests { + testall = mseteq all (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos ++ wasi ++ windows ++ embedded ++ mmix ++ js ++ genode ++ redox); + + testarm = mseteq arm [ "armv5tel-linux" "armv6l-linux" "armv6l-netbsd" "armv6l-none" "armv7a-linux" "armv7a-netbsd" "armv7l-linux" "armv7l-netbsd" "arm-none" "armv7a-darwin" ]; + testarmv7 = mseteq armv7 [ "armv7a-darwin" "armv7a-linux" "armv7l-linux" "armv7a-netbsd" "armv7l-netbsd" ]; + testi686 = mseteq i686 [ "i686-linux" "i686-freebsd13" "i686-genode" "i686-netbsd" "i686-openbsd" "i686-cygwin" "i686-windows" "i686-none" "i686-darwin" ]; + testmips = mseteq mips [ "mips-linux" "mips64-linux" "mips64el-linux" "mipsel-linux" "mipsel-netbsd" ]; + testmmix = mseteq mmix [ "mmix-mmixware" ]; + testpower = mseteq power [ "powerpc-netbsd" "powerpc-none" "powerpc64-linux" "powerpc64le-linux" "powerpcle-none" ]; + testriscv = mseteq riscv [ "riscv32-linux" "riscv64-linux" "riscv32-netbsd" "riscv64-netbsd" "riscv32-none" "riscv64-none" ]; + testriscv32 = mseteq riscv32 [ "riscv32-linux" "riscv32-netbsd" "riscv32-none" ]; + testriscv64 = mseteq riscv64 [ "riscv64-linux" "riscv64-netbsd" "riscv64-none" ]; + tests390x = mseteq s390x [ "s390x-linux" "s390x-none" ]; + testx86_64 = mseteq x86_64 [ "x86_64-linux" "x86_64-darwin" "x86_64-freebsd13" "x86_64-genode" "x86_64-redox" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin" "x86_64-solaris" "x86_64-windows" "x86_64-none" ]; + + testcygwin = mseteq cygwin [ "i686-cygwin" "x86_64-cygwin" ]; + testdarwin = mseteq darwin [ "x86_64-darwin" "i686-darwin" "aarch64-darwin" "armv7a-darwin" ]; + testfreebsd = mseteq freebsd [ "i686-freebsd13" "x86_64-freebsd13" ]; + testgenode = mseteq genode [ "aarch64-genode" "i686-genode" "x86_64-genode" ]; + testredox = mseteq redox [ "x86_64-redox" ]; + testgnu = mseteq gnu (linux /* ++ kfreebsd ++ ... */); + testillumos = mseteq illumos [ "x86_64-solaris" ]; + testlinux = mseteq linux [ "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" "armv7l-linux" "i686-linux" "loongarch64-linux" "m68k-linux" "microblaze-linux" "microblazeel-linux" "mips-linux" "mips64-linux" "mips64el-linux" "mipsel-linux" "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" "riscv64-linux" "s390-linux" "s390x-linux" "x86_64-linux" ]; + testnetbsd = mseteq netbsd [ "aarch64-netbsd" "armv6l-netbsd" "armv7a-netbsd" "armv7l-netbsd" "i686-netbsd" "m68k-netbsd" "mipsel-netbsd" "powerpc-netbsd" "riscv32-netbsd" "riscv64-netbsd" "x86_64-netbsd" ]; + testopenbsd = mseteq openbsd [ "i686-openbsd" "x86_64-openbsd" ]; + testwindows = mseteq windows [ "i686-cygwin" "x86_64-cygwin" "i686-windows" "x86_64-windows" ]; + testunix = mseteq unix (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos ++ cygwin ++ redox); +} diff --git a/tests/teams.nix b/tests/teams.nix new file mode 100644 index 000000000..8a0a5d272 --- /dev/null +++ b/tests/teams.nix @@ -0,0 +1,50 @@ +# to run these tests: +# nix-build nixpkgs/lib/tests/teams.nix +# If it builds, all tests passed +{ pkgs ? import ../.. {}, lib ? pkgs.lib }: + +let + inherit (lib) types; + + teamModule = { config, ... }: { + options = { + shortName = lib.mkOption { + type = types.str; + }; + scope = lib.mkOption { + type = types.str; + }; + enableFeatureFreezePing = lib.mkOption { + type = types.bool; + default = false; + }; + members = lib.mkOption { + type = types.listOf (types.submodule + (import ./maintainer-module.nix { inherit lib; }) + ); + default = []; + }; + githubTeams = lib.mkOption { + type = types.listOf types.str; + default = []; + }; + }; + }; + + checkTeam = team: uncheckedAttrs: + let + prefix = [ "lib" "maintainer-team" team ]; + checkedAttrs = (lib.modules.evalModules { + inherit prefix; + modules = [ + teamModule + { + _file = toString ../../maintainers/team-list.nix; + config = uncheckedAttrs; + } + ]; + }).config; + in checkedAttrs; + + checkedTeams = lib.mapAttrs checkTeam lib.teams; +in pkgs.writeTextDir "maintainer-teams.json" (builtins.toJSON checkedTeams) diff --git a/tests/test-to-plist-expected.plist b/tests/test-to-plist-expected.plist new file mode 100644 index 000000000..df0528a60 --- /dev/null +++ b/tests/test-to-plist-expected.plist @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>nested</key> + <dict> + <key>values</key> + <dict> + <key>attrs</key> + <dict> + <key>foo b/ar</key> + <string>baz</string> + </dict> + <key>bool</key> + <true/> + <key>emptyattrs</key> + <dict> + + </dict> + <key>emptylist</key> + <array> + + </array> + <key>emptystring</key> + <string></string> + <key>float</key> + <real>0.133700</real> + <key>int</key> + <integer>42</integer> + <key>list</key> + <array> + <integer>3</integer> + <integer>4</integer> + <string>test</string> + </array> + <key>newlinestring</key> + <string> +</string> + <key>path</key> + <string>/foo</string> + <key>string</key> + <string>fn${o}"r\d</string> + </dict> + </dict> +</dict> +</plist>
\ No newline at end of file diff --git a/trivial.nix b/trivial.nix new file mode 100644 index 000000000..26e4b3240 --- /dev/null +++ b/trivial.nix @@ -0,0 +1,522 @@ +{ lib }: + +rec { + + ## Simple (higher order) functions + + /* The identity function + For when you need a function that does “nothing”. + + Type: id :: a -> a + */ + id = + # The value to return + x: x; + + /* The constant function + + Ignores the second argument. If called with only one argument, + constructs a function that always returns a static value. + + Type: const :: a -> b -> a + Example: + let f = const 5; in f 10 + => 5 + */ + const = + # Value to return + x: + # Value to ignore + y: x; + + /* Pipes a value through a list of functions, left to right. + + Type: pipe :: a -> [<functions>] -> <return type of last function> + Example: + pipe 2 [ + (x: x + 2) # 2 + 2 = 4 + (x: x * 2) # 4 * 2 = 8 + ] + => 8 + + # ideal to do text transformations + pipe [ "a/b" "a/c" ] [ + + # create the cp command + (map (file: ''cp "${src}/${file}" $out\n'')) + + # concatenate all commands into one string + lib.concatStrings + + # make that string into a nix derivation + (pkgs.runCommand "copy-to-out" {}) + + ] + => <drv which copies all files to $out> + + The output type of each function has to be the input type + of the next function, and the last function returns the + final value. + */ + pipe = val: functions: + let reverseApply = x: f: f x; + in builtins.foldl' reverseApply val functions; + + # note please don’t add a function like `compose = flip pipe`. + # This would confuse users, because the order of the functions + # in the list is not clear. With pipe, it’s obvious that it + # goes first-to-last. With `compose`, not so much. + + ## Named versions corresponding to some builtin operators. + + /* Concatenate two lists + + Type: concat :: [a] -> [a] -> [a] + + Example: + concat [ 1 2 ] [ 3 4 ] + => [ 1 2 3 4 ] + */ + concat = x: y: x ++ y; + + /* boolean “or” */ + or = x: y: x || y; + + /* boolean “and” */ + and = x: y: x && y; + + /* bitwise “and” */ + bitAnd = builtins.bitAnd + or (import ./zip-int-bits.nix + (a: b: if a==1 && b==1 then 1 else 0)); + + /* bitwise “or” */ + bitOr = builtins.bitOr + or (import ./zip-int-bits.nix + (a: b: if a==1 || b==1 then 1 else 0)); + + /* bitwise “xor” */ + bitXor = builtins.bitXor + or (import ./zip-int-bits.nix + (a: b: if a!=b then 1 else 0)); + + /* bitwise “not” */ + bitNot = builtins.sub (-1); + + /* Convert a boolean to a string. + + This function uses the strings "true" and "false" to represent + boolean values. Calling `toString` on a bool instead returns "1" + and "" (sic!). + + Type: boolToString :: bool -> string + */ + boolToString = b: if b then "true" else "false"; + + /* Merge two attribute sets shallowly, right side trumps left + + mergeAttrs :: attrs -> attrs -> attrs + + Example: + mergeAttrs { a = 1; b = 2; } { b = 3; c = 4; } + => { a = 1; b = 3; c = 4; } + */ + mergeAttrs = + # Left attribute set + x: + # Right attribute set (higher precedence for equal keys) + y: x // y; + + /* Flip the order of the arguments of a binary function. + + Type: flip :: (a -> b -> c) -> (b -> a -> c) + + Example: + flip concat [1] [2] + => [ 2 1 ] + */ + flip = f: a: b: f b a; + + /* Apply function if the supplied argument is non-null. + + Example: + mapNullable (x: x+1) null + => null + mapNullable (x: x+1) 22 + => 23 + */ + mapNullable = + # Function to call + f: + # Argument to check for null before passing it to `f` + a: if a == null then a else f a; + + # Pull in some builtins not included elsewhere. + inherit (builtins) + pathExists readFile isBool + isInt isFloat add sub lessThan + seq deepSeq genericClosure; + + + ## nixpkgs version strings + + /* Returns the current full nixpkgs version number. */ + version = release + versionSuffix; + + /* Returns the current nixpkgs release number as string. */ + release = lib.strings.fileContents ../.version; + + /* The latest release that is supported, at the time of release branch-off, + if applicable. + + Ideally, out-of-tree modules should be able to evaluate cleanly with all + supported Nixpkgs versions (master, release and old release until EOL). + So if possible, deprecation warnings should take effect only when all + out-of-tree expressions/libs/modules can upgrade to the new way without + losing support for supported Nixpkgs versions. + + This release number allows deprecation warnings to be implemented such that + they take effect as soon as the oldest release reaches end of life. */ + oldestSupportedRelease = + # Update on master only. Do not backport. + 2211; + + /* Whether a feature is supported in all supported releases (at the time of + release branch-off, if applicable). See `oldestSupportedRelease`. */ + isInOldestRelease = + /* Release number of feature introduction as an integer, e.g. 2111 for 21.11. + Set it to the upcoming release, matching the nixpkgs/.version file. + */ + release: + release <= lib.trivial.oldestSupportedRelease; + + /* Returns the current nixpkgs release code name. + + On each release the first letter is bumped and a new animal is chosen + starting with that new letter. + */ + codeName = "Tapir"; + + /* Returns the current nixpkgs version suffix as string. */ + versionSuffix = + let suffixFile = ../.version-suffix; + in if pathExists suffixFile + then lib.strings.fileContents suffixFile + else "pre-git"; + + /* Attempts to return the the current revision of nixpkgs and + returns the supplied default value otherwise. + + Type: revisionWithDefault :: string -> string + */ + revisionWithDefault = + # Default value to return if revision can not be determined + default: + let + revisionFile = "${toString ./..}/.git-revision"; + gitRepo = "${toString ./..}/.git"; + in if lib.pathIsGitRepo gitRepo + then lib.commitIdFromGitRepo gitRepo + else if lib.pathExists revisionFile then lib.fileContents revisionFile + else default; + + nixpkgsVersion = builtins.trace "`lib.nixpkgsVersion` is deprecated, use `lib.version` instead!" version; + + /* Determine whether the function is being called from inside a Nix + shell. + + Type: inNixShell :: bool + */ + inNixShell = builtins.getEnv "IN_NIX_SHELL" != ""; + + /* Determine whether the function is being called from inside pure-eval mode + by seeing whether `builtins` contains `currentSystem`. If not, we must be in + pure-eval mode. + + Type: inPureEvalMode :: bool + */ + inPureEvalMode = ! builtins ? currentSystem; + + ## Integer operations + + /* Return minimum of two numbers. */ + min = x: y: if x < y then x else y; + + /* Return maximum of two numbers. */ + max = x: y: if x > y then x else y; + + /* Integer modulus + + Example: + mod 11 10 + => 1 + mod 1 10 + => 1 + */ + mod = base: int: base - (int * (builtins.div base int)); + + + ## Comparisons + + /* C-style comparisons + + a < b, compare a b => -1 + a == b, compare a b => 0 + a > b, compare a b => 1 + */ + compare = a: b: + if a < b + then -1 + else if a > b + then 1 + else 0; + + /* Split type into two subtypes by predicate `p`, take all elements + of the first subtype to be less than all the elements of the + second subtype, compare elements of a single subtype with `yes` + and `no` respectively. + + Type: (a -> bool) -> (a -> a -> int) -> (a -> a -> int) -> (a -> a -> int) + + Example: + let cmp = splitByAndCompare (hasPrefix "foo") compare compare; in + + cmp "a" "z" => -1 + cmp "fooa" "fooz" => -1 + + cmp "f" "a" => 1 + cmp "fooa" "a" => -1 + # while + compare "fooa" "a" => 1 + */ + splitByAndCompare = + # Predicate + p: + # Comparison function if predicate holds for both values + yes: + # Comparison function if predicate holds for neither value + no: + # First value to compare + a: + # Second value to compare + b: + if p a + then if p b then yes a b else -1 + else if p b then 1 else no a b; + + + /* Reads a JSON file. + + Type :: path -> any + */ + importJSON = path: + builtins.fromJSON (builtins.readFile path); + + /* Reads a TOML file. + + Type :: path -> any + */ + importTOML = path: + builtins.fromTOML (builtins.readFile path); + + ## Warnings + + # See https://github.com/NixOS/nix/issues/749. Eventually we'd like these + # to expand to Nix builtins that carry metadata so that Nix can filter out + # the INFO messages without parsing the message string. + # + # Usage: + # { + # foo = lib.warn "foo is deprecated" oldFoo; + # bar = lib.warnIf (bar == "") "Empty bar is deprecated" bar; + # } + # + # TODO: figure out a clever way to integrate location information from + # something like __unsafeGetAttrPos. + + /* + Print a warning before returning the second argument. This function behaves + like `builtins.trace`, but requires a string message and formats it as a + warning, including the `warning: ` prefix. + + To get a call stack trace and abort evaluation, set the environment variable + `NIX_ABORT_ON_WARN=true` and set the Nix options `--option pure-eval false --show-trace` + + Type: string -> a -> a + */ + warn = + if lib.elem (builtins.getEnv "NIX_ABORT_ON_WARN") ["1" "true" "yes"] + then msg: builtins.trace "[1;31mwarning: ${msg}[0m" (abort "NIX_ABORT_ON_WARN=true; warnings are treated as unrecoverable errors.") + else msg: builtins.trace "[1;31mwarning: ${msg}[0m"; + + /* + Like warn, but only warn when the first argument is `true`. + + Type: bool -> string -> a -> a + */ + warnIf = cond: msg: if cond then warn msg else x: x; + + /* + Like warnIf, but negated (warn if the first argument is `false`). + + Type: bool -> string -> a -> a + */ + warnIfNot = cond: msg: if cond then x: x else warn msg; + + /* + Like the `assert b; e` expression, but with a custom error message and + without the semicolon. + + If true, return the identity function, `r: r`. + + If false, throw the error message. + + Calls can be juxtaposed using function application, as `(r: r) a = a`, so + `(r: r) (r: r) a = a`, and so forth. + + Type: bool -> string -> a -> a + + Example: + + throwIfNot (lib.isList overlays) "The overlays argument to nixpkgs must be a list." + lib.foldr (x: throwIfNot (lib.isFunction x) "All overlays passed to nixpkgs must be functions.") (r: r) overlays + pkgs + + */ + throwIfNot = cond: msg: if cond then x: x else throw msg; + + /* + Like throwIfNot, but negated (throw if the first argument is `true`). + + Type: bool -> string -> a -> a + */ + throwIf = cond: msg: if cond then throw msg else x: x; + + /* Check if the elements in a list are valid values from a enum, returning the identity function, or throwing an error message otherwise. + + Example: + let colorVariants = ["bright" "dark" "black"] + in checkListOfEnum "color variants" [ "standard" "light" "dark" ] colorVariants; + => + error: color variants: bright, black unexpected; valid ones: standard, light, dark + + Type: String -> List ComparableVal -> List ComparableVal -> a -> a + */ + checkListOfEnum = msg: valid: given: + let + unexpected = lib.subtractLists valid given; + in + lib.throwIfNot (unexpected == []) + "${msg}: ${builtins.concatStringsSep ", " (builtins.map builtins.toString unexpected)} unexpected; valid ones: ${builtins.concatStringsSep ", " (builtins.map builtins.toString valid)}"; + + info = msg: builtins.trace "INFO: ${msg}"; + + showWarnings = warnings: res: lib.foldr (w: x: warn w x) res warnings; + + ## Function annotations + + /* Add metadata about expected function arguments to a function. + The metadata should match the format given by + builtins.functionArgs, i.e. a set from expected argument to a bool + representing whether that argument has a default or not. + setFunctionArgs : (a → b) → Map String Bool → (a → b) + + This function is necessary because you can't dynamically create a + function of the { a, b ? foo, ... }: format, but some facilities + like callPackage expect to be able to query expected arguments. + */ + setFunctionArgs = f: args: + { # TODO: Should we add call-time "type" checking like built in? + __functor = self: f; + __functionArgs = args; + }; + + /* Extract the expected function arguments from a function. + This works both with nix-native { a, b ? foo, ... }: style + functions and functions with args set with 'setFunctionArgs'. It + has the same return type and semantics as builtins.functionArgs. + setFunctionArgs : (a → b) → Map String Bool. + */ + functionArgs = f: + if f ? __functor + then f.__functionArgs or (lib.functionArgs (f.__functor f)) + else builtins.functionArgs f; + + /* Check whether something is a function or something + annotated with function args. + */ + isFunction = f: builtins.isFunction f || + (f ? __functor && isFunction (f.__functor f)); + + /* + Turns any non-callable values into constant functions. + Returns callable values as is. + + Example: + + nix-repl> lib.toFunction 1 2 + 1 + + nix-repl> lib.toFunction (x: x + 1) 2 + 3 + */ + toFunction = + # Any value + v: + if isFunction v + then v + else k: v; + + /* Convert the given positive integer to a string of its hexadecimal + representation. For example: + + toHexString 0 => "0" + + toHexString 16 => "10" + + toHexString 250 => "FA" + */ + toHexString = i: + let + toHexDigit = d: + if d < 10 + then toString d + else + { + "10" = "A"; + "11" = "B"; + "12" = "C"; + "13" = "D"; + "14" = "E"; + "15" = "F"; + }.${toString d}; + in + lib.concatMapStrings toHexDigit (toBaseDigits 16 i); + + /* `toBaseDigits base i` converts the positive integer i to a list of its + digits in the given base. For example: + + toBaseDigits 10 123 => [ 1 2 3 ] + + toBaseDigits 2 6 => [ 1 1 0 ] + + toBaseDigits 16 250 => [ 15 10 ] + */ + toBaseDigits = base: i: + let + go = i: + if i < base + then [i] + else + let + r = i - ((i / base) * base); + q = (i - r) / base; + in + [r] ++ go q; + in + assert (isInt base); + assert (isInt i); + assert (base >= 2); + assert (i >= 0); + lib.reverseList (go i); +} diff --git a/types.nix b/types.nix new file mode 100644 index 000000000..9360d42f5 --- /dev/null +++ b/types.nix @@ -0,0 +1,908 @@ +# Definitions related to run-time type checking. Used in particular +# to type-check NixOS configurations. +{ lib }: + +let + inherit (lib) + elem + flip + isAttrs + isBool + isDerivation + isFloat + isFunction + isInt + isList + isString + isStorePath + toDerivation + toList + ; + inherit (lib.lists) + all + concatLists + count + elemAt + filter + foldl' + head + imap1 + last + length + tail + ; + inherit (lib.attrsets) + attrNames + filterAttrs + hasAttr + mapAttrs + optionalAttrs + zipAttrsWith + ; + inherit (lib.options) + getFiles + getValues + mergeDefaultOption + mergeEqualOption + mergeOneOption + mergeUniqueOption + showFiles + showOption + ; + inherit (lib.strings) + concatMapStringsSep + concatStringsSep + escapeNixString + hasInfix + isStringLike + ; + inherit (lib.trivial) + boolToString + ; + + inherit (lib.modules) + mergeDefinitions + fixupOptionType + mergeOptionDecls + ; + outer_types = +rec { + isType = type: x: (x._type or "") == type; + + setType = typeName: value: value // { + _type = typeName; + }; + + + # Default type merging function + # takes two type functors and return the merged type + defaultTypeMerge = f: f': + let wrapped = f.wrapped.typeMerge f'.wrapped.functor; + payload = f.binOp f.payload f'.payload; + in + # cannot merge different types + if f.name != f'.name + then null + # simple types + else if (f.wrapped == null && f'.wrapped == null) + && (f.payload == null && f'.payload == null) + then f.type + # composed types + else if (f.wrapped != null && f'.wrapped != null) && (wrapped != null) + then f.type wrapped + # value types + else if (f.payload != null && f'.payload != null) && (payload != null) + then f.type payload + else null; + + # Default type functor + defaultFunctor = name: { + inherit name; + type = types.${name} or null; + wrapped = null; + payload = null; + binOp = a: b: null; + }; + + isOptionType = isType "option-type"; + mkOptionType = + { # Human-readable representation of the type, should be equivalent to + # the type function name. + name + , # Description of the type, defined recursively by embedding the wrapped type if any. + description ? null + # A hint for whether or not this description needs parentheses. Possible values: + # - "noun": a simple noun phrase such as "positive integer" + # - "conjunction": a phrase with a potentially ambiguous "or" connective. + # - "composite": a phrase with an "of" connective + # See the `optionDescriptionPhrase` function. + , descriptionClass ? null + , # DO NOT USE WITHOUT KNOWING WHAT YOU ARE DOING! + # Function applied to each definition that must return false when a definition + # does not match the type. It should not check more than the root of the value, + # because checking nested values reduces laziness, leading to unnecessary + # infinite recursions in the module system. + # Further checks of nested values should be performed by throwing in + # the merge function. + # Strict and deep type checking can be performed by calling lib.deepSeq on + # the merged value. + # + # See https://github.com/NixOS/nixpkgs/pull/6794 that introduced this change, + # https://github.com/NixOS/nixpkgs/pull/173568 and + # https://github.com/NixOS/nixpkgs/pull/168295 that attempted to revert this, + # https://github.com/NixOS/nixpkgs/issues/191124 and + # https://github.com/NixOS/nixos-search/issues/391 for what happens if you ignore + # this disclaimer. + check ? (x: true) + , # Merge a list of definitions together into a single value. + # This function is called with two arguments: the location of + # the option in the configuration as a list of strings + # (e.g. ["boot" "loader "grub" "enable"]), and a list of + # definition values and locations (e.g. [ { file = "/foo.nix"; + # value = 1; } { file = "/bar.nix"; value = 2 } ]). + merge ? mergeDefaultOption + , # Whether this type has a value representing nothingness. If it does, + # this should be a value of the form { value = <the nothing value>; } + # If it doesn't, this should be {} + # This may be used when a value is required for `mkIf false`. This allows the extra laziness in e.g. `lazyAttrsOf`. + emptyValue ? {} + , # Return a flat list of sub-options. Used to generate + # documentation. + getSubOptions ? prefix: {} + , # List of modules if any, or null if none. + getSubModules ? null + , # Function for building the same option type with a different list of + # modules. + substSubModules ? m: null + , # Function that merge type declarations. + # internal, takes a functor as argument and returns the merged type. + # returning null means the type is not mergeable + typeMerge ? defaultTypeMerge functor + , # The type functor. + # internal, representation of the type as an attribute set. + # name: name of the type + # type: type function. + # wrapped: the type wrapped in case of compound types. + # payload: values of the type, two payloads of the same type must be + # combinable with the binOp binary operation. + # binOp: binary operation that merge two payloads of the same type. + functor ? defaultFunctor name + , # The deprecation message to display when this type is used by an option + # If null, the type isn't deprecated + deprecationMessage ? null + , # The types that occur in the definition of this type. This is used to + # issue deprecation warnings recursively. Can also be used to reuse + # nested types + nestedTypes ? {} + }: + { _type = "option-type"; + inherit + name check merge emptyValue getSubOptions getSubModules substSubModules + typeMerge functor deprecationMessage nestedTypes descriptionClass; + description = if description == null then name else description; + }; + + # optionDescriptionPhrase :: (str -> bool) -> optionType -> str + # + # Helper function for producing unambiguous but readable natural language + # descriptions of types. + # + # Parameters + # + # optionDescriptionPhase unparenthesize optionType + # + # `unparenthesize`: A function from descriptionClass string to boolean. + # It must return true when the class of phrase will fit unambiguously into + # the description of the caller. + # + # `optionType`: The option type to parenthesize or not. + # The option whose description we're returning. + # + # Return value + # + # The description of the `optionType`, with parentheses if there may be an + # ambiguity. + optionDescriptionPhrase = unparenthesize: t: + if unparenthesize (t.descriptionClass or null) + then t.description + else "(${t.description})"; + + # When adding new types don't forget to document them in + # nixos/doc/manual/development/option-types.xml! + types = rec { + + raw = mkOptionType rec { + name = "raw"; + description = "raw value"; + descriptionClass = "noun"; + check = value: true; + merge = mergeOneOption; + }; + + anything = mkOptionType { + name = "anything"; + description = "anything"; + descriptionClass = "noun"; + check = value: true; + merge = loc: defs: + let + getType = value: + if isAttrs value && isStringLike value + then "stringCoercibleSet" + else builtins.typeOf value; + + # Returns the common type of all definitions, throws an error if they + # don't have the same type + commonType = foldl' (type: def: + if getType def.value == type + then type + else throw "The option `${showOption loc}' has conflicting option types in ${showFiles (getFiles defs)}" + ) (getType (head defs).value) defs; + + mergeFunction = { + # Recursively merge attribute sets + set = (attrsOf anything).merge; + # Safe and deterministic behavior for lists is to only accept one definition + # listOf only used to apply mkIf and co. + list = + if length defs > 1 + then throw "The option `${showOption loc}' has conflicting definitions, in ${showFiles (getFiles defs)}." + else (listOf anything).merge; + # This is the type of packages, only accept a single definition + stringCoercibleSet = mergeOneOption; + lambda = loc: defs: arg: anything.merge + (loc ++ [ "<function body>" ]) + (map (def: { + file = def.file; + value = def.value arg; + }) defs); + # Otherwise fall back to only allowing all equal definitions + }.${commonType} or mergeEqualOption; + in mergeFunction loc defs; + }; + + unspecified = mkOptionType { + name = "unspecified"; + description = "unspecified value"; + descriptionClass = "noun"; + }; + + bool = mkOptionType { + name = "bool"; + description = "boolean"; + descriptionClass = "noun"; + check = isBool; + merge = mergeEqualOption; + }; + + int = mkOptionType { + name = "int"; + description = "signed integer"; + descriptionClass = "noun"; + check = isInt; + merge = mergeEqualOption; + }; + + # Specialized subdomains of int + ints = + let + betweenDesc = lowest: highest: + "${toString lowest} and ${toString highest} (both inclusive)"; + between = lowest: highest: + assert lib.assertMsg (lowest <= highest) + "ints.between: lowest must be smaller than highest"; + addCheck int (x: x >= lowest && x <= highest) // { + name = "intBetween"; + description = "integer between ${betweenDesc lowest highest}"; + }; + ign = lowest: highest: name: docStart: + between lowest highest // { + inherit name; + description = docStart + "; between ${betweenDesc lowest highest}"; + }; + unsign = bit: range: ign 0 (range - 1) + "unsignedInt${toString bit}" "${toString bit} bit unsigned integer"; + sign = bit: range: ign (0 - (range / 2)) (range / 2 - 1) + "signedInt${toString bit}" "${toString bit} bit signed integer"; + + in { + /* An int with a fixed range. + * + * Example: + * (ints.between 0 100).check (-1) + * => false + * (ints.between 0 100).check (101) + * => false + * (ints.between 0 0).check 0 + * => true + */ + inherit between; + + unsigned = addCheck types.int (x: x >= 0) // { + name = "unsignedInt"; + description = "unsigned integer, meaning >=0"; + }; + positive = addCheck types.int (x: x > 0) // { + name = "positiveInt"; + description = "positive integer, meaning >0"; + }; + u8 = unsign 8 256; + u16 = unsign 16 65536; + # the biggest int Nix accepts is 2^63 - 1 (9223372036854775808) + # the smallest int Nix accepts is -2^63 (-9223372036854775807) + u32 = unsign 32 4294967296; + # u64 = unsign 64 18446744073709551616; + + s8 = sign 8 256; + s16 = sign 16 65536; + s32 = sign 32 4294967296; + }; + + # Alias of u16 for a port number + port = ints.u16; + + float = mkOptionType { + name = "float"; + description = "floating point number"; + descriptionClass = "noun"; + check = isFloat; + merge = mergeEqualOption; + }; + + number = either int float; + + numbers = let + betweenDesc = lowest: highest: + "${builtins.toJSON lowest} and ${builtins.toJSON highest} (both inclusive)"; + in { + between = lowest: highest: + assert lib.assertMsg (lowest <= highest) + "numbers.between: lowest must be smaller than highest"; + addCheck number (x: x >= lowest && x <= highest) // { + name = "numberBetween"; + description = "integer or floating point number between ${betweenDesc lowest highest}"; + }; + + nonnegative = addCheck number (x: x >= 0) // { + name = "numberNonnegative"; + description = "nonnegative integer or floating point number, meaning >=0"; + }; + positive = addCheck number (x: x > 0) // { + name = "numberPositive"; + description = "positive integer or floating point number, meaning >0"; + }; + }; + + str = mkOptionType { + name = "str"; + description = "string"; + descriptionClass = "noun"; + check = isString; + merge = mergeEqualOption; + }; + + nonEmptyStr = mkOptionType { + name = "nonEmptyStr"; + description = "non-empty string"; + descriptionClass = "noun"; + check = x: str.check x && builtins.match "[ \t\n]*" x == null; + inherit (str) merge; + }; + + # Allow a newline character at the end and trim it in the merge function. + singleLineStr = + let + inherit (strMatching "[^\n\r]*\n?") check merge; + in + mkOptionType { + name = "singleLineStr"; + description = "(optionally newline-terminated) single-line string"; + descriptionClass = "noun"; + inherit check; + merge = loc: defs: + lib.removeSuffix "\n" (merge loc defs); + }; + + strMatching = pattern: mkOptionType { + name = "strMatching ${escapeNixString pattern}"; + description = "string matching the pattern ${pattern}"; + descriptionClass = "noun"; + check = x: str.check x && builtins.match pattern x != null; + inherit (str) merge; + }; + + # Merge multiple definitions by concatenating them (with the given + # separator between the values). + separatedString = sep: mkOptionType rec { + name = "separatedString"; + description = if sep == "" + then "Concatenated string" # for types.string. + else "strings concatenated with ${builtins.toJSON sep}" + ; + descriptionClass = "noun"; + check = isString; + merge = loc: defs: concatStringsSep sep (getValues defs); + functor = (defaultFunctor name) // { + payload = sep; + binOp = sepLhs: sepRhs: + if sepLhs == sepRhs then sepLhs + else null; + }; + }; + + lines = separatedString "\n"; + commas = separatedString ","; + envVar = separatedString ":"; + + # Deprecated; should not be used because it quietly concatenates + # strings, which is usually not what you want. + string = separatedString "" // { + name = "string"; + deprecationMessage = "See https://github.com/NixOS/nixpkgs/pull/66346 for better alternative types."; + }; + + passwdEntry = entryType: addCheck entryType (str: !(hasInfix ":" str || hasInfix "\n" str)) // { + name = "passwdEntry ${entryType.name}"; + description = "${optionDescriptionPhrase (class: class == "noun") entryType}, not containing newlines or colons"; + }; + + attrs = mkOptionType { + name = "attrs"; + description = "attribute set"; + check = isAttrs; + merge = loc: foldl' (res: def: res // def.value) {}; + emptyValue = { value = {}; }; + }; + + # A package is a top-level store path (/nix/store/hash-name). This includes: + # - derivations + # - more generally, attribute sets with an `outPath` or `__toString` attribute + # pointing to a store path, e.g. flake inputs + # - strings with context, e.g. "${pkgs.foo}" or (toString pkgs.foo) + # - hardcoded store path literals (/nix/store/hash-foo) or strings without context + # ("/nix/store/hash-foo"). These get a context added to them using builtins.storePath. + package = mkOptionType { + name = "package"; + descriptionClass = "noun"; + check = x: isDerivation x || isStorePath x; + merge = loc: defs: + let res = mergeOneOption loc defs; + in if builtins.isPath res || (builtins.isString res && ! builtins.hasContext res) + then toDerivation res + else res; + }; + + shellPackage = package // { + check = x: isDerivation x && hasAttr "shellPath" x; + }; + + pkgs = addCheck + (unique { message = "A Nixpkgs pkgs set can not be merged with another pkgs set."; } attrs // { + name = "pkgs"; + descriptionClass = "noun"; + description = "Nixpkgs package set"; + }) + (x: (x._type or null) == "pkgs"); + + path = mkOptionType { + name = "path"; + descriptionClass = "noun"; + check = x: isStringLike x && builtins.substring 0 1 (toString x) == "/"; + merge = mergeEqualOption; + }; + + listOf = elemType: mkOptionType rec { + name = "listOf"; + description = "list of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isList; + merge = loc: defs: + map (x: x.value) (filter (x: x ? value) (concatLists (imap1 (n: def: + imap1 (m: def': + (mergeDefinitions + (loc ++ ["[definition ${toString n}-entry ${toString m}]"]) + elemType + [{ inherit (def) file; value = def'; }] + ).optionalValue + ) def.value + ) defs))); + emptyValue = { value = []; }; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ ["*"]); + getSubModules = elemType.getSubModules; + substSubModules = m: listOf (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + nonEmptyListOf = elemType: + let list = addCheck (types.listOf elemType) (l: l != []); + in list // { + description = "non-empty ${optionDescriptionPhrase (class: class == "noun") list}"; + emptyValue = { }; # no .value attr, meaning unset + }; + + attrsOf = elemType: mkOptionType rec { + name = "attrsOf"; + description = "attribute set of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isAttrs; + merge = loc: defs: + mapAttrs (n: v: v.value) (filterAttrs (n: v: v ? value) (zipAttrsWith (name: defs: + (mergeDefinitions (loc ++ [name]) elemType defs).optionalValue + ) + # Push down position info. + (map (def: mapAttrs (n: v: { inherit (def) file; value = v; }) def.value) defs))); + emptyValue = { value = {}; }; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ ["<name>"]); + getSubModules = elemType.getSubModules; + substSubModules = m: attrsOf (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + # A version of attrsOf that's lazy in its values at the expense of + # conditional definitions not working properly. E.g. defining a value with + # `foo.attr = mkIf false 10`, then `foo ? attr == true`, whereas with + # attrsOf it would correctly be `false`. Accessing `foo.attr` would throw an + # error that it's not defined. Use only if conditional definitions don't make sense. + lazyAttrsOf = elemType: mkOptionType rec { + name = "lazyAttrsOf"; + description = "lazy attribute set of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isAttrs; + merge = loc: defs: + zipAttrsWith (name: defs: + let merged = mergeDefinitions (loc ++ [name]) elemType defs; + # mergedValue will trigger an appropriate error when accessed + in merged.optionalValue.value or elemType.emptyValue.value or merged.mergedValue + ) + # Push down position info. + (map (def: mapAttrs (n: v: { inherit (def) file; value = v; }) def.value) defs); + emptyValue = { value = {}; }; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ ["<name>"]); + getSubModules = elemType.getSubModules; + substSubModules = m: lazyAttrsOf (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + # TODO: deprecate this in the future: + loaOf = elemType: types.attrsOf elemType // { + name = "loaOf"; + deprecationMessage = "Mixing lists with attribute values is no longer" + + " possible; please use `types.attrsOf` instead. See" + + " https://github.com/NixOS/nixpkgs/issues/1800 for the motivation."; + nestedTypes.elemType = elemType; + }; + + # Value of given type but with no merging (i.e. `uniq list`s are not concatenated). + uniq = elemType: mkOptionType rec { + name = "uniq"; + inherit (elemType) description descriptionClass check; + merge = mergeOneOption; + emptyValue = elemType.emptyValue; + getSubOptions = elemType.getSubOptions; + getSubModules = elemType.getSubModules; + substSubModules = m: uniq (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + unique = { message }: type: mkOptionType rec { + name = "unique"; + inherit (type) description descriptionClass check; + merge = mergeUniqueOption { inherit message; }; + emptyValue = type.emptyValue; + getSubOptions = type.getSubOptions; + getSubModules = type.getSubModules; + substSubModules = m: uniq (type.substSubModules m); + functor = (defaultFunctor name) // { wrapped = type; }; + nestedTypes.elemType = type; + }; + + # Null or value of ... + nullOr = elemType: mkOptionType rec { + name = "nullOr"; + description = "null or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") elemType}"; + descriptionClass = "conjunction"; + check = x: x == null || elemType.check x; + merge = loc: defs: + let nrNulls = count (def: def.value == null) defs; in + if nrNulls == length defs then null + else if nrNulls != 0 then + throw "The option `${showOption loc}` is defined both null and not null, in ${showFiles (getFiles defs)}." + else elemType.merge loc defs; + emptyValue = { value = null; }; + getSubOptions = elemType.getSubOptions; + getSubModules = elemType.getSubModules; + substSubModules = m: nullOr (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + functionTo = elemType: mkOptionType { + name = "functionTo"; + description = "function that evaluates to a(n) ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isFunction; + merge = loc: defs: + fnArgs: (mergeDefinitions (loc ++ [ "<function body>" ]) elemType (map (fn: { inherit (fn) file; value = fn.value fnArgs; }) defs)).mergedValue; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ [ "<function body>" ]); + getSubModules = elemType.getSubModules; + substSubModules = m: functionTo (elemType.substSubModules m); + functor = (defaultFunctor "functionTo") // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + # A submodule (like typed attribute set). See NixOS manual. + submodule = modules: submoduleWith { + shorthandOnlyDefinesConfig = true; + modules = toList modules; + }; + + # A module to be imported in some other part of the configuration. + deferredModule = deferredModuleWith { }; + + # A module to be imported in some other part of the configuration. + # `staticModules`' options will be added to the documentation, unlike + # options declared via `config`. + deferredModuleWith = attrs@{ staticModules ? [] }: mkOptionType { + name = "deferredModule"; + description = "module"; + descriptionClass = "noun"; + check = x: isAttrs x || isFunction x || path.check x; + merge = loc: defs: { + imports = staticModules ++ map (def: lib.setDefaultModuleLocation "${def.file}, via option ${showOption loc}" def.value) defs; + }; + inherit (submoduleWith { modules = staticModules; }) + getSubOptions + getSubModules; + substSubModules = m: deferredModuleWith (attrs // { + staticModules = m; + }); + functor = defaultFunctor "deferredModuleWith" // { + type = types.deferredModuleWith; + payload = { + inherit staticModules; + }; + binOp = lhs: rhs: { + staticModules = lhs.staticModules ++ rhs.staticModules; + }; + }; + }; + + # The type of a type! + optionType = mkOptionType { + name = "optionType"; + description = "optionType"; + descriptionClass = "noun"; + check = value: value._type or null == "option-type"; + merge = loc: defs: + if length defs == 1 + then (head defs).value + else let + # Prepares the type definitions for mergeOptionDecls, which + # annotates submodules types with file locations + optionModules = map ({ value, file }: + { + _file = file; + # There's no way to merge types directly from the module system, + # but we can cheat a bit by just declaring an option with the type + options = lib.mkOption { + type = value; + }; + } + ) defs; + # Merges all the types into a single one, including submodule merging. + # This also propagates file information to all submodules + mergedOption = fixupOptionType loc (mergeOptionDecls loc optionModules); + in mergedOption.type; + }; + + submoduleWith = + { modules + , specialArgs ? {} + , shorthandOnlyDefinesConfig ? false + , description ? null + , class ? null + }@attrs: + let + inherit (lib.modules) evalModules; + + allModules = defs: map ({ value, file }: + if isAttrs value && shorthandOnlyDefinesConfig + then { _file = file; config = value; } + else { _file = file; imports = [ value ]; } + ) defs; + + base = evalModules { + inherit class specialArgs; + modules = [{ + # This is a work-around for the fact that some sub-modules, + # such as the one included in an attribute set, expects an "args" + # attribute to be given to the sub-module. As the option + # evaluation does not have any specific attribute name yet, we + # provide a default for the documentation and the freeform type. + # + # This is necessary as some option declaration might use the + # "name" attribute given as argument of the submodule and use it + # as the default of option declarations. + # + # We use lookalike unicode single angle quotation marks because + # of the docbook transformation the options receive. In all uses + # > and < wouldn't be encoded correctly so the encoded values + # would be used, and use of `<` and `>` would break the XML document. + # It shouldn't cause an issue since this is cosmetic for the manual. + _module.args.name = lib.mkOptionDefault "‹name›"; + }] ++ modules; + }; + + freeformType = base._module.freeformType; + + name = "submodule"; + + in + mkOptionType { + inherit name; + description = + if description != null then description + else freeformType.description or name; + check = x: isAttrs x || isFunction x || path.check x; + merge = loc: defs: + (base.extendModules { + modules = [ { _module.args.name = last loc; } ] ++ allModules defs; + prefix = loc; + }).config; + emptyValue = { value = {}; }; + getSubOptions = prefix: (base.extendModules + { inherit prefix; }).options // optionalAttrs (freeformType != null) { + # Expose the sub options of the freeform type. Note that the option + # discovery doesn't care about the attribute name used here, so this + # is just to avoid conflicts with potential options from the submodule + _freeformOptions = freeformType.getSubOptions prefix; + }; + getSubModules = modules; + substSubModules = m: submoduleWith (attrs // { + modules = m; + }); + nestedTypes = lib.optionalAttrs (freeformType != null) { + freeformType = freeformType; + }; + functor = defaultFunctor name // { + type = types.submoduleWith; + payload = { + inherit modules class specialArgs shorthandOnlyDefinesConfig description; + }; + binOp = lhs: rhs: { + class = + # `or null` was added for backwards compatibility only. `class` is + # always set in the current version of the module system. + if lhs.class or null == null then rhs.class or null + else if rhs.class or null == null then lhs.class or null + else if lhs.class or null == rhs.class then lhs.class or null + else throw "A submoduleWith option is declared multiple times with conflicting class values \"${toString lhs.class}\" and \"${toString rhs.class}\"."; + modules = lhs.modules ++ rhs.modules; + specialArgs = + let intersecting = builtins.intersectAttrs lhs.specialArgs rhs.specialArgs; + in if intersecting == {} + then lhs.specialArgs // rhs.specialArgs + else throw "A submoduleWith option is declared multiple times with the same specialArgs \"${toString (attrNames intersecting)}\""; + shorthandOnlyDefinesConfig = + if lhs.shorthandOnlyDefinesConfig == null + then rhs.shorthandOnlyDefinesConfig + else if rhs.shorthandOnlyDefinesConfig == null + then lhs.shorthandOnlyDefinesConfig + else if lhs.shorthandOnlyDefinesConfig == rhs.shorthandOnlyDefinesConfig + then lhs.shorthandOnlyDefinesConfig + else throw "A submoduleWith option is declared multiple times with conflicting shorthandOnlyDefinesConfig values"; + description = + if lhs.description == null + then rhs.description + else if rhs.description == null + then lhs.description + else if lhs.description == rhs.description + then lhs.description + else throw "A submoduleWith option is declared multiple times with conflicting descriptions"; + }; + }; + }; + + # A value from a set of allowed ones. + enum = values: + let + inherit (lib.lists) unique; + show = v: + if builtins.isString v then ''"${v}"'' + else if builtins.isInt v then builtins.toString v + else if builtins.isBool v then boolToString v + else ''<${builtins.typeOf v}>''; + in + mkOptionType rec { + name = "enum"; + description = + # Length 0 or 1 enums may occur in a design pattern with type merging + # where an "interface" module declares an empty enum and other modules + # provide implementations, each extending the enum with their own + # identifier. + if values == [] then + "impossible (empty enum)" + else if builtins.length values == 1 then + "value ${show (builtins.head values)} (singular enum)" + else + "one of ${concatMapStringsSep ", " show values}"; + descriptionClass = + if builtins.length values < 2 + then "noun" + else "conjunction"; + check = flip elem values; + merge = mergeEqualOption; + functor = (defaultFunctor name) // { payload = values; binOp = a: b: unique (a ++ b); }; + }; + + # Either value of type `t1` or `t2`. + either = t1: t2: mkOptionType rec { + name = "either"; + description = "${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") t1} or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction" || class == "composite") t2}"; + descriptionClass = "conjunction"; + check = x: t1.check x || t2.check x; + merge = loc: defs: + let + defList = map (d: d.value) defs; + in + if all (x: t1.check x) defList + then t1.merge loc defs + else if all (x: t2.check x) defList + then t2.merge loc defs + else mergeOneOption loc defs; + typeMerge = f': + let mt1 = t1.typeMerge (elemAt f'.wrapped 0).functor; + mt2 = t2.typeMerge (elemAt f'.wrapped 1).functor; + in + if (name == f'.name) && (mt1 != null) && (mt2 != null) + then functor.type mt1 mt2 + else null; + functor = (defaultFunctor name) // { wrapped = [ t1 t2 ]; }; + nestedTypes.left = t1; + nestedTypes.right = t2; + }; + + # Any of the types in the given list + oneOf = ts: + let + head' = if ts == [] then throw "types.oneOf needs to get at least one type in its argument" else head ts; + tail' = tail ts; + in foldl' either head' tail'; + + # Either value of type `coercedType` or `finalType`, the former is + # converted to `finalType` using `coerceFunc`. + coercedTo = coercedType: coerceFunc: finalType: + assert lib.assertMsg (coercedType.getSubModules == null) + "coercedTo: coercedType must not have submodules (it’s a ${ + coercedType.description})"; + mkOptionType rec { + name = "coercedTo"; + description = "${optionDescriptionPhrase (class: class == "noun") finalType} or ${optionDescriptionPhrase (class: class == "noun") coercedType} convertible to it"; + check = x: (coercedType.check x && finalType.check (coerceFunc x)) || finalType.check x; + merge = loc: defs: + let + coerceVal = val: + if coercedType.check val then coerceFunc val + else val; + in finalType.merge loc (map (def: def // { value = coerceVal def.value; }) defs); + emptyValue = finalType.emptyValue; + getSubOptions = finalType.getSubOptions; + getSubModules = finalType.getSubModules; + substSubModules = m: coercedTo coercedType coerceFunc (finalType.substSubModules m); + typeMerge = t1: t2: null; + functor = (defaultFunctor name) // { wrapped = finalType; }; + nestedTypes.coercedType = coercedType; + nestedTypes.finalType = finalType; + }; + + # Augment the given type with an additional type check function. + addCheck = elemType: check: elemType // { check = x: elemType.check x && check x; }; + + }; +}; + +in outer_types // outer_types.types diff --git a/versions.nix b/versions.nix new file mode 100644 index 000000000..986e7e5f9 --- /dev/null +++ b/versions.nix @@ -0,0 +1,64 @@ +/* Version string functions. */ +{ lib }: + +rec { + + /* Break a version string into its component parts. + + Example: + splitVersion "1.2.3" + => ["1" "2" "3"] + */ + splitVersion = builtins.splitVersion or (lib.splitString "."); + + /* Get the major version string from a string. + + Example: + major "1.2.3" + => "1" + */ + major = v: builtins.elemAt (splitVersion v) 0; + + /* Get the minor version string from a string. + + Example: + minor "1.2.3" + => "2" + */ + minor = v: builtins.elemAt (splitVersion v) 1; + + /* Get the patch version string from a string. + + Example: + patch "1.2.3" + => "3" + */ + patch = v: builtins.elemAt (splitVersion v) 2; + + /* Get string of the first two parts (major and minor) + of a version string. + + Example: + majorMinor "1.2.3" + => "1.2" + */ + majorMinor = v: + builtins.concatStringsSep "." + (lib.take 2 (splitVersion v)); + + /* Pad a version string with zeros to match the given number of components. + + Example: + pad 3 "1.2" + => "1.2.0" + pad 3 "1.3-rc1" + => "1.3.0-rc1" + pad 3 "1.2.3.4" + => "1.2.3" + */ + pad = n: version: let + numericVersion = lib.head (lib.splitString "-" version); + versionSuffix = lib.removePrefix numericVersion version; + in lib.concatStringsSep "." (lib.take n (lib.splitVersion numericVersion ++ lib.genList (_: "0") n)) + versionSuffix; + +} diff --git a/zip-int-bits.nix b/zip-int-bits.nix new file mode 100644 index 000000000..53efd2bb0 --- /dev/null +++ b/zip-int-bits.nix @@ -0,0 +1,39 @@ +/* Helper function to implement a fallback for the bit operators + `bitAnd`, `bitOr` and `bitXor` on older nix version. + See ./trivial.nix +*/ +f: x: y: + let + # (intToBits 6) -> [ 0 1 1 ] + intToBits = x: + if x == 0 || x == -1 then + [] + else + let + headbit = if (x / 2) * 2 != x then 1 else 0; # x & 1 + tailbits = if x < 0 then ((x + 1) / 2) - 1 else x / 2; # x >> 1 + in + [headbit] ++ (intToBits tailbits); + + # (bitsToInt [ 0 1 1 ] 0) -> 6 + # (bitsToInt [ 0 1 0 ] 1) -> -6 + bitsToInt = l: signum: + if l == [] then + (if signum == 0 then 0 else -1) + else + (builtins.head l) + (2 * (bitsToInt (builtins.tail l) signum)); + + xsignum = if x < 0 then 1 else 0; + ysignum = if y < 0 then 1 else 0; + zipListsWith' = fst: snd: + if fst==[] && snd==[] then + [] + else if fst==[] then + [(f xsignum (builtins.head snd))] ++ (zipListsWith' [] (builtins.tail snd)) + else if snd==[] then + [(f (builtins.head fst) ysignum )] ++ (zipListsWith' (builtins.tail fst) [] ) + else + [(f (builtins.head fst) (builtins.head snd))] ++ (zipListsWith' (builtins.tail fst) (builtins.tail snd)); + in + assert (builtins.isInt x) && (builtins.isInt y); + bitsToInt (zipListsWith' (intToBits x) (intToBits y)) (f xsignum ysignum) |