populate: add passage source type

Update README.md to fix broken link with infos to the `target` attribute

README.md

@@ -1,4 +1,4 @@
-# krops (krebs ops)
+# krops (krebs operations)
 
 krops is a lightweight toolkit to deploy NixOS systems, remotely or locally.
 
@@ -6,7 +6,8 @@ krops is a lightweight toolkit to deploy NixOS systems, remotely or locally.
 ## Some Features
 
 - store your secrets in [password store](https://www.passwordstore.org/)
-- build your system remotely
+                     or [passage](https://github.com/FiloSottile/passage)
+- build your systems remotely
 - minimal overhead (it's basically just `nixos-rebuild switch`!)
 - run from custom nixpkgs branch/checkout/fork
 
@@ -19,8 +20,8 @@ Create a file named `krops.nix` (name doesn't matter) with following content:
 let
   krops = (import <nixpkgs> {}).fetchgit {
     url = https://cgit.krebsco.de/krops/;
-    rev = "v1.17.0";
+    rev = "v1.25.0";
-    sha256 = "150jlz0hlb3ngf9a1c9xgcwzz1zz8v2lfgnzw08l3ajlaaai8smd";
+    sha256 = "07mg3iaqjf1w49vmwfchi7b1w55bh7rvsbgicp2m47gnj9alwdb6";
   };
 
   lib = import "${krops}/lib";
@@ -89,14 +90,14 @@ pkgs.krops.writeDeploy "deploy" {
   source = /* ... */;
   target = lib.mkTarget "user@host/path" // {
     extraOptions = [
-      "-oLogLevel=DEBUG"
+      "-o" "LogLevel=DEBUG"
     ];
     sudo = true;
   };
 }
 ```
 For more details about the `target` attribute, please check the `mkTarget`
-function in [lib/default.nix](lib/defaults.nix).
+function in [lib/default.nix](lib/default.nix).
 
 ### `backup` (optional, defaults to false)
 
@@ -125,13 +126,40 @@ architecture.
 
 ### `fast` (optional, defaults to false)
 
-Run `nixos-rebuild switch` immediately without building the system
+Run `nixos-rebuild` immediately without building the system in a dedicated `nix
-in a dedicated `nix build` step.
+build` step.
 
 ### `force` (optional, defaults to false)
 
 Create the sentinel file (`/var/src/.populate`) before syncing the new source.
 
+### `operation` (optional, defaults to "switch")
+
+Specifies which `nixos-rebuild` operation to perform.
+
+### `useNixOutputMonitor` (optional, defaults to `"opportunistic"`)
+
+Specifies when to pipe `nixos-rebuild`'s output to
+[nom](https://github.com/maralorn/nix-output-monitor).
+
+Supported values:
+
+* `"opportunistic"` (default) -
+  Use `nom` only if it is present on the target machine.
+
+* `"optimistic"` -
+  Use `nom`, assuming it is present on the target machine.
+
+* `"pessimistic"` -
+  Use `nom` via `nix-shell` on the target machine.
+
+* `true` -
+  Use `nom`.
+  If it is not present on the target machine, then use it via `nix-shell`.
+
+* `false` -
+  Don't use `nom`
+
 ## writeTest
 
 Very similiar to writeDeploy, but just builds the system on the target without
@@ -152,6 +180,10 @@ below `/var/src`, and executes `NIX_PATH=/var/src nix-build -A system '<nixpkgs/
 
 [see `writeDeploy`](#writeDeploy)
 
+### `trace` (optional, defaults to false)
+
+run nix-build with `--show-trace`
+
 ## writeCommand
 
 This can be used to run other commands than `nixos-rebuild` or pre/post build hooks.
@@ -185,6 +217,10 @@ pkgs.krops.writeCommand "deploy-with-swap" {
 
 [see `writeDeploy`](#writeDeploy)
 
+### `allocateTTY` (optional, defaults to false)
+
+whether the ssh session should do a pseudo-terminal allocation.
+sets `-t` on the ssh command.
 
 ## Source Types
 
@@ -206,13 +242,15 @@ using [`rsync`](https://rsync.samba.org/).
 Supported attributes:
 
 * `path` -
-  absolute path to files that should by transfered
+  absolute path to files that should by transferred.
 
 * `useChecksum` (optional) -
   boolean that controls whether file contents should be checked to decide
   whether a file has changed.  This is useful when `path` points at files
   with mangled timestamps, e.g. the Nix store.
 
+  The default value is `true` if `path` is a derivation, and `false` otherwise.
+
 * `filters` (optional)
   List of filters that should be passed to [`rsync`](https://rsync.samba.org/).
   Filters are specified as attribute sets with the attributes `type` and
@@ -261,6 +299,29 @@ Supported attributes:
   sub-directory in the password store.
 
 
+### `passage`
+
+The passage source type decrypts files from a local
+[passage store](https://github.com/FiloSottile/passage)
+and transfers them to the target using
+[`rsync`](https://rsync.samba.org/).
+
+Supported attributes:
+
+* `dir` -
+  Path to the passage store.
+  For a partial transfer, this may point to a subdirectory.
+  Example: `~/.passage/store/hosts/MYHOSTNAME`
+
+* `identities_file` (optional) -
+  Path to the identities file.
+  Defaults to `~/.passage/identities`.
+
+* `age` (optional) -
+  Path of the age binary.
+  Defaults to `age` (absolute path gets resolved using `passage`'s search path.)
+
+
 ### `pipe`
 
 Executes a local command, capture its stdout, and send that as a file to the
@@ -292,6 +353,6 @@ Supported attributes:
 Comments, questions, pull-requests and patches, etc. are very welcome, and can be directed
 at:
 
-- IRC: #krebs at freenode
+- IRC: #krebs at hackint
 - Mail: [spam@krebsco.de](mailto:spam@krebsco.de)
 - Github: https://github.com/krebs/krops/

ci.nix

@@ -5,7 +5,7 @@ let
   pkgs = import "${krops}/pkgs" {};
 
   source = lib.evalSource [{
-    nixos-config.file = toString (pkgs.writeText "nixos-config" ''
+    nixos-config.file = pkgs.writeText "nixos-config" ''
       { pkgs, ... }: {
 
         fileSystems."/" = { device = "/dev/sda1"; };
@@ -13,7 +13,7 @@ let
         services.openssh.enable = true;
         environment.systemPackages = [ pkgs.git ];
       }
-    '');
+    '';
     nixpkgs.symlink = toString <nixpkgs>;
   }];
 in {

flake.lock

@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1689940971,
+        "narHash": "sha256-397xShPnFqPC59Bmpo3lS+/Aw0yoDRMACGo1+h2VJMo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9ca785644d067445a4aa749902b29ccef61f7476",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,29 @@
+{
+  description = "krops - krebs operations";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs, ... }:
+    let
+      supportedSystems = [
+        "x86_64-linux"
+        "i686-linux"
+        "aarch64-linux"
+        "riscv64-linux"
+      ];
+      forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
+    in
+    {
+      lib = forAllSystems (system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          krops = pkgs.callPackage ./pkgs/krops {};
+          populate = pkgs.callPackage ./pkgs/populate {};
+        in {
+          inherit populate;
+          inherit (krops) rebuild runShell withNixOutputMonitor writeCommand writeDeploy writeTest;
+        });
+    };
+}

lib/default.nix

@@ -28,13 +28,20 @@ let {
       # This function's return value can be used as pkgs.populate input.
       source: sanitize (eval source).config.source;
 
-    getHostName = let
+    maybeHostName = default: let
       # We're parsing /etc/hostname here because reading
       # /proc/sys/kernel/hostname yields ""
-      y = lib.filter lib.types.label.check (lib.splitString "\n" (lib.readFile /etc/hostname));
+      path = "/etc/hostname";
+      lines = lib.splitString "\n" (lib.readFile path);
+      hostNames = lib.filter lib.types.label.check lines;
     in
-      if lib.length y != 1 then throw "malformed /etc/hostname" else
+      if lib.pathExists path then
-      lib.elemAt y 0;
+        if lib.length hostNames == 1 then
+          lib.head hostNames
+        else
+          lib.trace "malformed ${path}" default
+      else
+        default;
 
     firstWord = s:
       lib.head (lib.match "^([^[:space:]]*).*" s);
@@ -50,9 +57,9 @@ let {
       elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null;
       filterNull = lib.filterAttrs (n: v: v != null);
     in {
-      user = lib.getEnv "LOGNAME";
+      user = lib.maybeEnv "LOGNAME" null;
-      host = lib.maybeEnv "HOSTNAME" lib.getHostName;
+      host = lib.maybeEnv "HOSTNAME" (lib.maybeHostName "localhost");
-      port = "22";
+      port = null;
       path = "/var/src";
       sudo = false;
       extraOptions = [];
@@ -63,6 +70,10 @@ let {
       path = elemAt' parse 6;
     } else s);
 
+    mkUserPortSSHOpts = target:
+      (lib.optionals (target.user != null) ["-l" target.user]) ++
+      (lib.optionals (target.port != null) ["-p" target.port]);
+
     shell = let
       isSafeChar = lib.testString "[-+./0-9:=A-Z_a-z]";
       quoteChar = c:

lib/types/populate.nix

@@ -21,11 +21,15 @@
       };
       file = lib.mkOption {
         apply = x:
-          if lib.types.absolute-pathname.check x
+          if lib.types.absolute-pathname.check x || lib.types.package.check x
             then { path = x; }
             else x;
         default = null;
-        type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file);
+        type = lib.types.nullOr (lib.types.oneOf [
+          lib.types.absolute-pathname
+          lib.types.package
+          source-types.file
+        ]);
       };
       git = lib.mkOption {
         default = null;
@@ -35,6 +39,17 @@
         default = null;
         type = lib.types.nullOr source-types.pass;
       };
+      passage = lib.mkOption {
+        apply = x:
+          if lib.types.pathname.check x
+            then { dir = x; }
+            else x;
+        default = null;
+        type = lib.types.nullOr (lib.types.oneOf [
+          lib.types.pathname
+          source-types.passage
+        ]);
+      };
       pipe = lib.mkOption {
         apply = x:
           if lib.types.absolute-pathname.check x
@@ -156,6 +171,21 @@
         };
       };
     };
+    passage = lib.types.submodule {
+      options = {
+        age = lib.mkOption {
+          default = "age";
+          type = lib.types.pathname;
+        };
+        dir = lib.mkOption {
+          type = lib.types.pathname;
+        };
+        identities_file = lib.mkOption {
+          default = toString ~/.passage/identities;
+          type = lib.types.pathname;
+        };
+      };
+    };
     pipe = lib.types.submodule {
       options = {
         command = lib.mkOption {

pkgs/krops/default.nix

@@ -4,40 +4,77 @@ in
 
 { nix, openssh, populate, writers }: rec {
 
-  build = target:
+  rebuild = {
-    runShell target (lib.concatStringsSep " " [
+    useNixOutputMonitor
-      "nix build"
+  }:
-      "-I ${lib.escapeShellArg target.path}"
+  args: target:
-      "--no-link -f '<nixpkgs/nixos>'"
+    runShell target {}
-      "config.system.build.toplevel"
+      (withNixOutputMonitor target useNixOutputMonitor /* sh */ ''
-    ]);
+        NIX_PATH=${lib.escapeShellArg target.path} \
+        nixos-rebuild ${lib.escapeShellArgs args}
+      '');
 
-  rebuild = args: target:
+  runShell = target: {
-    runShell target "nixos-rebuild -I ${lib.escapeShellArg target.path} ${
+    allocateTTY ? false
-      lib.concatMapStringsSep " " lib.escapeShellArg args
+  }: command:
-    }";
-
-  runShell = target: command:
     let
-      command' = if target.sudo then "sudo ${command}" else command;
+      command' = /* sh */ ''
+        ${lib.optionalString target.sudo "sudo"} \
+        /bin/sh -c ${lib.escapeShellArg command}
+      '';
     in
       if lib.isLocalTarget target
       then command'
       else
         writers.writeDash "krops.${target.host}.${lib.firstWord command}" ''
           exec ${openssh}/bin/ssh ${lib.escapeShellArgs (lib.flatten [
-            (lib.optionals (target.user != "") ["-l" target.user])
+            (lib.mkUserPortSSHOpts target)
-            "-p" target.port
+            (if allocateTTY then "-t" else "-T")
-            "-T"
             target.extraOptions
             target.host
-            command'])}
+            command'
+          ])}
         '';
 
+  withNixOutputMonitor = target: mode_: command: let
+    mode =
+      lib.getAttr (lib.typeOf mode_)  {
+        bool = lib.toJSON mode_;
+        string = mode_;
+      };
+  in /* sh */ ''
+    printf '# use nix-output-monitor: %s\n' ${lib.escapeShellArg mode} >&2
+    ${lib.getAttr mode rec {
+      opportunistic = /* sh */ ''
+        if command -v nom >/dev/null; then
+          ${optimistic}
+        else
+          ${false}
+        fi
+      '';
+      optimistic = /* sh */ ''
+        (${command}) 2>&1 | nom
+      '';
+      pessimistic = /* sh */ ''
+        NIX_PATH=${lib.escapeShellArg target.path} \
+        nix-shell -p nix-output-monitor --run ${lib.escapeShellArg optimistic}
+      '';
+      true = /* sh */ ''
+        if command -v nom >/dev/null; then
+          ${optimistic}
+        else
+          ${pessimistic}
+        fi
+      '';
+      false = command;
+    }}
+  '';
+
   writeCommand = name: {
     command ? (targetPath: "echo ${targetPath}"),
     backup ? false,
     force ? false,
+    allocateTTY ? false,
     source,
     target
   }: let
@@ -46,17 +83,19 @@ in
     writers.writeDash name ''
       set -efu
       ${populate { inherit backup force source; target = target'; }}
-      ${runShell target' (command target'.path)}
+      ${runShell target' { inherit allocateTTY; } (command target'.path)}
     '';
 
   writeDeploy = name: {
     backup ? false,
     buildTarget ? null,
     crossDeploy ? false,
-    fast ? false,
+    fast ? null,
     force ? false,
+    operation ? "switch",
     source,
-    target
+    target,
+    useNixOutputMonitor ? "opportunistic"
   }: let
     buildTarget' =
       if buildTarget == null
@@ -64,32 +103,31 @@ in
         else lib.mkTarget buildTarget;
     target' = lib.mkTarget target;
   in
-    writers.writeDash name ''
+    lib.traceIf (fast != null) "writeDeploy: it's now always fast, setting the `fast` attribute is deprecated and will be removed in future" (
-      set -efu
+      writers.writeDash name ''
-      ${lib.optionalString (buildTarget' != target')
+        set -efu
-        (populate { inherit backup force source; target = buildTarget'; })}
+        ${lib.optionalString (buildTarget' != target')
-      ${populate { inherit backup force source; target = target'; }}
+          (populate { inherit backup force source; target = buildTarget'; })}
-      ${lib.optionalString (! fast) ''
+        ${populate { inherit backup force source; target = target'; }}
-        ${rebuild ["dry-build"] buildTarget'}
+        ${rebuild { inherit useNixOutputMonitor; } ([
-        ${build buildTarget'}
+          operation
-      ''}
+        ] ++ lib.optionals crossDeploy [
-      ${rebuild ([
+          "--no-build-nix"
-        "switch"
+        ] ++ lib.optionals (buildTarget' != target') [
-      ] ++ lib.optionals crossDeploy [
+          "--build-host" "${buildTarget'.user}@${buildTarget'.host}"
-        "--no-build-nix"
+          "--target-host" "${target'.user}@${target'.host}"
-      ] ++ lib.optionals (buildTarget' != target') [
+        ] ++ lib.optionals target'.sudo [
-        "--build-host" "${buildTarget'.user}@${buildTarget'.host}"
+          "--use-remote-sudo"
-        "--target-host" "${target'.user}@${target'.host}"
+        ]) buildTarget'}
-      ] ++ lib.optionals target'.sudo [
+      ''
-        "--use-remote-sudo"
+    );
-      ]) buildTarget'}
-    '';
 
   writeTest = name: {
     backup ? false,
     force ? false,
     source,
-    target
+    target,
+    trace ? false
   }: let
     target' = lib.mkTarget target;
   in
@@ -102,7 +140,7 @@ in
           -A system \
           --keep-going \
           --no-out-link \
-          --show-trace \
+          ${lib.optionalString trace "--show-trace"} \
           '<nixpkgs/nixos>'
     '';
 }

pkgs/populate/default.nix

@@ -1,7 +1,7 @@
 with import ../../lib;
 with shell;
 
-{ coreutils, dash, findutils, git, jq, openssh, pass, rsync, writers }:
+{ coreutils, dash, findutils, git, jq, openssh, pass, passage, rsync, writers }:
 
 let
   check = { force, target }: let
@@ -45,18 +45,34 @@ let
   '';
 
   pop.file = target: source: let
-    configAttrs = ["useChecksum" "exclude" "filters" "deleteExcluded"];
+    config = rsyncDefaultConfig // derivedConfig // sourceConfig;
-    config = filterAttrs (name: _: elem name configAttrs) source;
+    derivedConfig = {
+      useChecksum =
+        if isStorePath source.path
+          then true
+          else rsyncDefaultConfig.useChecksum;
+    };
+    sourceConfig =
+      filterAttrs (name: _: elem name (attrNames rsyncDefaultConfig)) source;
+    sourcePath =
+      if isStorePath source.path
+        then quote (toString source.path)
+        else quote source.path;
   in
-    rsync' target config (quote source.path);
+    rsync' target config sourcePath;
 
   pop.git = target: source: runShell target /* sh */ ''
     set -efu
+    # Remove target path if it doesn't look like a git worktree.
+    # This can happen e.g. when it had a different type earlier.
+    if ! test -e ${quote target.path}/.git; then
+      rm -fR ${quote target.path}
+    fi
     if ! test -e ${quote target.path}; then
       ${if source.shallow then /* sh */ ''
         git init ${quote target.path}
       '' else /* sh */ ''
-        git clone --recurse-submodules ${quote source.url} ${quote target.path}  
+        git clone --recurse-submodules ${quote source.url} ${quote target.path}
       ''}
     fi
     cd ${quote target.path}
@@ -103,7 +119,15 @@ let
     umask 0077
 
     if test -e ${quote source.dir}/.git; then
-      local_pass_info=${quote source.name}\ $(${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name})
+      local_pass_info=${quote source.name}\ $(
+        ${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name}
+        # we append a hash for every symlink, otherwise we would miss updates on
+        # files where the symlink points to
+        ${findutils}/bin/find ${quote source.dir}/${quote source.name} -type l \
+            -exec ${coreutils}/bin/realpath {} + |
+          ${coreutils}/bin/sort |
+          ${findutils}/bin/xargs -r -n 1 ${git}/bin/git -C ${quote source.dir} log -1 --format=%H
+      )
       remote_pass_info=$(${runShell target /* sh */ ''
         cat ${quote target.path}/.pass_info || :
       ''})
@@ -119,28 +143,63 @@ let
       rm -fR "$tmp_dir"
     }
 
-    ${findutils}/bin/find ${quote passPrefix} -type f -follow |
+    ${findutils}/bin/find ${quote passPrefix} -type f -follow ! -name .gpg-id |
     while read -r gpg_path; do
 
       rel_name=''${gpg_path#${quote passPrefix}}
       rel_name=''${rel_name%.gpg}
 
       pass_date=$(
-        ${git}/bin/git -C ${quote source.dir} log -1 --format=%aI "$gpg_path"
+        if test -e ${quote source.dir}/.git; then
+          ${git}/bin/git -C ${quote source.dir} log -1 --format=%aI "$gpg_path"
+        fi
       )
       pass_name=${quote source.name}/$rel_name
       tmp_path=$tmp_dir/$rel_name
 
       ${coreutils}/bin/mkdir -p "$(${coreutils}/bin/dirname "$tmp_path")"
       PASSWORD_STORE_DIR=${quote source.dir} ${pass}/bin/pass show "$pass_name" > "$tmp_path"
-      ${coreutils}/bin/touch -d "$pass_date" "$tmp_path"
+      if [ -n "$pass_date" ]; then
+        ${coreutils}/bin/touch -d "$pass_date" "$tmp_path"
+      fi
     done
 
     if test -n "''${local_pass_info-}"; then
       echo "$local_pass_info" > "$tmp_dir"/.pass_info
     fi
 
-    ${rsync' target {} /* sh */ "$tmp_dir"}
+    ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"}
+  '';
+
+  pop.passage = target: source: /* sh */ ''
+    set -efu
+
+    export PASSAGE_AGE=${quote source.age}
+    export PASSAGE_DIR=${quote source.dir}
+    export PASSAGE_IDENTITIES_FILE=${quote source.identities_file}
+
+    umask 0077
+
+    tmp_dir=$(${coreutils}/bin/mktemp -dt populate-passage.XXXXXXXX)
+    trap cleanup EXIT
+    cleanup() {
+      rm -fR "$tmp_dir"
+    }
+
+    ${findutils}/bin/find "$PASSAGE_DIR" -type f -name \*.age -follow |
+    while read -r age_path; do
+
+      rel_name=''${age_path#$PASSAGE_DIR}
+      rel_name=''${rel_name%.age}
+
+      tmp_path=$tmp_dir/$rel_name
+
+      ${coreutils}/bin/mkdir -p "$(${coreutils}/bin/dirname "$tmp_path")"
+      ${passage}/bin/passage show "$rel_name" > "$tmp_path"
+      ${coreutils}/bin/touch -r "$age_path" "$tmp_path"
+    done
+
+    ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"}
   '';
 
   pop.pipe = target: source: /* sh */ ''
@@ -168,17 +227,17 @@ let
       source_path=$source_path/
     fi
     ${rsync}/bin/rsync \
-        ${optionalString (config.useChecksum or false) /* sh */ "--checksum"} \
+        ${optionalString config.useChecksum /* sh */ "--checksum"} \
         ${optionalString target.sudo /* sh */ "--rsync-path=\"sudo rsync\""} \
         ${concatMapStringsSep " "
           (pattern: /* sh */ "--exclude ${quote pattern}")
-          (config.exclude or [])} \
+          config.exclude} \
         ${concatMapStringsSep " "
           (filter: /* sh */ "--${filter.type} ${quote filter.pattern}")
-          (config.filters or [])} \
+          config.filters} \
         -e ${quote (ssh' target)} \
         -vFrlptD \
-        ${optionalString (config.deleteExcluded or true) /* sh */ "--delete-excluded"} \
+        ${optionalString config.deleteExcluded /* sh */ "--delete-excluded"} \
         "$source_path" \
         ${quote (
           optionalString (!isLocalTarget target) (
@@ -190,6 +249,13 @@ let
       >&2
   '';
 
+  rsyncDefaultConfig = {
+    useChecksum = false;
+    exclude = [];
+    filters = [];
+    deleteExcluded = true;
+  };
+
   runShell = target: command:
     if isLocalTarget target
       then command
@@ -202,8 +268,7 @@ let
 
   ssh' = target: concatMapStringsSep " " quote (flatten [
     "${openssh}/bin/ssh"
-    (optionals (target.user != "") ["-l" target.user])
+    (mkUserPortSSHOpts target)
-    "-p" target.port
     "-T"
     target.extraOptions
   ]);