populate: add passage source type

Update README.md to fix broken link with infos to the `target` attribute

README.md

@@ -6,6 +6,7 @@ krops is a lightweight toolkit to deploy NixOS systems, remotely or locally.
 ## Some Features
 
 - store your secrets in [password store](https://www.passwordstore.org/)
+                     or [passage](https://github.com/FiloSottile/passage)
 - build your systems remotely
 - minimal overhead (it's basically just `nixos-rebuild switch`!)
 - run from custom nixpkgs branch/checkout/fork
@@ -96,7 +97,7 @@ pkgs.krops.writeDeploy "deploy" {
 }
 ```
 For more details about the `target` attribute, please check the `mkTarget`
-function in [lib/default.nix](lib/defaults.nix).
+function in [lib/default.nix](lib/default.nix).
 
 ### `backup` (optional, defaults to false)
 
@@ -125,13 +126,40 @@ architecture.
 
 ### `fast` (optional, defaults to false)
 
-Run `nixos-rebuild switch` immediately without building the system
-in a dedicated `nix build` step.
+Run `nixos-rebuild` immediately without building the system in a dedicated `nix
+build` step.
 
 ### `force` (optional, defaults to false)
 
 Create the sentinel file (`/var/src/.populate`) before syncing the new source.
 
+### `operation` (optional, defaults to "switch")
+
+Specifies which `nixos-rebuild` operation to perform.
+
+### `useNixOutputMonitor` (optional, defaults to `"opportunistic"`)
+
+Specifies when to pipe `nixos-rebuild`'s output to
+[nom](https://github.com/maralorn/nix-output-monitor).
+
+Supported values:
+
+* `"opportunistic"` (default) -
+  Use `nom` only if it is present on the target machine.
+
+* `"optimistic"` -
+  Use `nom`, assuming it is present on the target machine.
+
+* `"pessimistic"` -
+  Use `nom` via `nix-shell` on the target machine.
+
+* `true` -
+  Use `nom`.
+  If it is not present on the target machine, then use it via `nix-shell`.
+
+* `false` -
+  Don't use `nom`
+
 ## writeTest
 
 Very similiar to writeDeploy, but just builds the system on the target without
@@ -152,6 +180,10 @@ below `/var/src`, and executes `NIX_PATH=/var/src nix-build -A system '<nixpkgs/
 
 [see `writeDeploy`](#writeDeploy)
 
+### `trace` (optional, defaults to false)
+
+run nix-build with `--show-trace`
+
 ## writeCommand
 
 This can be used to run other commands than `nixos-rebuild` or pre/post build hooks.
@@ -267,6 +299,29 @@ Supported attributes:
   sub-directory in the password store.
 
 
+### `passage`
+
+The passage source type decrypts files from a local
+[passage store](https://github.com/FiloSottile/passage)
+and transfers them to the target using
+[`rsync`](https://rsync.samba.org/).
+
+Supported attributes:
+
+* `dir` -
+  Path to the passage store.
+  For a partial transfer, this may point to a subdirectory.
+  Example: `~/.passage/store/hosts/MYHOSTNAME`
+
+* `identities_file` (optional) -
+  Path to the identities file.
+  Defaults to `~/.passage/identities`.
+
+* `age` (optional) -
+  Path of the age binary.
+  Defaults to `age` (absolute path gets resolved using `passage`'s search path.)
+
+
 ### `pipe`
 
 Executes a local command, capture its stdout, and send that as a file to the

flake.lock

@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1689940971,
+        "narHash": "sha256-397xShPnFqPC59Bmpo3lS+/Aw0yoDRMACGo1+h2VJMo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9ca785644d067445a4aa749902b29ccef61f7476",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,29 @@
+{
+  description = "krops - krebs operations";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs, ... }:
+    let
+      supportedSystems = [
+        "x86_64-linux"
+        "i686-linux"
+        "aarch64-linux"
+        "riscv64-linux"
+      ];
+      forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
+    in
+    {
+      lib = forAllSystems (system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          krops = pkgs.callPackage ./pkgs/krops {};
+          populate = pkgs.callPackage ./pkgs/populate {};
+        in {
+          inherit populate;
+          inherit (krops) rebuild runShell withNixOutputMonitor writeCommand writeDeploy writeTest;
+        });
+    };
+}

lib/default.nix

@@ -57,9 +57,9 @@ let {
       elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null;
       filterNull = lib.filterAttrs (n: v: v != null);
     in {
-      user = lib.getEnv "LOGNAME";
+      user = lib.maybeEnv "LOGNAME" null;
       host = lib.maybeEnv "HOSTNAME" (lib.maybeHostName "localhost");
-      port = "22";
+      port = null;
       path = "/var/src";
       sudo = false;
       extraOptions = [];
@@ -70,6 +70,10 @@ let {
       path = elemAt' parse 6;
     } else s);
 
+    mkUserPortSSHOpts = target:
+      (lib.optionals (target.user != null) ["-l" target.user]) ++
+      (lib.optionals (target.port != null) ["-p" target.port]);
+
     shell = let
       isSafeChar = lib.testString "[-+./0-9:=A-Z_a-z]";
       quoteChar = c:

lib/types/populate.nix

@@ -39,6 +39,17 @@
         default = null;
         type = lib.types.nullOr source-types.pass;
       };
+      passage = lib.mkOption {
+        apply = x:
+          if lib.types.pathname.check x
+            then { dir = x; }
+            else x;
+        default = null;
+        type = lib.types.nullOr (lib.types.oneOf [
+          lib.types.pathname
+          source-types.passage
+        ]);
+      };
       pipe = lib.mkOption {
         apply = x:
           if lib.types.absolute-pathname.check x
@@ -160,6 +171,21 @@
         };
       };
     };
+    passage = lib.types.submodule {
+      options = {
+        age = lib.mkOption {
+          default = "age";
+          type = lib.types.pathname;
+        };
+        dir = lib.mkOption {
+          type = lib.types.pathname;
+        };
+        identities_file = lib.mkOption {
+          default = toString ~/.passage/identities;
+          type = lib.types.pathname;
+        };
+      };
+    };
     pipe = lib.types.submodule {
       options = {
         command = lib.mkOption {

pkgs/krops/default.nix

@@ -4,30 +4,72 @@ in
 
 { nix, openssh, populate, writers }: rec {
 
-  rebuild = args: target:
-    runShell target {} "nixos-rebuild -I ${lib.escapeShellArg target.path} ${
-      lib.concatMapStringsSep " " lib.escapeShellArg args
-    }";
+  rebuild = {
+    useNixOutputMonitor
+  }:
+  args: target:
+    runShell target {}
+      (withNixOutputMonitor target useNixOutputMonitor /* sh */ ''
+        NIX_PATH=${lib.escapeShellArg target.path} \
+        nixos-rebuild ${lib.escapeShellArgs args}
+      '');
 
   runShell = target: {
     allocateTTY ? false
   }: command:
     let
-      command' = if target.sudo then "sudo ${command}" else command;
+      command' = /* sh */ ''
+        ${lib.optionalString target.sudo "sudo"} \
+        /bin/sh -c ${lib.escapeShellArg command}
+      '';
     in
       if lib.isLocalTarget target
       then command'
       else
         writers.writeDash "krops.${target.host}.${lib.firstWord command}" ''
           exec ${openssh}/bin/ssh ${lib.escapeShellArgs (lib.flatten [
-            (lib.optionals (target.user != "") ["-l" target.user])
-            "-p" target.port
+            (lib.mkUserPortSSHOpts target)
             (if allocateTTY then "-t" else "-T")
             target.extraOptions
             target.host
-            command'])}
+            command'
+          ])}
         '';
 
+  withNixOutputMonitor = target: mode_: command: let
+    mode =
+      lib.getAttr (lib.typeOf mode_)  {
+        bool = lib.toJSON mode_;
+        string = mode_;
+      };
+  in /* sh */ ''
+    printf '# use nix-output-monitor: %s\n' ${lib.escapeShellArg mode} >&2
+    ${lib.getAttr mode rec {
+      opportunistic = /* sh */ ''
+        if command -v nom >/dev/null; then
+          ${optimistic}
+        else
+          ${false}
+        fi
+      '';
+      optimistic = /* sh */ ''
+        (${command}) 2>&1 | nom
+      '';
+      pessimistic = /* sh */ ''
+        NIX_PATH=${lib.escapeShellArg target.path} \
+        nix-shell -p nix-output-monitor --run ${lib.escapeShellArg optimistic}
+      '';
+      true = /* sh */ ''
+        if command -v nom >/dev/null; then
+          ${optimistic}
+        else
+          ${pessimistic}
+        fi
+      '';
+      false = command;
+    }}
+  '';
+
   writeCommand = name: {
     command ? (targetPath: "echo ${targetPath}"),
     backup ? false,
@@ -50,8 +92,10 @@ in
     crossDeploy ? false,
     fast ? null,
     force ? false,
+    operation ? "switch",
     source,
-    target
+    target,
+    useNixOutputMonitor ? "opportunistic"
   }: let
     buildTarget' =
       if buildTarget == null
@@ -65,8 +109,8 @@ in
         ${lib.optionalString (buildTarget' != target')
           (populate { inherit backup force source; target = buildTarget'; })}
         ${populate { inherit backup force source; target = target'; }}
-        ${rebuild ([
-          "switch"
+        ${rebuild { inherit useNixOutputMonitor; } ([
+          operation
         ] ++ lib.optionals crossDeploy [
           "--no-build-nix"
         ] ++ lib.optionals (buildTarget' != target') [
@@ -82,7 +126,8 @@ in
     backup ? false,
     force ? false,
     source,
-    target
+    target,
+    trace ? false
   }: let
     target' = lib.mkTarget target;
   in
@@ -95,7 +140,7 @@ in
           -A system \
           --keep-going \
           --no-out-link \
-          --show-trace \
+          ${lib.optionalString trace "--show-trace"} \
           '<nixpkgs/nixos>'
     '';
 }

pkgs/populate/default.nix

@@ -1,7 +1,7 @@
 with import ../../lib;
 with shell;
 
-{ coreutils, dash, findutils, git, jq, openssh, pass, rsync, writers }:
+{ coreutils, dash, findutils, git, jq, openssh, pass, passage, rsync, writers }:
 
 let
   check = { force, target }: let
@@ -48,14 +48,14 @@ let
     config = rsyncDefaultConfig // derivedConfig // sourceConfig;
     derivedConfig = {
       useChecksum =
-        if isDerivation source.path
+        if isStorePath source.path
           then true
           else rsyncDefaultConfig.useChecksum;
     };
     sourceConfig =
       filterAttrs (name: _: elem name (attrNames rsyncDefaultConfig)) source;
     sourcePath =
-      if isDerivation source.path
+      if isStorePath source.path
         then quote (toString source.path)
         else quote source.path;
   in
@@ -63,11 +63,16 @@ let
 
   pop.git = target: source: runShell target /* sh */ ''
     set -efu
+    # Remove target path if it doesn't look like a git worktree.
+    # This can happen e.g. when it had a different type earlier.
+    if ! test -e ${quote target.path}/.git; then
+      rm -fR ${quote target.path}
+    fi
     if ! test -e ${quote target.path}; then
       ${if source.shallow then /* sh */ ''
         git init ${quote target.path}
       '' else /* sh */ ''
-        git clone --recurse-submodules ${quote source.url} ${quote target.path}  
+        git clone --recurse-submodules ${quote source.url} ${quote target.path}
       ''}
     fi
     cd ${quote target.path}
@@ -114,7 +119,15 @@ let
     umask 0077
 
     if test -e ${quote source.dir}/.git; then
-      local_pass_info=${quote source.name}\ $(${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name})
+      local_pass_info=${quote source.name}\ $(
+        ${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name}
+        # we append a hash for every symlink, otherwise we would miss updates on
+        # files where the symlink points to
+        ${findutils}/bin/find ${quote source.dir}/${quote source.name} -type l \
+            -exec ${coreutils}/bin/realpath {} + |
+          ${coreutils}/bin/sort |
+          ${findutils}/bin/xargs -r -n 1 ${git}/bin/git -C ${quote source.dir} log -1 --format=%H
+      )
       remote_pass_info=$(${runShell target /* sh */ ''
         cat ${quote target.path}/.pass_info || :
       ''})
@@ -158,6 +171,37 @@ let
     ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"}
   '';
 
+  pop.passage = target: source: /* sh */ ''
+    set -efu
+
+    export PASSAGE_AGE=${quote source.age}
+    export PASSAGE_DIR=${quote source.dir}
+    export PASSAGE_IDENTITIES_FILE=${quote source.identities_file}
+
+    umask 0077
+
+    tmp_dir=$(${coreutils}/bin/mktemp -dt populate-passage.XXXXXXXX)
+    trap cleanup EXIT
+    cleanup() {
+      rm -fR "$tmp_dir"
+    }
+
+    ${findutils}/bin/find "$PASSAGE_DIR" -type f -name \*.age -follow |
+    while read -r age_path; do
+
+      rel_name=''${age_path#$PASSAGE_DIR}
+      rel_name=''${rel_name%.age}
+
+      tmp_path=$tmp_dir/$rel_name
+
+      ${coreutils}/bin/mkdir -p "$(${coreutils}/bin/dirname "$tmp_path")"
+      ${passage}/bin/passage show "$rel_name" > "$tmp_path"
+      ${coreutils}/bin/touch -r "$age_path" "$tmp_path"
+    done
+
+    ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"}
+  '';
+
   pop.pipe = target: source: /* sh */ ''
     ${quote source.command} | {
       ${runShell target /* sh */ "cat > ${quote target.path}"}
@@ -224,8 +268,7 @@ let
 
   ssh' = target: concatMapStringsSep " " quote (flatten [
     "${openssh}/bin/ssh"
-    (optionals (target.user != "") ["-l" target.user])
-    "-p" target.port
+    (mkUserPortSSHOpts target)
     "-T"
     target.extraOptions
   ]);