From 16399f6ebf05ae6ef7d21327ffd42c37ae446de5 Mon Sep 17 00:00:00 2001 From: Jess Date: Sat, 16 Nov 2024 07:25:39 +1300 Subject: [PATCH] rclone: add module Rclone is a command-line program to manage files on cloud storage, it also featrues support for FUSE mounts. "Users call rclone *"The Swiss army knife of cloud storage"* and *"Technology indistinguishable from magic"*" - https://rclone.org/ This module manages the configuration of rclone remotes. --- modules/lib/maintainers.nix | 9 ++ modules/modules.nix | 1 + modules/programs/rclone.nix | 142 ++++++++++++++++++ tests/integration/default.nix | 1 + .../integration/standalone/rclone/default.nix | 91 +++++++++++ tests/integration/standalone/rclone/home.nix | 13 ++ .../standalone/rclone/no-secrets.conf | 5 + .../standalone/rclone/no-secrets.nix | 10 ++ .../rclone/with-secrets-in-store.conf | 6 + .../rclone/with-secrets-in-store.nix | 18 +++ 10 files changed, 296 insertions(+) create mode 100644 modules/programs/rclone.nix create mode 100644 tests/integration/standalone/rclone/default.nix create mode 100644 tests/integration/standalone/rclone/home.nix create mode 100644 tests/integration/standalone/rclone/no-secrets.conf create mode 100644 tests/integration/standalone/rclone/no-secrets.nix create mode 100644 tests/integration/standalone/rclone/with-secrets-in-store.conf create mode 100644 tests/integration/standalone/rclone/with-secrets-in-store.nix diff --git a/modules/lib/maintainers.nix b/modules/lib/maintainers.nix index 91ecb47be..2bd2db28a 100644 --- a/modules/lib/maintainers.nix +++ b/modules/lib/maintainers.nix @@ -138,6 +138,15 @@ github = "Janik-Haag"; githubId = 80165193; }; + jess = { + name = "Jessica"; + email = "jess+hm@jessie.cafe"; + githubId = 43591752; + keys = [{ + longkeyid = "rsa3072/0xBA3350686C918606"; + fingerprint = "8092 3BD1 ECD0 E436 671D C8E9 BA33 5068 6C91 8606"; + }]; + }; jkarlson = { email = "jekarlson@gmail.com"; github = "jkarlson"; diff --git a/modules/modules.nix b/modules/modules.nix index 75f8ac461..05cdfe669 100644 --- a/modules/modules.nix +++ b/modules/modules.nix @@ -205,6 +205,7 @@ let ./programs/qutebrowser.nix ./programs/ranger.nix ./programs/rbw.nix + ./programs/rclone.nix ./programs/readline.nix ./programs/rio.nix ./programs/ripgrep.nix diff --git a/modules/programs/rclone.nix b/modules/programs/rclone.nix new file mode 100644 index 000000000..d9b7e83d0 --- /dev/null +++ b/modules/programs/rclone.nix @@ -0,0 +1,142 @@ +{ config, lib, pkgs, ... }: + +let + + cfg = config.programs.rclone; + iniFormat = pkgs.formats.ini { }; + +in { + options = { + programs.rclone = { + enable = lib.mkEnableOption "rclone"; + + package = lib.mkPackageOption pkgs "rclone" { }; + + remotes = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + options = { + config = lib.mkOption { + type = with lib.types; + attrsOf (nullOr (oneOf [ bool int float str ])); + default = { }; + description = '' + Regular configuration options as described in rclone's documentation + . When specifying options follow the formatting + process outlined here , namley: + - Remove the leading double-dash (--) from the rclone option name + - Replace hyphens (-) with underscores (_) + - Convert to lowercase + - Use the resulting string as your configuration key + + For example, the rclone option "--mega-hard-delete" would use "hard_delete" + as the config key. + + Security Note: Always use the {option}`secrets` option for sensitive data + instead of the {option}`config` option to prevent exposing credentials to + the world-readable Nix store. + ''; + example = lib.literalExpression '' + { + type = "mega"; # Required - specifies the remote type + user = "you@example.com"; + hard_delete = true; + }''; + }; + + secrets = lib.mkOption { + type = with lib.types; attrsOf str; + default = { }; + description = '' + Sensitive configuration values such as passwords, API keys, and tokens. These + must be provided as file paths to the secrets, which will be read at activation + time. + + Note: If using secret management solutions like agenix or sops-nix with + home-manager, you need to ensure their services are activated before switching + to this home-manager generation. Consider setting + {option}`systemd.user.startServices` to `"sd-switch"` for automatic service + startup. + ''; + example = lib.literalExpression '' + { + password = "/run/secrets/password"; + api_key = config.age.secrets.api-key.path; + }''; + }; + }; + }); + default = { }; + description = '' + An attribute set of remote configurations. Each remote consists of regular + configuration options and optional secrets. + + See for more information on configuring specific + remotes. + ''; + example = lib.literalExpression '' + { + b2 = { + config = { + type = "b2"; + hard_delete = true; + }; + secrets = { + # using sops + account = config.sops.secrets.b2-acc-id.path; + # using agenix + key = config.age.secrets.b2-key.path; + }; + }; + + server.config = { + type = "sftp"; + host = "server"; + user = "backup"; + key_file = "''${home.homeDirectory}/.ssh/id_ed25519"; + }; + }''; + }; + + writeAfter = lib.mkOption { + type = lib.types.str; + default = "reloadSystemd"; + description = '' + Controls when the rclone configuration is written during Home Manager activation. + You should not need to change this unless you have very specific activation order + requirements. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + home = { + packages = [ cfg.package ]; + + activation.createRcloneConfig = let + safeConfig = lib.pipe cfg.remotes [ + (lib.mapAttrs (_: v: v.config)) + (iniFormat.generate "rclone.conf@pre-secrets") + ]; + + # https://github.com/rclone/rclone/issues/8190 + injectSecret = remote: + lib.mapAttrsToList (secret: secretFile: '' + ${lib.getExe cfg.package} config update \ + ${remote.name} config_refresh_token=false \ + ${secret} $(cat ${secretFile}) \ + --quiet > /dev/null + '') remote.value.secrets or { }; + + injectAllSecrets = lib.concatMap injectSecret + (lib.mapAttrsToList lib.nameValuePair cfg.remotes); + in lib.mkIf (cfg.remotes != { }) + (lib.hm.dag.entryAfter [ "writeBoundary" cfg.writeAfter ] '' + run install $VERBOSE_ARG -D -m600 ${safeConfig} "${config.xdg.configHome}/rclone/rclone.conf" + ${lib.concatLines injectAllSecrets} + ''); + }; + }; + + meta.maintainers = with lib.hm.maintainers; [ jess ]; +} diff --git a/tests/integration/default.nix b/tests/integration/default.nix index 262f28faf..e8672f904 100644 --- a/tests/integration/default.nix +++ b/tests/integration/default.nix @@ -12,6 +12,7 @@ let tests = { kitty = runTest ./standalone/kitty.nix; nixos-basics = runTest ./nixos/basics.nix; + rclone = runTest ./standalone/rclone; standalone-flake-basics = runTest ./standalone/flake-basics.nix; standalone-standard-basics = runTest ./standalone/standard-basics.nix; }; diff --git a/tests/integration/standalone/rclone/default.nix b/tests/integration/standalone/rclone/default.nix new file mode 100644 index 000000000..067be8c26 --- /dev/null +++ b/tests/integration/standalone/rclone/default.nix @@ -0,0 +1,91 @@ +{ pkgs, ... }: + +{ + name = "rclone"; + + nodes.machine = { ... }: { + imports = [ "${pkgs.path}/nixos/modules/installer/cd-dvd/channel.nix" ]; + virtualisation.memorySize = 2048; + users.users.alice = { + isNormalUser = true; + description = "Alice Foobar"; + password = "foobar"; + uid = 1000; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("network.target") + machine.wait_for_unit("multi-user.target") + + home_manager = "${../../../..}" + + def login_as_alice(): + machine.wait_until_tty_matches("1", "login: ") + machine.send_chars("alice\n") + machine.wait_until_tty_matches("1", "Password: ") + machine.send_chars("foobar\n") + machine.wait_until_tty_matches("1", "alice\\@machine") + + def logout_alice(): + machine.send_chars("exit\n") + + def alice_cmd(cmd): + return f"su -l alice --shell /bin/sh -c $'export XDG_RUNTIME_DIR=/run/user/$UID ; {cmd}'" + + def succeed_as_alice(*cmds): + return machine.succeed(*map(alice_cmd,cmds)) + + def fail_as_alice(*cmds): + return machine.fail(*map(alice_cmd,cmds)) + + # Create a persistent login so that Alice has a systemd session. + login_as_alice() + + # Set up a home-manager channel. + succeed_as_alice(" ; ".join([ + "mkdir -p /home/alice/.nix-defexpr/channels", + f"ln -s {home_manager} /home/alice/.nix-defexpr/channels/home-manager" + ])) + + with subtest("Home Manager installation"): + succeed_as_alice("nix-shell \"\" -A install") + + succeed_as_alice("cp ${ + ./home.nix + } /home/alice/.config/home-manager/home.nix") + + with subtest("Generate with no secrets"): + succeed_as_alice("install -m644 ${ + ./no-secrets.nix + } /home/alice/.config/home-manager/test-remote.nix") + + actual = succeed_as_alice("home-manager switch") + expected = "Activating createRcloneConfig" + assert expected in actual, \ + f"expected home-manager switch to contain {expected}, but got {actual}" + + succeed_as_alice("diff -u ${ + ./no-secrets.conf + } /home/alice/.config/rclone/rclone.conf") + + with subtest("Generate with secrets from store"): + succeed_as_alice("install -m644 ${ + ./with-secrets-in-store.nix + } /home/alice/.config/home-manager/test-remote.nix") + + actual = succeed_as_alice("home-manager switch") + expected = "Activating createRcloneConfig" + assert expected in actual, \ + f"expected home-manager switch to contain {expected}, but got {actual}" + + succeed_as_alice("diff -u ${ + ./with-secrets-in-store.conf + } /home/alice/.config/rclone/rclone.conf") + + # TODO: verify correct activation order with the agenix and sops hm modules + + logout_alice() + ''; +} diff --git a/tests/integration/standalone/rclone/home.nix b/tests/integration/standalone/rclone/home.nix new file mode 100644 index 000000000..0bc0f8a51 --- /dev/null +++ b/tests/integration/standalone/rclone/home.nix @@ -0,0 +1,13 @@ +{ + imports = [ ./test-remote.nix ]; + + home.username = "alice"; + home.homeDirectory = "/home/alice"; + + home.stateVersion = "24.05"; # Please read the comment before changing. + + # Let Home Manager install and manage itself. + programs.home-manager.enable = true; + + programs.rclone.enable = true; +} diff --git a/tests/integration/standalone/rclone/no-secrets.conf b/tests/integration/standalone/rclone/no-secrets.conf new file mode 100644 index 000000000..d1fc379a1 --- /dev/null +++ b/tests/integration/standalone/rclone/no-secrets.conf @@ -0,0 +1,5 @@ +[alices-cool-remote] +host=backup-server +key_file=/key/path/foo +type=sftp +user=alice diff --git a/tests/integration/standalone/rclone/no-secrets.nix b/tests/integration/standalone/rclone/no-secrets.nix new file mode 100644 index 000000000..077a7ff15 --- /dev/null +++ b/tests/integration/standalone/rclone/no-secrets.nix @@ -0,0 +1,10 @@ +{ + programs.rclone.remotes = { + alices-cool-remote.config = { + type = "sftp"; + host = "backup-server"; + user = "alice"; + key_file = "/key/path/foo"; + }; + }; +} diff --git a/tests/integration/standalone/rclone/with-secrets-in-store.conf b/tests/integration/standalone/rclone/with-secrets-in-store.conf new file mode 100644 index 000000000..12e2777d1 --- /dev/null +++ b/tests/integration/standalone/rclone/with-secrets-in-store.conf @@ -0,0 +1,6 @@ +[alices-cool-remote-v2] +hard_delete = true +type = b2 +account = super-secret-account-id +key = api-key-from-file + diff --git a/tests/integration/standalone/rclone/with-secrets-in-store.nix b/tests/integration/standalone/rclone/with-secrets-in-store.nix new file mode 100644 index 000000000..4c47a60e2 --- /dev/null +++ b/tests/integration/standalone/rclone/with-secrets-in-store.nix @@ -0,0 +1,18 @@ +{ pkgs, ... }: { + programs.rclone.remotes = { + alices-cool-remote-v2 = { + config = { + type = "b2"; + hard_delete = true; + }; + secrets = { + account = "${pkgs.writeText "acc" '' + super-secret-account-id + ''}"; + key = "${pkgs.writeText "key" '' + api-key-from-file + ''}"; + }; + }; + }; +}