From e89cf20d4310070a877c2e24a287659546b561c9 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 28 Feb 2018 21:02:21 +0100 Subject: [PATCH] import stockholm's deployment tools https://cgit.krebsco.de/stockholm 877b4104370c1ea9698a449e376e2842d7c372fd --- lib/default.nix | 56 ++++++++ lib/types/default.nix | 8 ++ lib/types/populate.nix | 93 +++++++++++++ lib/types/posix.nix | 54 ++++++++ pkgs/default.nix | 7 + pkgs/kops/default.nix | 48 +++++++ pkgs/overlay.nix | 18 +++ pkgs/populate/default.nix | 20 +++ pkgs/populate/populate.sh | 276 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 580 insertions(+) create mode 100644 lib/default.nix create mode 100644 lib/types/default.nix create mode 100644 lib/types/populate.nix create mode 100644 lib/types/posix.nix create mode 100644 pkgs/default.nix create mode 100644 pkgs/kops/default.nix create mode 100644 pkgs/overlay.nix create mode 100644 pkgs/populate/default.nix create mode 100755 pkgs/populate/populate.sh diff --git a/lib/default.nix b/lib/default.nix new file mode 100644 index 0000000..7197fe9 --- /dev/null +++ b/lib/default.nix @@ -0,0 +1,56 @@ +let { + + body = lib; + + lib = nixpkgs.lib // builtins // { + + evalSource = let + eval = source: lib.evalModules { + modules = lib.singleton { + _file = toString ./.; + imports = map (source: { inherit source; }) (lib.toList source); + options.source = lib.mkOption { + default = {}; + type = lib.types.attrsOf lib.types.source; + }; + }; + }; + sanitize = x: lib.getAttr (lib.typeOf x) { + set = lib.mapAttrs + (lib.const sanitize) + (lib.filterAttrs + (name: value: name != "_module" && value != null) x); + string = x; + }; + in + # This function's return value can be used as pkgs.populate input. + source: sanitize (eval source).config.source; + + getHostName = let + # We're parsing /etc/hostname here because reading + # /proc/sys/kernel/hostname yields "" + y = lib.filter lib.types.label.check (lib.splitString "\n" (lib.readFile /etc/hostname)); + in + if lib.length y != 1 then throw "malformed /etc/hostname" else + lib.elemAt y 0; + + mkTarget = s: let + default = defVal: val: if val != null then val else defVal; + parse = lib.match "(([^@]+)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s; + elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null; + in { + user = default (lib.getEnv "LOGNAME") (elemAt' parse 1); + host = default (lib.maybeEnv "HOSTNAME" lib.getHostName) (elemAt' parse 3); + port = default "22" /* "ssh"? */ (elemAt' parse 5); + path = default "/var/src" /* no default? */ (elemAt' parse 6); + }; + + test = re: x: lib.isString x && lib.testString re x; + testString = re: x: lib.match re x != null; + + types = nixpkgs.lib.types // import ./types { lib = body; }; + }; + + nixpkgs.lib = import ; + +} diff --git a/lib/types/default.nix b/lib/types/default.nix new file mode 100644 index 0000000..c4bf517 --- /dev/null +++ b/lib/types/default.nix @@ -0,0 +1,8 @@ +{ lib }@args: let { + + body = lib.foldl' (res: path: res // import path args) {} [ + ./populate.nix + ./posix.nix + ]; + +} diff --git a/lib/types/populate.nix b/lib/types/populate.nix new file mode 100644 index 0000000..3b13df0 --- /dev/null +++ b/lib/types/populate.nix @@ -0,0 +1,93 @@ +{ lib }: rec { + + source = lib.types.submodule ({ config, ... }: { + options = { + type = let + known-types = lib.attrNames source-types; + type-candidates = lib.filter (k: config.${k} != null) known-types; + in lib.mkOption { + default = if lib.length type-candidates == 1 + then lib.head type-candidates + else throw "cannot determine type"; + type = lib.types.enum known-types; + }; + file = lib.mkOption { + apply = x: + if lib.types.absolute-pathname.check x + then { path = x; } + else x; + default = null; + type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file); + }; + git = lib.mkOption { + default = null; + type = lib.types.nullOr source-types.git; + }; + pass = lib.mkOption { + default = null; + type = lib.types.nullOr source-types.pass; + }; + pipe = lib.mkOption { + apply = x: + if lib.types.absolute-pathname.check x + then { command = x; } + else x; + default = null; + type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.pipe); + }; + symlink = lib.mkOption { + apply = x: + if lib.types.pathname.check x + then { target = x; } + else x; + default = null; + type = lib.types.nullOr (lib.types.either lib.types.pathname source-types.symlink); + }; + }; + }); + + source-types = { + file = lib.types.submodule { + options = { + path = lib.mkOption { + type = lib.types.absolute-pathname; + }; + }; + }; + git = lib.types.submodule { + options = { + ref = lib.mkOption { + type = lib.types.str; # TODO lib.types.git.ref + }; + url = lib.mkOption { + type = lib.types.str; # TODO lib.types.git.url + }; + }; + }; + pass = lib.types.submodule { + options = { + dir = lib.mkOption { + type = lib.types.absolute-pathname; + }; + name = lib.mkOption { + type = lib.types.pathname; # TODO relative-pathname + }; + }; + }; + pipe = lib.types.submodule { + options = { + command = lib.mkOption { + type = lib.types.absolute-pathname; + }; + }; + }; + symlink = lib.types.submodule { + options = { + target = lib.mkOption { + type = lib.types.pathname; # TODO relative-pathname + }; + }; + }; + }; + +} diff --git a/lib/types/posix.nix b/lib/types/posix.nix new file mode 100644 index 0000000..e8f464e --- /dev/null +++ b/lib/types/posix.nix @@ -0,0 +1,54 @@ +{ lib }: rec { + + # RFC952, B. Lexical grammar, + hostname = lib.mkOptionType { + name = "hostname"; + check = x: lib.isString x && lib.all label.check (lib.splitString "." x); + merge = lib.mergeOneOption; + }; + + # RFC952, B. Lexical grammar, + # RFC1123, 2.1 Host Names and Numbers + label = lib.mkOptionType { + name = "label"; + # TODO case-insensitive labels + check = lib.test "[0-9A-Za-z]([0-9A-Za-z-]*[0-9A-Za-z])?"; + merge = lib.mergeOneOption; + }; + + # POSIX.1‐2013, 3.278 Portable Filename Character Set + filename = lib.mkOptionType { + name = "POSIX filename"; + check = lib.test "([0-9A-Za-z._])[0-9A-Za-z._-]*"; + merge = lib.mergeOneOption; + }; + + # POSIX.1‐2013, 3.2 Absolute Pathname + absolute-pathname = lib.mkOptionType { + name = "POSIX absolute pathname"; + check = x: lib.isString x && lib.substring 0 1 x == "/" && pathname.check x; + merge = lib.mergeOneOption; + }; + + # POSIX.1‐2013, 3.267 Pathname + pathname = lib.mkOptionType { + name = "POSIX pathname"; + check = x: + let + # The filter is used to normalize paths, i.e. to remove duplicated and + # trailing slashes. It also removes leading slashes, thus we have to + # check for "/" explicitly below. + xs = lib.filter (s: lib.stringLength s > 0) (lib.splitString "/" x); + in + lib.isString x && (x == "/" || (lib.length xs > 0 && lib.all filename.check xs)); + merge = lib.mergeOneOption; + }; + + # POSIX.1-2013, 3.431 User Name + username = lib.mkOptionType { + name = "POSIX username"; + check = filename.check; + merge = lib.mergeOneOption; + }; + +} diff --git a/pkgs/default.nix b/pkgs/default.nix new file mode 100644 index 0000000..639ed13 --- /dev/null +++ b/pkgs/default.nix @@ -0,0 +1,7 @@ +{ overlays ? [], ... }@args: + +import (args // { + overlays = overlays ++ [ + (import ./overlay.nix) + ]; +}) diff --git a/pkgs/kops/default.nix b/pkgs/kops/default.nix new file mode 100644 index 0000000..fc52327 --- /dev/null +++ b/pkgs/kops/default.nix @@ -0,0 +1,48 @@ +let + lib = import ../../lib // { + isLocalTarget = let + origin = lib.mkTarget ""; + in target: + target.host == origin.host && + target.user == origin.user; + }; +in + +{ nix, openssh, populate, writeDash, writeJSON }: { + + writeDeploy = name: { source, target }: let + target' = lib.mkTarget target; + in + writeDash name '' + set -efu + + ${populate}/bin/populate \ + ${target'.user}@${target'.host}:${target'.port}${target'.path} \ + < ${writeJSON "${name}-source.json" source} + + ${openssh}/bin/ssh \ + ${target'.user}@${target'.host} -p ${target'.port} \ + nixos-rebuild switch -I ${target'.path} + ''; + + writeTest = name: { source, target }: let + target' = lib.mkTarget target; + in + assert lib.isLocalTarget target'; + writeDash name '' + set -efu + + ${populate}/bin/populate --force \ + ${target'.path} \ + < ${writeJSON "${name}-source.json" source} + + ${nix}/bin/nix-build \ + -A config.system.build.toplevel \ + -I ${target'.path} \ + --arg modules '[]' \ + --no-out-link \ + --show-trace \ + '' + ''; + +} diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix new file mode 100644 index 0000000..89024bd --- /dev/null +++ b/pkgs/overlay.nix @@ -0,0 +1,18 @@ +let + lib = import ../lib; +in + +self: super: { + kops = self.callPackage ./kops {}; + populate = self.callPackage ./populate {}; + writeDash = name: text: self.writeScript name '' + #! ${self.dash}/bin/dash + ${text} + ''; + writeJSON = name: value: self.runCommand name { + json = lib.toJSON value; + passAsFile = [ "json" ]; + } /* sh */ '' + ${self.jq}/bin/jq . "$jsonPath" > "$out" + ''; +} diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix new file mode 100644 index 0000000..acb5a5f --- /dev/null +++ b/pkgs/populate/default.nix @@ -0,0 +1,20 @@ +{ coreutils, findutils, git, gnused, jq, openssh, pass, rsync, runCommand, stdenv }: + +let + PATH = stdenv.lib.makeBinPath [ + coreutils + findutils + git + gnused + jq + openssh + pass + rsync + ]; +in + +runCommand "populate-2.2.0" {} '' + mkdir -p $out/bin + cp ${./populate.sh} $out/bin/populate + sed -i '1s,.*,&\nPATH=${PATH},' $out/bin/populate +'' diff --git a/pkgs/populate/populate.sh b/pkgs/populate/populate.sh new file mode 100755 index 0000000..670d25b --- /dev/null +++ b/pkgs/populate/populate.sh @@ -0,0 +1,276 @@ +#! /bin/sh +set -efu + +main() {( + self=$(readlink -f "$0") + basename=${0##*/} + + debug=false + force=false + origin_host=${HOSTNAME-cat /proc/sys/kernel/hostname} + origin_user=$LOGNAME + target_spec= + + + abort=false + + error() { + echo "$basename: error: $1" >&2 + abort=true + } + + for arg; do + case $arg in + --force) + force=true + ;; + -*) + error "bad argument: $arg" + ;; + *) + if test -n "$target_spec"; then + error "bad argument: $arg" + else + target_spec=$arg + fi + ;; + esac + done + + if test -z "$target_spec"; then + error 'no target specified' + fi + + if test "$abort" = true; then + exit 11 + fi + + target=$( + export origin_host + export origin_user + echo "$target_spec" | jq -R ' + def default(value; f): if . == null then value else f end; + def default(value): default(value; .); + + match("^(?:([^@]+)@)?(?:([^:/]+))?(?::([^/]+))?(/.*)?") + | { + user: .captures[0].string | default(env.origin_user), + host: .captures[1].string | default(env.origin_host), + port: .captures[2].string | default(22; + if test("^[0-9]+$") then fromjson else + error(@json "bad target port: \(.)") + end), + path: .captures[3].string | default("/var/src"), + } + ' + ) + + echo $target | jq . >&2 + + target_host=$(echo $target | jq -r .host) + target_path=$(echo $target | jq -r .path) + target_port=$(echo $target | jq -r .port) + target_user=$(echo $target | jq -r .user) + + if test "$force" = true; then + force_target + else + check_target + fi + + jq -c 'to_entries | group_by(.value.type) | flatten[]' | + while read -r source; do + key=$(echo "$source" | jq -r .key) + type=$(echo "$source" | jq -r .value.type) + conf=$(echo "$source" | jq -r .value.${type}) + + printf '\e[1;33m%s\e[m\n' "populate_$type $key $conf" >&2 + + populate_"$type" "$key" "$conf" + done +)} + +# Safeguard to prevent clobbering of misspelled targets. +# This function has to be called first. +check_target() { + { + echo target_host=$(quote "$target_host") + echo target_path=$(quote "$target_path") + echo 'sentinel_file=$target_path/.populate' + echo 'if ! test -f "$sentinel_file"; then' + echo ' echo "error: missing sentinel file: $target_host:$sentinel_file" >&2' + echo ' exit 1' + echo 'fi' + } \ + | + target_shell +} + +force_target() { + { + echo target_path=$(quote "$target_path") + echo 'sentinel_file=$target_path/.populate' + echo 'mkdir -vp "$target_path"' + echo 'touch "$sentinel_file"' + } \ + | + target_shell +} + +is_local_target() { + test "$target_host" = "$origin_host" && + test "$target_user" = "$origin_user" +} + +populate_file() {( + file_name=$1 + file_path=$(echo "$2" | jq -r .path) + + if is_local_target; then + file_target=$target_path/$file_name + else + file_target=$target_user@$target_host:$target_path/$file_name + fi + + rsync \ + -vFrlptD \ + --delete-excluded \ + "$file_path"/ \ + -e "ssh -o ControlPersist=no -p $target_port" \ + "$file_target" +)} + +populate_git() {( + git_name=$1 + git_url=$(echo "$2" | jq -r .url) + git_ref=$(echo "$2" | jq -r .ref) + + git_work_tree=$target_path/$git_name + + { + echo set -efu + + echo git_url=$(quote "$git_url") + echo git_ref=$(quote "$git_ref") + + echo git_work_tree=$(quote "$git_work_tree") + + echo 'if ! test -e "$git_work_tree"; then' + echo ' git clone "$git_url" "$git_work_tree"' + echo 'fi' + + echo 'cd $git_work_tree' + + echo 'if ! url=$(git config remote.origin.url); then' + echo ' git remote add origin "$git_url"' + echo 'elif test "$url" != "$git_url"; then' + echo ' git remote set-url origin "$git_url"' + echo 'fi' + + # TODO resolve git_ref to commit hash + echo 'hash=$git_ref' + + echo 'if ! test "$(git log --format=%H -1)" = "$hash"; then' + echo ' if ! git log -1 "$hash" >/dev/null 2>&1; then' + echo ' git fetch origin' + echo ' fi' + echo ' git checkout "$hash" -- "$git_work_tree"' + echo ' git -c advice.detachedHead=false checkout -f "$hash"' + echo 'fi' + + echo 'git clean -dfx' + + } \ + | + target_shell +)} + +populate_pass() {( + pass_target_name=$1 + pass_dir=$(echo "$2" | jq -r .dir) + pass_name_root=$(echo "$2" | jq -r .name) + + if is_local_target; then + pass_target=$target_path/$pass_target_name + else + pass_target=$target_user@$target_host:$target_path/$pass_target_name + fi + + umask 0077 + + tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX) + trap cleanup EXIT + cleanup() { + rm -fR "$tmp_dir" + } + + pass_prefix=$pass_dir/$pass_name_root/ + + find "$pass_prefix" -type f | + while read -r pass_gpg_file_path; do + + rel_name=${pass_gpg_file_path:${#pass_prefix}} + rel_name=${rel_name%.gpg} + + pass_name=$pass_name_root/$rel_name + tmp_path=$tmp_dir/$rel_name + + mkdir -p "$(dirname "$tmp_path")" + PASSWORD_STORE_DIR=$pass_dir pass show "$pass_name" > "$tmp_path" + done + + rsync \ + --checksum \ + -vFrlptD \ + --delete-excluded \ + "$tmp_dir"/ \ + -e "ssh -o ControlPersist=no -p $target_port" \ + "$pass_target" +)} + +populate_pipe() {( + pipe_target_name=$1 + pipe_command=$(echo "$2" | jq -r .command) + + result_path=$target_path/$pipe_target_name + + "$pipe_command" | target_shell -c "cat > $(quote "$result_path")" +)} + +populate_symlink() {( + symlink_name=$1 + symlink_target=$(echo "$2" | jq -r .target) + link_name=$target_path/$symlink_name + + { + # TODO rm -fR instead of ln -f? + echo ln -fns $(quote "$symlink_target" "$link_name") + } \ + | + target_shell +)} + +quote() { + printf %s "$1" | sed 's/./\\&/g' + while test $# -gt 1; do + printf ' ' + shift + printf %s "$1" | sed 's/./\\&/g' + done + echo +} + +target_shell() { + if is_local_target; then + /bin/sh "$@" + else + ssh "$target_host" \ + -l "$target_user" \ + -o ControlPersist=no \ + -p "$target_port" \ + -T \ + /bin/sh "$@" + fi +} + +main "$@"