1
0
Fork 0
mirror of https://cgit.krebsco.de/krops synced 2024-12-23 10:19:52 +01:00

import stockholm's deployment tools

https://cgit.krebsco.de/stockholm 877b4104370c1ea9698a449e376e2842d7c372fd
This commit is contained in:
tv 2018-02-28 21:02:21 +01:00
commit e89cf20d43
9 changed files with 580 additions and 0 deletions

56
lib/default.nix Normal file
View file

@ -0,0 +1,56 @@
let {
body = lib;
lib = nixpkgs.lib // builtins // {
evalSource = let
eval = source: lib.evalModules {
modules = lib.singleton {
_file = toString ./.;
imports = map (source: { inherit source; }) (lib.toList source);
options.source = lib.mkOption {
default = {};
type = lib.types.attrsOf lib.types.source;
};
};
};
sanitize = x: lib.getAttr (lib.typeOf x) {
set = lib.mapAttrs
(lib.const sanitize)
(lib.filterAttrs
(name: value: name != "_module" && value != null) x);
string = x;
};
in
# This function's return value can be used as pkgs.populate input.
source: sanitize (eval source).config.source;
getHostName = let
# We're parsing /etc/hostname here because reading
# /proc/sys/kernel/hostname yields ""
y = lib.filter lib.types.label.check (lib.splitString "\n" (lib.readFile /etc/hostname));
in
if lib.length y != 1 then throw "malformed /etc/hostname" else
lib.elemAt y 0;
mkTarget = s: let
default = defVal: val: if val != null then val else defVal;
parse = lib.match "(([^@]+)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s;
elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null;
in {
user = default (lib.getEnv "LOGNAME") (elemAt' parse 1);
host = default (lib.maybeEnv "HOSTNAME" lib.getHostName) (elemAt' parse 3);
port = default "22" /* "ssh"? */ (elemAt' parse 5);
path = default "/var/src" /* no default? */ (elemAt' parse 6);
};
test = re: x: lib.isString x && lib.testString re x;
testString = re: x: lib.match re x != null;
types = nixpkgs.lib.types // import ./types { lib = body; };
};
nixpkgs.lib = import <nixpkgs/lib>;
}

8
lib/types/default.nix Normal file
View file

@ -0,0 +1,8 @@
{ lib }@args: let {
body = lib.foldl' (res: path: res // import path args) {} [
./populate.nix
./posix.nix
];
}

93
lib/types/populate.nix Normal file
View file

@ -0,0 +1,93 @@
{ lib }: rec {
source = lib.types.submodule ({ config, ... }: {
options = {
type = let
known-types = lib.attrNames source-types;
type-candidates = lib.filter (k: config.${k} != null) known-types;
in lib.mkOption {
default = if lib.length type-candidates == 1
then lib.head type-candidates
else throw "cannot determine type";
type = lib.types.enum known-types;
};
file = lib.mkOption {
apply = x:
if lib.types.absolute-pathname.check x
then { path = x; }
else x;
default = null;
type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file);
};
git = lib.mkOption {
default = null;
type = lib.types.nullOr source-types.git;
};
pass = lib.mkOption {
default = null;
type = lib.types.nullOr source-types.pass;
};
pipe = lib.mkOption {
apply = x:
if lib.types.absolute-pathname.check x
then { command = x; }
else x;
default = null;
type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.pipe);
};
symlink = lib.mkOption {
apply = x:
if lib.types.pathname.check x
then { target = x; }
else x;
default = null;
type = lib.types.nullOr (lib.types.either lib.types.pathname source-types.symlink);
};
};
});
source-types = {
file = lib.types.submodule {
options = {
path = lib.mkOption {
type = lib.types.absolute-pathname;
};
};
};
git = lib.types.submodule {
options = {
ref = lib.mkOption {
type = lib.types.str; # TODO lib.types.git.ref
};
url = lib.mkOption {
type = lib.types.str; # TODO lib.types.git.url
};
};
};
pass = lib.types.submodule {
options = {
dir = lib.mkOption {
type = lib.types.absolute-pathname;
};
name = lib.mkOption {
type = lib.types.pathname; # TODO relative-pathname
};
};
};
pipe = lib.types.submodule {
options = {
command = lib.mkOption {
type = lib.types.absolute-pathname;
};
};
};
symlink = lib.types.submodule {
options = {
target = lib.mkOption {
type = lib.types.pathname; # TODO relative-pathname
};
};
};
};
}

54
lib/types/posix.nix Normal file
View file

@ -0,0 +1,54 @@
{ lib }: rec {
# RFC952, B. Lexical grammar, <hname>
hostname = lib.mkOptionType {
name = "hostname";
check = x: lib.isString x && lib.all label.check (lib.splitString "." x);
merge = lib.mergeOneOption;
};
# RFC952, B. Lexical grammar, <name>
# RFC1123, 2.1 Host Names and Numbers
label = lib.mkOptionType {
name = "label";
# TODO case-insensitive labels
check = lib.test "[0-9A-Za-z]([0-9A-Za-z-]*[0-9A-Za-z])?";
merge = lib.mergeOneOption;
};
# POSIX.12013, 3.278 Portable Filename Character Set
filename = lib.mkOptionType {
name = "POSIX filename";
check = lib.test "([0-9A-Za-z._])[0-9A-Za-z._-]*";
merge = lib.mergeOneOption;
};
# POSIX.12013, 3.2 Absolute Pathname
absolute-pathname = lib.mkOptionType {
name = "POSIX absolute pathname";
check = x: lib.isString x && lib.substring 0 1 x == "/" && pathname.check x;
merge = lib.mergeOneOption;
};
# POSIX.12013, 3.267 Pathname
pathname = lib.mkOptionType {
name = "POSIX pathname";
check = x:
let
# The filter is used to normalize paths, i.e. to remove duplicated and
# trailing slashes. It also removes leading slashes, thus we have to
# check for "/" explicitly below.
xs = lib.filter (s: lib.stringLength s > 0) (lib.splitString "/" x);
in
lib.isString x && (x == "/" || (lib.length xs > 0 && lib.all filename.check xs));
merge = lib.mergeOneOption;
};
# POSIX.1-2013, 3.431 User Name
username = lib.mkOptionType {
name = "POSIX username";
check = filename.check;
merge = lib.mergeOneOption;
};
}

7
pkgs/default.nix Normal file
View file

@ -0,0 +1,7 @@
{ overlays ? [], ... }@args:
import <nixpkgs> (args // {
overlays = overlays ++ [
(import ./overlay.nix)
];
})

48
pkgs/kops/default.nix Normal file
View file

@ -0,0 +1,48 @@
let
lib = import ../../lib // {
isLocalTarget = let
origin = lib.mkTarget "";
in target:
target.host == origin.host &&
target.user == origin.user;
};
in
{ nix, openssh, populate, writeDash, writeJSON }: {
writeDeploy = name: { source, target }: let
target' = lib.mkTarget target;
in
writeDash name ''
set -efu
${populate}/bin/populate \
${target'.user}@${target'.host}:${target'.port}${target'.path} \
< ${writeJSON "${name}-source.json" source}
${openssh}/bin/ssh \
${target'.user}@${target'.host} -p ${target'.port} \
nixos-rebuild switch -I ${target'.path}
'';
writeTest = name: { source, target }: let
target' = lib.mkTarget target;
in
assert lib.isLocalTarget target';
writeDash name ''
set -efu
${populate}/bin/populate --force \
${target'.path} \
< ${writeJSON "${name}-source.json" source}
${nix}/bin/nix-build \
-A config.system.build.toplevel \
-I ${target'.path} \
--arg modules '[<nixos-config>]' \
--no-out-link \
--show-trace \
'<nixpkgs/nixos/lib/eval-config.nix>'
'';
}

18
pkgs/overlay.nix Normal file
View file

@ -0,0 +1,18 @@
let
lib = import ../lib;
in
self: super: {
kops = self.callPackage ./kops {};
populate = self.callPackage ./populate {};
writeDash = name: text: self.writeScript name ''
#! ${self.dash}/bin/dash
${text}
'';
writeJSON = name: value: self.runCommand name {
json = lib.toJSON value;
passAsFile = [ "json" ];
} /* sh */ ''
${self.jq}/bin/jq . "$jsonPath" > "$out"
'';
}

20
pkgs/populate/default.nix Normal file
View file

@ -0,0 +1,20 @@
{ coreutils, findutils, git, gnused, jq, openssh, pass, rsync, runCommand, stdenv }:
let
PATH = stdenv.lib.makeBinPath [
coreutils
findutils
git
gnused
jq
openssh
pass
rsync
];
in
runCommand "populate-2.2.0" {} ''
mkdir -p $out/bin
cp ${./populate.sh} $out/bin/populate
sed -i '1s,.*,&\nPATH=${PATH},' $out/bin/populate
''

276
pkgs/populate/populate.sh Executable file
View file

@ -0,0 +1,276 @@
#! /bin/sh
set -efu
main() {(
self=$(readlink -f "$0")
basename=${0##*/}
debug=false
force=false
origin_host=${HOSTNAME-cat /proc/sys/kernel/hostname}
origin_user=$LOGNAME
target_spec=
abort=false
error() {
echo "$basename: error: $1" >&2
abort=true
}
for arg; do
case $arg in
--force)
force=true
;;
-*)
error "bad argument: $arg"
;;
*)
if test -n "$target_spec"; then
error "bad argument: $arg"
else
target_spec=$arg
fi
;;
esac
done
if test -z "$target_spec"; then
error 'no target specified'
fi
if test "$abort" = true; then
exit 11
fi
target=$(
export origin_host
export origin_user
echo "$target_spec" | jq -R '
def default(value; f): if . == null then value else f end;
def default(value): default(value; .);
match("^(?:([^@]+)@)?(?:([^:/]+))?(?::([^/]+))?(/.*)?")
| {
user: .captures[0].string | default(env.origin_user),
host: .captures[1].string | default(env.origin_host),
port: .captures[2].string | default(22;
if test("^[0-9]+$") then fromjson else
error(@json "bad target port: \(.)")
end),
path: .captures[3].string | default("/var/src"),
}
'
)
echo $target | jq . >&2
target_host=$(echo $target | jq -r .host)
target_path=$(echo $target | jq -r .path)
target_port=$(echo $target | jq -r .port)
target_user=$(echo $target | jq -r .user)
if test "$force" = true; then
force_target
else
check_target
fi
jq -c 'to_entries | group_by(.value.type) | flatten[]' |
while read -r source; do
key=$(echo "$source" | jq -r .key)
type=$(echo "$source" | jq -r .value.type)
conf=$(echo "$source" | jq -r .value.${type})
printf '\e[1;33m%s\e[m\n' "populate_$type $key $conf" >&2
populate_"$type" "$key" "$conf"
done
)}
# Safeguard to prevent clobbering of misspelled targets.
# This function has to be called first.
check_target() {
{
echo target_host=$(quote "$target_host")
echo target_path=$(quote "$target_path")
echo 'sentinel_file=$target_path/.populate'
echo 'if ! test -f "$sentinel_file"; then'
echo ' echo "error: missing sentinel file: $target_host:$sentinel_file" >&2'
echo ' exit 1'
echo 'fi'
} \
|
target_shell
}
force_target() {
{
echo target_path=$(quote "$target_path")
echo 'sentinel_file=$target_path/.populate'
echo 'mkdir -vp "$target_path"'
echo 'touch "$sentinel_file"'
} \
|
target_shell
}
is_local_target() {
test "$target_host" = "$origin_host" &&
test "$target_user" = "$origin_user"
}
populate_file() {(
file_name=$1
file_path=$(echo "$2" | jq -r .path)
if is_local_target; then
file_target=$target_path/$file_name
else
file_target=$target_user@$target_host:$target_path/$file_name
fi
rsync \
-vFrlptD \
--delete-excluded \
"$file_path"/ \
-e "ssh -o ControlPersist=no -p $target_port" \
"$file_target"
)}
populate_git() {(
git_name=$1
git_url=$(echo "$2" | jq -r .url)
git_ref=$(echo "$2" | jq -r .ref)
git_work_tree=$target_path/$git_name
{
echo set -efu
echo git_url=$(quote "$git_url")
echo git_ref=$(quote "$git_ref")
echo git_work_tree=$(quote "$git_work_tree")
echo 'if ! test -e "$git_work_tree"; then'
echo ' git clone "$git_url" "$git_work_tree"'
echo 'fi'
echo 'cd $git_work_tree'
echo 'if ! url=$(git config remote.origin.url); then'
echo ' git remote add origin "$git_url"'
echo 'elif test "$url" != "$git_url"; then'
echo ' git remote set-url origin "$git_url"'
echo 'fi'
# TODO resolve git_ref to commit hash
echo 'hash=$git_ref'
echo 'if ! test "$(git log --format=%H -1)" = "$hash"; then'
echo ' if ! git log -1 "$hash" >/dev/null 2>&1; then'
echo ' git fetch origin'
echo ' fi'
echo ' git checkout "$hash" -- "$git_work_tree"'
echo ' git -c advice.detachedHead=false checkout -f "$hash"'
echo 'fi'
echo 'git clean -dfx'
} \
|
target_shell
)}
populate_pass() {(
pass_target_name=$1
pass_dir=$(echo "$2" | jq -r .dir)
pass_name_root=$(echo "$2" | jq -r .name)
if is_local_target; then
pass_target=$target_path/$pass_target_name
else
pass_target=$target_user@$target_host:$target_path/$pass_target_name
fi
umask 0077
tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
trap cleanup EXIT
cleanup() {
rm -fR "$tmp_dir"
}
pass_prefix=$pass_dir/$pass_name_root/
find "$pass_prefix" -type f |
while read -r pass_gpg_file_path; do
rel_name=${pass_gpg_file_path:${#pass_prefix}}
rel_name=${rel_name%.gpg}
pass_name=$pass_name_root/$rel_name
tmp_path=$tmp_dir/$rel_name
mkdir -p "$(dirname "$tmp_path")"
PASSWORD_STORE_DIR=$pass_dir pass show "$pass_name" > "$tmp_path"
done
rsync \
--checksum \
-vFrlptD \
--delete-excluded \
"$tmp_dir"/ \
-e "ssh -o ControlPersist=no -p $target_port" \
"$pass_target"
)}
populate_pipe() {(
pipe_target_name=$1
pipe_command=$(echo "$2" | jq -r .command)
result_path=$target_path/$pipe_target_name
"$pipe_command" | target_shell -c "cat > $(quote "$result_path")"
)}
populate_symlink() {(
symlink_name=$1
symlink_target=$(echo "$2" | jq -r .target)
link_name=$target_path/$symlink_name
{
# TODO rm -fR instead of ln -f?
echo ln -fns $(quote "$symlink_target" "$link_name")
} \
|
target_shell
)}
quote() {
printf %s "$1" | sed 's/./\\&/g'
while test $# -gt 1; do
printf ' '
shift
printf %s "$1" | sed 's/./\\&/g'
done
echo
}
target_shell() {
if is_local_target; then
/bin/sh "$@"
else
ssh "$target_host" \
-l "$target_user" \
-o ControlPersist=no \
-p "$target_port" \
-T \
/bin/sh "$@"
fi
}
main "$@"