Add parallel test runner

The new test runner will evaluate all test profiles from the README.md
in parallel in separate nix-build processes.
Since we do not load all processes into one process,
this also helps saving memory.

.travis.yml

@@ -1,2 +1,3 @@
 language: nix
-script: nix-build release.nix --dry-run --show-trace
+script:
+  ./tests/run.py

README.md

@@ -97,3 +97,9 @@ See code for all available configurations.
 [Samsung Series 9 NP900X3C]: samsung/np900x3c
 [Purism Librem 13v3]: purism/librem/13v3
 [Toshiba Chromebook 2 `swanky`]: toshiba/swanky
+
+## How to contribute a new device profile
+
+1. Add your device profile expression in the appropriate directory
+2. Link it in the table in README.md
+3. Run ./tests/run.py to test it. The test script script will parse all the profiles from the README.md

raspberry-pi/2/default.nix

@@ -4,7 +4,7 @@
   boot = {
     consoleLogLevel = lib.mkDefault 7;
     extraTTYs = [ "ttyAMA0" ];
-    kernelPackages = lib.mkDefault pkgs.linuxPackages_rpi;
+    kernelPackages = lib.mkDefault pkgs.linuxPackages_rpi2;
     kernelParams = [
       "dwc_otg.lpm_enable=0"
       "console=ttyAMA0,115200"

release.nix

@@ -1,55 +0,0 @@
-{ ... }:
-
-let
-  shim = {
-    boot.loader.systemd-boot.enable = true;
-
-    fileSystems."/" = {
-      device = "/dev/disk/by-uuid/00000000-0000-0000-0000-000000000000";
-      fsType = "btrfs";
-    };
-
-    nixpkgs.config = {
-      allowBroken = true;
-      allowUnfree = true;
-    };
-  };
-
-  buildProfile = profile: (import <nixpkgs/nixos> {
-    configuration.imports = [ profile shim ];
-  }).system;
-in
-
-{
-  acer-aspire-4810t = buildProfile ./acer/aspire/4810t;
-
-  airis-n990 = buildProfile ./airis/n990;
-
-  apple-macbook-air-4 = buildProfile ./apple/macbook-air/4;
-  apple-macbook-air-6 = buildProfile ./apple/macbook-air/6;
-  apple-macbook-pro-10-1 = buildProfile ./apple/macbook-pro/10-1;
-  apple-macbook-pro-11-5 = buildProfile ./apple/macbook-pro/11-5;
-  apple-macbook-pro-12-1 = buildProfile ./apple/macbook-pro/12-1;
-
-  dell-e7240 = buildProfile ./dell/e7240;
-  dell-xps-13-9380 = buildProfile ./dell/xps/13-9380;
-  dell-xps-13-9370 = buildProfile ./dell/xps/13-9370;
-  dell-xps-15-9550 = buildProfile ./dell/xps/15-9550;
-
-  lenovo-thinkpad-t410 = buildProfile ./lenovo/thinkpad/t410;
-  lenovo-thinkpad-t440p = buildProfile ./lenovo/thinkpad/t440p;
-  lenovo-thinkpad-t450s = buildProfile ./lenovo/thinkpad/t450s;
-  lenovo-thinkpad-t460s = buildProfile ./lenovo/thinkpad/t460s;
-  lenovo-thinkpad-x140e = buildProfile ./lenovo/thinkpad/x140e;
-  lenovo-thinkpad-x220 = buildProfile ./lenovo/thinkpad/x220;
-  lenovo-thinkpad-x230 = buildProfile ./lenovo/thinkpad/x230;
-  lenovo-thinkpad-x250 = buildProfile ./lenovo/thinkpad/x250;
-  lenovo-thinkpad-x260 = buildProfile ./lenovo/thinkpad/x260;
-  lenovo-thinkpad-x280 = buildProfile ./lenovo/thinkpad/x280;
-
-  microsoft-surface-pro-3 = buildProfile ./microsoft/surface-pro/3;
-
-  pcengines-apu = buildProfile ./pcengines/apu;
-
-  toshiba-swanky = buildProfile ./toshiba/swanky;
-}

tests/build-profile.nix

@@ -0,0 +1,19 @@
+{ profile }:
+
+let
+  shim = {
+    boot.loader.systemd-boot.enable = true;
+
+    fileSystems."/" = {
+      device = "/dev/disk/by-uuid/00000000-0000-0000-0000-000000000000";
+      fsType = "btrfs";
+    };
+
+    nixpkgs.config = {
+      allowBroken = true;
+      allowUnfree = true;
+    };
+  };
+in (import <nixpkgs/nixos> {
+  configuration.imports = [ profile shim ];
+}).system

tests/run.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p nix -p python3 -i python
+
+import argparse
+import multiprocessing
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Tuple
+
+TEST_ROOT = Path(__file__).resolve().parent
+ROOT = TEST_ROOT.parent
+
+GREEN = "\033[92m"
+RED = "\033[91m"
+RESET = "\033[0m"
+
+
+def parse_readme() -> List[str]:
+    profiles = set()
+    with open(ROOT.joinpath("README.md")) as f:
+        for line in f:
+            results = re.findall(r"<nixos-hardware/[^>]+>", line)
+            profiles.update(results)
+    return list(profiles)
+
+
+def build_profile(profile: str) -> Tuple[str, subprocess.CompletedProcess]:
+    # Hard-code this for now until we have enough other architectures to care about this.
+    system = "x86_64-linux"
+    if "raspberry-pi/2" in profile:
+        system = "armv7l-linux"
+
+    cmd = [
+        "nix-build",
+        "-I",
+        f"nixos-hardware={ROOT}",
+        "--dry-run",
+        "--show-trace",
+        "build-profile.nix",
+        "--system",
+        system,
+        "--arg",
+        "profile",
+        profile,
+    ]
+    res = subprocess.run(
+        cmd, cwd=TEST_ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+    )
+    return (profile, res)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run hardware tests")
+    parser.add_argument(
+        "--jobs",
+        type=int,
+        default=multiprocessing.cpu_count(),
+        help="Number of parallel evaluations."
+        "If set to 1 it disable multi processing (suitable for debugging)",
+    )
+    parser.add_argument("profiles", nargs="*")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    if len(args.profiles) == 0:
+        profiles = parse_readme()
+    else:
+        profiles = args.profiles
+
+    failed_profiles = []
+
+    def eval_finished(args: Tuple[str, subprocess.CompletedProcess]) -> None:
+        profile, res = args
+        if res.returncode == 0:
+            print(f"{GREEN}OK {profile}{RESET}")
+        else:
+            print(f"{RED}FAIL {profile}:{RESET}", file=sys.stderr)
+            if res.stdout != "":
+                print(f"{RED}{res.stdout.rstrip()}{RESET}", file=sys.stderr)
+            print(f"{RED}{res.stderr.rstrip()}{RESET}", file=sys.stderr)
+            failed_profiles.append(profile)
+
+    if len(profiles) == 0 or args.jobs == 1:
+        for profile in profiles:
+            eval_finished(build_profile(profile))
+    else:
+        pool = multiprocessing.Pool(processes=args.jobs)
+        for r in pool.imap(build_profile, profiles):
+            eval_finished(r)
+    if len(failed_profiles) > 0:
+        print(f"\n{RED}The following {len(failed_profiles)} test(s) failed:{RESET}")
+        for profile in failed_profiles:
+            print(f"{sys.argv[0]} '{profile}'")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()