speed up ci using nix-eval-jobs

tests/build-profile.nix

@@ -1,7 +1,9 @@
-{ profile }:
+{ profile, pkgs }:
 
-let
+(pkgs.nixos [
-  shim = { config, lib, pkgs, ... }: {
+  profile
+  ({ config, lib, ... }: {
+    nixpkgs.pkgs = pkgs;
     boot.loader.systemd-boot.enable = !config.boot.loader.generic-extlinux-compatible.enable && !config.boot.loader.raspberryPi.enable;
     # we forcefully disable grub here just for testing purposes, even though some profiles might still use grub in the end.
     boot.loader.grub.enable = false;
@@ -14,13 +16,6 @@ let
       device = "/dev/disk/by-uuid/00000000-0000-0000-0000-000000000000";
       fsType = "btrfs";
     };
-
+    system.stateVersion = lib.version;
-    nixpkgs.config = {
+  })
-      allowBroken = true;
+]).config.system.build.toplevel
-      allowUnfree = true;
-      nvidia.acceptLicense = true;
-    };
-  };
-in (import <nixpkgs/nixos> {
-  configuration.imports = [ profile shim ];
-}).system

tests/run.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env nix-shell
-#!nix-shell --quiet -p nix -p python3 -i python
+#!nix-shell --quiet -p nix-eval-jobs -p nix -p python3 -i python
 
 import argparse
+import json
 import multiprocessing
 import re
 import subprocess
 import sys
-from functools import partial
+import textwrap
 from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import IO
 
 TEST_ROOT = Path(__file__).resolve().parent
 ROOT = TEST_ROOT.parent
@@ -16,58 +19,18 @@ GREEN = "\033[92m"
 RED = "\033[91m"
 RESET = "\033[0m"
 
+re_nixos_hardware = re.compile(r"<nixos-hardware/([^>]+)>")
+
 
 def parse_readme() -> list[str]:
     profiles = set()
     with ROOT.joinpath("README.md").open() as f:
         for line in f:
-            results = re.findall(r"<nixos-hardware/[^>]+>", line)
+            if (m := re_nixos_hardware.search(line)) is not None:
-            profiles.update(results)
+                profiles.add(m.group(1).strip())
     return list(profiles)
 
 
-def build_profile(
-    profile: str, verbose: bool
-) -> tuple[str, subprocess.CompletedProcess]:
-    # Hard-code this for now until we have enough other architectures to care about this.
-    system = "x86_64-linux"
-    if "raspberry-pi/2" in profile:
-        system = "armv7l-linux"
-    if "raspberry-pi/4" in profile:
-        system = "aarch64-linux"
-
-    cmd = [
-        "nix",
-        "build",
-        "--extra-experimental-features",
-        "nix-command",
-        "-f",
-        "build-profile.nix",
-        "-I",
-        f"nixos-hardware={ROOT}",
-        "--show-trace",
-        "--system",
-        system,
-        "--arg",
-        "profile",
-        profile,
-    ]
-
-    # uses import from derivation
-    if profile != "<nixos-hardware/toshiba/swanky>":
-        cmd += ["--dry-run"]
-    if verbose:
-        print(f"$ {' '.join(cmd)}")
-    res = subprocess.run(
-        cmd,
-        cwd=TEST_ROOT,
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    return (profile, res)
-
-
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Run hardware tests")
     parser.add_argument(
@@ -86,31 +49,90 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
+def write_eval_test(f: IO[str], profiles: list[str]) -> None:
+    build_profile = TEST_ROOT.joinpath("build-profile.nix")
+    f.write(
+        textwrap.dedent(
+            f"""
+            let
+              purePkgs = system: import <nixpkgs> {{
+                config = {{
+                  allowBroken = true;
+                  allowUnfree = true;
+                  nvidia.acceptLicense = true;
+                }};
+                overlays = [];
+                inherit system;
+              }};
+              pkgs.x86_64-linux = purePkgs "x86_64-linux";
+              pkgs.aarch64-linux = purePkgs "aarch64-linux";
+              buildProfile = import {build_profile};
+            in
+            """
+        )
+    )
+    f.write("{\n")
+    for profile in profiles:
+        # does import-from-derivation
+        if profile == "toshiba/swanky":
+            continue
+        # uses custom nixpkgs config
+        if profile == "raspberry-pi/2":
+            continue
+
+        system = "x86_64-linux"
+        if "raspberry-pi/4" == profile:
+            system = "aarch64-linux"
+
+        f.write(
+            f'  "{profile}" = buildProfile {{ profile = import {ROOT}/{profile}; pkgs = pkgs.{system}; }};\n'
+        )
+    f.write("}\n")
+
+
+def run_eval_test(eval_test: Path, gcroot_dir: Path, jobs: int) -> list[str]:
+    failed_profiles = []
+    cmd = [
+        "nix-eval-jobs",
+        "--gc-roots-dir",
+        gcroot_dir,
+        "--max-memory-size",
+        "2048",
+        "--workers",
+        str(jobs),
+        str(eval_test),
+    ]
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    with proc as p:
+        assert p.stdout is not None
+        for line in p.stdout:
+            data = json.loads(line)
+            attr = data.get("attr")
+            if "error" in data:
+                failed_profiles.append(attr)
+                print(f"{RED}FAIL {attr}:{RESET}", file=sys.stderr)
+                print(f"{RED}{data['error']}{RESET}", file=sys.stderr)
+            else:
+                print(f"{GREEN}OK {attr}{RESET}")
+    return failed_profiles
+
+
 def main() -> None:
     args = parse_args()
     profiles = parse_readme() if len(args.profiles) == 0 else args.profiles
 
     failed_profiles = []
+    with TemporaryDirectory() as tmpdir:
+        eval_test = Path(tmpdir) / "eval-test.nix"
+        gcroot_dir = Path(tmpdir) / "gcroot"
+        with eval_test.open("w") as f:
+            write_eval_test(f, profiles)
+        failed_profiles = run_eval_test(eval_test, gcroot_dir, args.jobs)
 
-    def eval_finished(args: tuple[str, subprocess.CompletedProcess]) -> None:
-        profile, res = args
-        if res.returncode == 0:
-            print(f"{GREEN}OK {profile}{RESET}")
-        else:
-            print(f"{RED}FAIL {profile}:{RESET}", file=sys.stderr)
-            if res.stdout != "":
-                print(f"{RED}{res.stdout.rstrip()}{RESET}", file=sys.stderr)
-            print(f"{RED}{res.stderr.rstrip()}{RESET}", file=sys.stderr)
-            failed_profiles.append(profile)
-
-    build = partial(build_profile, verbose=args.verbose)
-    if len(profiles) == 0 or args.jobs == 1:
-        for profile in profiles:
-            eval_finished(build(profile))
-    else:
-        pool = multiprocessing.Pool(processes=args.jobs)
-        for r in pool.imap(build, profiles):
-            eval_finished(r)
     if len(failed_profiles) > 0:
         print(f"\n{RED}The following {len(failed_profiles)} test(s) failed:{RESET}")
         for profile in failed_profiles: