fix ci and make it reproducible

.github/workflows/test.yml

@@ -7,15 +7,7 @@ on:
 jobs:
   tests:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # when updating channels, also update .mergify.yml
-        channel: [ nixos-unstable, nixos-24.05 ]
     steps:
     - uses: actions/checkout@v4
     - uses: cachix/install-nix-action@V27
-      with:
+    - run: nix run ./tests#run .
-        nix_path: nixpkgs=channel:${{ matrix.channel }}
-    - name: Show nixpkgs version
-      run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
-    - run: ./tests/run.py

tests/build-profile.nix

@@ -1,4 +1,4 @@
-{ profile, pkgs }:
+{ pkgs, profile }:
 
 (pkgs.nixos [
   profile

tests/flake.lock

@@ -0,0 +1,81 @@
+{
+  "nodes": {
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nixos-unstable-small"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "nixos-hardware": {
+      "locked": {
+        "lastModified": 1723310128,
+        "narHash": "sha256-IiH8jG6PpR4h9TxSGMYh+2/gQiJW9MwehFvheSb5rPc=",
+        "owner": "NixOS",
+        "repo": "nixos-hardware",
+        "rev": "c54cf53e022b0b3c1d3b8207aa0f9b194c24f0cf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixos-hardware",
+        "type": "github"
+      }
+    },
+    "nixos-stable": {
+      "locked": {
+        "lastModified": 1723938990,
+        "narHash": "sha256-9tUadhnZQbWIiYVXH8ncfGXGvkNq3Hag4RCBEMUk7MI=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "c42fcfbdfeae23e68fc520f9182dde9f38ad1890",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixos-unstable-small": {
+      "locked": {
+        "lastModified": 1723999580,
+        "narHash": "sha256-mVAwJQkqcVS+BU9i2YqDz61YQt/q0KRPz8vetezs7ZE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "7bae0943064987c775f857b698522ff2db2df292",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixos-hardware": "nixos-hardware",
+        "nixos-stable": "nixos-stable",
+        "nixos-unstable-small": "nixos-unstable-small"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

tests/flake.nix

@@ -0,0 +1,97 @@
+{
+  description = "Test flake for nixos-hardware";
+
+  inputs = {
+    nixos-unstable-small.url = "github:NixOS/nixpkgs/nixos-unstable-small";
+    nixos-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
+    # override in the test
+    nixos-hardware.url = "github:NixOS/nixos-hardware";
+    flake-parts.url = "github:hercules-ci/flake-parts";
+    flake-parts.inputs.nixpkgs-lib.follows = "nixos-unstable-small";
+  };
+
+  outputs =
+    inputs@{ flake-parts, ... }:
+    flake-parts.lib.mkFlake { inherit inputs; } {
+      systems = [
+        "aarch64-linux"
+        "x86_64-linux"
+        "riscv64-linux"
+      ];
+      perSystem =
+        {
+          system,
+          lib,
+          pkgs,
+          inputs',
+          ...
+        }:
+        let
+          blackList = [
+            # does import-from-derivation
+            "toshiba-swanky"
+            # uses custom nixpkgs config
+            "raspberry-pi-2"
+
+            # deprecated profiles
+            "framework"
+            "asus-zephyrus-ga402x"
+            "lenovo-yoga-7-14ARH7"
+          ];
+
+          # There are more, but for those we need to force it.
+          # In future we should probably already define it in our module.
+          aarch64Systems = [
+            "raspberry-pi-3"
+            "raspberry-pi-4"
+            "raspberry-pi-5"
+          ];
+
+          matchArch =
+            moduleName:
+            if builtins.elem moduleName aarch64Systems then
+              pkgs.hostPlatform.system == "aarch64-linux"
+            else
+              # TODO also add riscv64
+              pkgs.hostPlatform.system == "x86_64-linux";
+
+          modules = lib.filterAttrs (
+            name: _: !(builtins.elem name blackList || lib.hasPrefix "common-" name) && matchArch name
+          ) inputs.nixos-hardware.nixosModules;
+          buildProfile = import ./build-profile.nix;
+
+          unfreeNixpkgs =
+            importPath:
+            import inputs.nixos-unstable-small {
+              config = {
+                allowBroken = true;
+                allowUnfree = true;
+                nvidia.acceptLicense = true;
+              };
+              overlays = [ ];
+              inherit system;
+            };
+          nixpkgsUnstable = unfreeNixpkgs inputs'.nixos-unstable-small;
+          nixpkgsStable = unfreeNixpkgs inputs.nixos-stable;
+
+          checksForNixpkgs =
+            channel: nixpkgs:
+            lib.mapAttrs' (
+              name: module:
+              lib.nameValuePair "${channel}-${name}" (buildProfile {
+                pkgs = nixpkgs;
+                profile = module;
+              })
+            ) modules;
+        in
+        {
+          _module.args.pkgs = nixpkgsUnstable;
+          checks = checksForNixpkgs "nixos-unstable" nixpkgsUnstable // checksForNixpkgs "nixos-stable" nixpkgsStable;
+          packages.run = pkgs.writeShellScriptBin "run.py" ''
+            #!${pkgs.bash}/bin/bash
+            export PATH=${lib.makeBinPath [ pkgs.nix-eval-jobs pkgs.nix-eval-jobs.nix ]}
+            exec ${pkgs.python3.interpreter} ${./.}/run.py --nixos-hardware "$@"
+          '';
+        };
+    };
+}

tests/run.py

@@ -1,16 +1,14 @@
-#!/usr/bin/env nix-shell
+#!/usr/bin/env python3
-#!nix-shell --quiet -p nix-eval-jobs -p nix -p python3 -i python
 
 import argparse
 import json
 import multiprocessing
 import re
+import shlex
 import subprocess
 import sys
-import textwrap
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import IO
 
 TEST_ROOT = Path(__file__).resolve().parent
 ROOT = TEST_ROOT.parent
@@ -22,15 +20,6 @@ RESET = "\033[0m"
 re_nixos_hardware = re.compile(r"<nixos-hardware/([^>]+)>")
 
 
-def parse_readme() -> list[str]:
-    profiles = set()
-    with ROOT.joinpath("README.md").open() as f:
-        for line in f:
-            if (m := re_nixos_hardware.search(line)) is not None:
-                profiles.add(m.group(1).strip())
-    return list(profiles)
-
-
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Run hardware tests")
     parser.add_argument(
@@ -45,68 +34,39 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Print evaluation commands executed",
     )
-    parser.add_argument("profiles", nargs="*")
+    parser.add_argument(
+        "--nixos-hardware",
+        help="Print evaluation commands executed",
+    )
     return parser.parse_args()
 
 
-def write_eval_test(f: IO[str], profiles: list[str]) -> None:
+def run_eval_test(nixos_hardware: str, gcroot_dir: Path, jobs: int) -> list[str]:
-    build_profile = TEST_ROOT.joinpath("build-profile.nix")
-    f.write(
-        textwrap.dedent(
-            f"""
-            let
-              purePkgs = system: import <nixpkgs> {{
-                config = {{
-                  allowBroken = true;
-                  allowUnfree = true;
-                  nvidia.acceptLicense = true;
-                }};
-                overlays = [];
-                inherit system;
-              }};
-              pkgs.x86_64-linux = purePkgs "x86_64-linux";
-              pkgs.aarch64-linux = purePkgs "aarch64-linux";
-              buildProfile = import {build_profile};
-            in
-            """
-        )
-    )
-    f.write("{\n")
-    for profile in profiles:
-        # does import-from-derivation
-        if profile == "toshiba/swanky":
-            continue
-        # uses custom nixpkgs config
-        if profile == "raspberry-pi/2":
-            continue
-
-        system = "x86_64-linux"
-        if profile in ("raspberry-pi/3", "raspberry-pi/4", "raspberry-pi/5"):
-            system = "aarch64-linux"
-
-        f.write(
-            f'  "{profile}" = buildProfile {{ profile = import {ROOT}/{profile}; pkgs = pkgs.{system}; }};\n'
-        )
-    f.write("}\n")
-
-
-def run_eval_test(eval_test: Path, gcroot_dir: Path, jobs: int) -> list[str]:
     failed_profiles = []
     cmd = [
         "nix-eval-jobs",
+        "--extra-experimental-features",
+        "flakes",
+        "--override-input",
+        "nixos-hardware",
+        nixos_hardware,
         "--gc-roots-dir",
-        gcroot_dir,
+        str(gcroot_dir),
         "--max-memory-size",
         "2048",
         "--workers",
         str(jobs),
-        str(eval_test),
+        "--flake",
+        str(TEST_ROOT) + "#checks",
+        "--force-recurse",
     ]
+    print(" ".join(map(shlex.quote,cmd)))
     proc = subprocess.Popen(
         cmd,
         stdout=subprocess.PIPE,
         text=True,
     )
+
     with proc as p:
         assert p.stdout is not None
         for line in p.stdout:
@@ -123,20 +83,17 @@ def run_eval_test(eval_test: Path, gcroot_dir: Path, jobs: int) -> list[str]:
 
 def main() -> None:
     args = parse_args()
-    profiles = parse_readme() if len(args.profiles) == 0 else args.profiles
 
     failed_profiles = []
+
     with TemporaryDirectory() as tmpdir:
-        eval_test = Path(tmpdir) / "eval-test.nix"
         gcroot_dir = Path(tmpdir) / "gcroot"
-        with eval_test.open("w") as f:
+        failed_profiles = run_eval_test(args.nixos_hardware, gcroot_dir, args.jobs)
-            write_eval_test(f, profiles)
-        failed_profiles = run_eval_test(eval_test, gcroot_dir, args.jobs)
 
     if len(failed_profiles) > 0:
         print(f"\n{RED}The following {len(failed_profiles)} test(s) failed:{RESET}")
         for profile in failed_profiles:
-            print(f"{sys.argv[0]} '{profile}'")
+            print(f" '{profile}'")
         sys.exit(1)