From 04a366f28cd492eda94d07fcf67b3cc99f4ef6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 19 Aug 2024 09:56:10 +0200 Subject: [PATCH 1/4] fix ci and make it reproducible --- .github/workflows/test.yml | 10 +--- tests/build-profile.nix | 2 +- tests/flake.lock | 81 +++++++++++++++++++++++++++++++ tests/flake.nix | 97 ++++++++++++++++++++++++++++++++++++++ tests/run.py | 85 +++++++++------------------------ 5 files changed, 201 insertions(+), 74 deletions(-) create mode 100644 tests/flake.lock create mode 100644 tests/flake.nix diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d1999e..f27ceed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,15 +7,7 @@ on: jobs: tests: runs-on: ubuntu-latest - strategy: - matrix: - # when updating channels, also update .mergify.yml - channel: [ nixos-unstable, nixos-24.05 ] steps: - uses: actions/checkout@v4 - uses: cachix/install-nix-action@V27 - with: - nix_path: nixpkgs=channel:${{ matrix.channel }} - - name: Show nixpkgs version - run: nix-instantiate --eval -E '(import {}).lib.version' - - run: ./tests/run.py + - run: nix run ./tests#run . diff --git a/tests/build-profile.nix b/tests/build-profile.nix index e4e1cdb..aa3714a 100644 --- a/tests/build-profile.nix +++ b/tests/build-profile.nix @@ -1,4 +1,4 @@ -{ profile, pkgs }: +{ pkgs, profile }: (pkgs.nixos [ profile diff --git a/tests/flake.lock b/tests/flake.lock new file mode 100644 index 0000000..4fdd2f9 --- /dev/null +++ b/tests/flake.lock @@ -0,0 +1,81 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixos-unstable-small" + ] + }, + "locked": { + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixos-hardware": { + "locked": { + "lastModified": 1723310128, + "narHash": "sha256-IiH8jG6PpR4h9TxSGMYh+2/gQiJW9MwehFvheSb5rPc=", + "owner": "NixOS", + "repo": "nixos-hardware", + "rev": "c54cf53e022b0b3c1d3b8207aa0f9b194c24f0cf", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixos-hardware", + "type": "github" + } + }, + "nixos-stable": { + "locked": { + "lastModified": 1723938990, + "narHash": "sha256-9tUadhnZQbWIiYVXH8ncfGXGvkNq3Hag4RCBEMUk7MI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c42fcfbdfeae23e68fc520f9182dde9f38ad1890", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixos-unstable-small": { + "locked": { + "lastModified": 1723999580, + "narHash": "sha256-mVAwJQkqcVS+BU9i2YqDz61YQt/q0KRPz8vetezs7ZE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "7bae0943064987c775f857b698522ff2db2df292", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixos-hardware": "nixos-hardware", + "nixos-stable": "nixos-stable", + "nixos-unstable-small": "nixos-unstable-small" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/tests/flake.nix b/tests/flake.nix new file mode 100644 index 0000000..a00f0f2 --- /dev/null +++ b/tests/flake.nix @@ -0,0 +1,97 @@ +{ + description = "Test flake for nixos-hardware"; + + inputs = { + nixos-unstable-small.url = "github:NixOS/nixpkgs/nixos-unstable-small"; + nixos-stable.url = "github:NixOS/nixpkgs/nixos-24.05"; + # override in the test + nixos-hardware.url = "github:NixOS/nixos-hardware"; + flake-parts.url = "github:hercules-ci/flake-parts"; + flake-parts.inputs.nixpkgs-lib.follows = "nixos-unstable-small"; + }; + + outputs = + inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ + "aarch64-linux" + "x86_64-linux" + "riscv64-linux" + ]; + perSystem = + { + system, + lib, + pkgs, + inputs', + ... + }: + let + blackList = [ + # does import-from-derivation + "toshiba-swanky" + # uses custom nixpkgs config + "raspberry-pi-2" + + # deprecated profiles + "framework" + "asus-zephyrus-ga402x" + "lenovo-yoga-7-14ARH7" + ]; + + # There are more, but for those we need to force it. + # In future we should probably already define it in our module. + aarch64Systems = [ + "raspberry-pi-3" + "raspberry-pi-4" + "raspberry-pi-5" + ]; + + matchArch = + moduleName: + if builtins.elem moduleName aarch64Systems then + pkgs.hostPlatform.system == "aarch64-linux" + else + # TODO also add riscv64 + pkgs.hostPlatform.system == "x86_64-linux"; + + modules = lib.filterAttrs ( + name: _: !(builtins.elem name blackList || lib.hasPrefix "common-" name) && matchArch name + ) inputs.nixos-hardware.nixosModules; + buildProfile = import ./build-profile.nix; + + unfreeNixpkgs = + importPath: + import inputs.nixos-unstable-small { + config = { + allowBroken = true; + allowUnfree = true; + nvidia.acceptLicense = true; + }; + overlays = [ ]; + inherit system; + }; + nixpkgsUnstable = unfreeNixpkgs inputs'.nixos-unstable-small; + nixpkgsStable = unfreeNixpkgs inputs.nixos-stable; + + checksForNixpkgs = + channel: nixpkgs: + lib.mapAttrs' ( + name: module: + lib.nameValuePair "${channel}-${name}" (buildProfile { + pkgs = nixpkgs; + profile = module; + }) + ) modules; + in + { + _module.args.pkgs = nixpkgsUnstable; + checks = checksForNixpkgs "nixos-unstable" nixpkgsUnstable // checksForNixpkgs "nixos-stable" nixpkgsStable; + packages.run = pkgs.writeShellScriptBin "run.py" '' + #!${pkgs.bash}/bin/bash + export PATH=${lib.makeBinPath [ pkgs.nix-eval-jobs pkgs.nix-eval-jobs.nix ]} + exec ${pkgs.python3.interpreter} ${./.}/run.py --nixos-hardware "$@" + ''; + }; + }; +} diff --git a/tests/run.py b/tests/run.py index 6668b4e..7e03532 100755 --- a/tests/run.py +++ b/tests/run.py @@ -1,16 +1,14 @@ -#!/usr/bin/env nix-shell -#!nix-shell --quiet -p nix-eval-jobs -p nix -p python3 -i python +#!/usr/bin/env python3 import argparse import json import multiprocessing import re +import shlex import subprocess import sys -import textwrap from pathlib import Path from tempfile import TemporaryDirectory -from typing import IO TEST_ROOT = Path(__file__).resolve().parent ROOT = TEST_ROOT.parent @@ -22,15 +20,6 @@ RESET = "\033[0m" re_nixos_hardware = re.compile(r"]+)>") -def parse_readme() -> list[str]: - profiles = set() - with ROOT.joinpath("README.md").open() as f: - for line in f: - if (m := re_nixos_hardware.search(line)) is not None: - profiles.add(m.group(1).strip()) - return list(profiles) - - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run hardware tests") parser.add_argument( @@ -45,68 +34,39 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Print evaluation commands executed", ) - parser.add_argument("profiles", nargs="*") + parser.add_argument( + "--nixos-hardware", + help="Print evaluation commands executed", + ) return parser.parse_args() -def write_eval_test(f: IO[str], profiles: list[str]) -> None: - build_profile = TEST_ROOT.joinpath("build-profile.nix") - f.write( - textwrap.dedent( - f""" - let - purePkgs = system: import {{ - config = {{ - allowBroken = true; - allowUnfree = true; - nvidia.acceptLicense = true; - }}; - overlays = []; - inherit system; - }}; - pkgs.x86_64-linux = purePkgs "x86_64-linux"; - pkgs.aarch64-linux = purePkgs "aarch64-linux"; - buildProfile = import {build_profile}; - in - """ - ) - ) - f.write("{\n") - for profile in profiles: - # does import-from-derivation - if profile == "toshiba/swanky": - continue - # uses custom nixpkgs config - if profile == "raspberry-pi/2": - continue - - system = "x86_64-linux" - if profile in ("raspberry-pi/3", "raspberry-pi/4", "raspberry-pi/5"): - system = "aarch64-linux" - - f.write( - f' "{profile}" = buildProfile {{ profile = import {ROOT}/{profile}; pkgs = pkgs.{system}; }};\n' - ) - f.write("}\n") - - -def run_eval_test(eval_test: Path, gcroot_dir: Path, jobs: int) -> list[str]: +def run_eval_test(nixos_hardware: str, gcroot_dir: Path, jobs: int) -> list[str]: failed_profiles = [] cmd = [ "nix-eval-jobs", + "--extra-experimental-features", + "flakes", + "--override-input", + "nixos-hardware", + nixos_hardware, "--gc-roots-dir", - gcroot_dir, + str(gcroot_dir), "--max-memory-size", "2048", "--workers", str(jobs), - str(eval_test), + "--flake", + str(TEST_ROOT) + "#checks", + "--force-recurse", ] + print(" ".join(map(shlex.quote,cmd))) proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, text=True, ) + with proc as p: assert p.stdout is not None for line in p.stdout: @@ -123,20 +83,17 @@ def run_eval_test(eval_test: Path, gcroot_dir: Path, jobs: int) -> list[str]: def main() -> None: args = parse_args() - profiles = parse_readme() if len(args.profiles) == 0 else args.profiles failed_profiles = [] + with TemporaryDirectory() as tmpdir: - eval_test = Path(tmpdir) / "eval-test.nix" gcroot_dir = Path(tmpdir) / "gcroot" - with eval_test.open("w") as f: - write_eval_test(f, profiles) - failed_profiles = run_eval_test(eval_test, gcroot_dir, args.jobs) + failed_profiles = run_eval_test(args.nixos_hardware, gcroot_dir, args.jobs) if len(failed_profiles) > 0: print(f"\n{RED}The following {len(failed_profiles)} test(s) failed:{RESET}") for profile in failed_profiles: - print(f"{sys.argv[0]} '{profile}'") + print(f" '{profile}'") sys.exit(1) From 15c8c47fe2a5f6c72f8327ed6a43c99e4a924d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 19 Aug 2024 10:12:00 +0200 Subject: [PATCH 2/4] kobol/helios4: fix eval --- kobol/helios4/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kobol/helios4/default.nix b/kobol/helios4/default.nix index 532a757..5b4207f 100644 --- a/kobol/helios4/default.nix +++ b/kobol/helios4/default.nix @@ -1,4 +1,4 @@ -{ pkgs, ... }: +{ pkgs, lib, ... }: let linux_5_15_helios4 = pkgs.linux_5_15.override { kernelPatches = [ @@ -26,8 +26,8 @@ let ]; defconfig = "mvebu_v7_defconfig"; # Make the kernel build a bit faster by disabling GPU modules, which we don't need anyways - structuredExtraConfig = with pkgs.lib.kernel; { - DRM = no; + structuredExtraConfig = { + DRM = lib.mkForce pkgs.lib.kernel.no; }; }; in From a3efd46620a6b9f60a82b66f22c1c363223d60d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 19 Aug 2024 09:57:39 +0200 Subject: [PATCH 3/4] apple/imac/18-2: fix gpu import path --- apple/imac/18-2/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apple/imac/18-2/default.nix b/apple/imac/18-2/default.nix index fbbfeba..f44a8fc 100644 --- a/apple/imac/18-2/default.nix +++ b/apple/imac/18-2/default.nix @@ -6,7 +6,7 @@ }: { imports = [ ../. - ../../../common/cpu/intel/kaby-lake + ../../../common/gpu/intel/kaby-lake ../../../common/gpu/intel ../../../common/gpu/amd ../../../common/hidpi.nix @@ -14,16 +14,16 @@ ../../../common/pc/laptop/acpi_call.nix ]; - # apple smc (TODO: check spi) + # apple smc (TODO: check spi) boot = { initrd.kernelModules = ["applespi" "spi_pxa2xx_platform" "intel_lpss_pci" "applesmc" ]; kernelParams = [ "intel_iommu=on" ]; kernelPackages = lib.mkIf (lib.versionOlder pkgs.linux.version "6.0") pkgs.linuxPackages_latest; }; - # Wifi, CPU Microcode FW updates + # Wifi, CPU Microcode FW updates networking.enableB43Firmware = lib.mkDefault true; - hardware = { + hardware = { enableRedistributableFirmware = lib.mkDefault true; cpu.intel.updateMicrocode = lib.mkDefault true; }; From 70e7e3fa140d55efc43e78379ae09d8872d8dc7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 19 Aug 2024 09:56:10 +0200 Subject: [PATCH 4/4] update mergify --- .mergify.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index dc3bf77..45ffb1e 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,8 +1,7 @@ queue_rules: - name: default merge_conditions: - - check-success=tests (nixos-24.05) - - check-success=tests (nixos-unstable) + - check-success=tests defaults: actions: queue: