nixgl: API rework for flexibility and dual-GPU

docs/manual/usage.md

@@ -59,5 +59,6 @@ usage/configuration.md
 usage/rollbacks.md
 usage/dotfiles.md
 usage/graphical.md
+usage/gpu-non-nixos.md
 usage/updating.md
 ```

docs/manual/usage/gpu-non-nixos.md

@@ -0,0 +1,70 @@
+# GPU on non-NixOS systems {#sec-usage-gpu-non-nixos}
+
+To access the GPU, programs need access to OpenGL and Vulkan libraries. While
+this works transparently on NixOS, it does not on other Linux systems. A
+solution is provided by [NixGL](https://github.com/nix-community/nixGL), which
+can be integrated into Home Manager.
+
+To enable the integration, import NixGL into your home configuration, either as
+a channel, or as a flake input passed via `extraSpecialArgs`. Then, set the
+`nixGL.packages` option to the package set provided by NixGL.
+
+Once integration is enabled, it can be used in two ways: as Nix functions for
+wrapping programs installed via Home Manager, and as shell commands for running
+programs installed by other means (such as `nix shell`). In either case, there
+are several wrappers available. They can be broadly categorized
+
+- by vendor: as Mesa (for Free drivers of all vendors) and Nvidia (for
+  Nvidia-specific proprietary drivers).
+- by GPU selection: as primary and secondary (offloading).
+
+For example, the `mesa` wrapper provides support for running programs on the
+primary GPU for Intel, AMD and Nouveau drivers, while the `mesaPrime` wrapper
+does the same for the secondary GPU.
+
+**Note:** when using Nvidia wrappers together with flakes, your home
+configuration will not be pure and needs to be built using `home-manager switch
+--impure`. Otherwise, the build will fail, complaining about missing attribute
+`currentTime`.
+
+Wrapper functions are available under `config.lib.nixGL.wrappers`. However, it
+can be more convenient to use the `config.lib.nixGL.wrap` alias, which can be
+configured to use any of the wrappers. It is intended to provide a customization
+point when the same home configuration is used across several machines with
+different hardware. There is also the `config.lib.nixGL.wrapOffload` alias for
+two-GPU systems.
+
+Another convenience is that all wrapper functions are always available. However,
+when `nixGL.packages` option is unset, they are no-ops. This allows them to be
+used even when the home configuration is used on NixOS machines. The exception
+is the `prime-offload` script which ignores `nixGL.packages` and is installed
+into the environment whenever `nixGL.prime.installScript` is set. This script,
+which can be used to start a program on a secondary GPU, does not depend on
+NixGL and is useful on NixOS systems as well.
+
+Below is an abbreviated example for an Optimus laptop that makes use of both
+Mesa and Nvidia wrappers, where the latter is used in dGPU offloading mode. It
+demonstrates how to wrap `mpv` to run on the integrated Intel GPU, wrap FreeCAD
+to run on the Nvidia dGPU, and how to install the wrapper scripts. It also wraps
+Xonotic to run on the dGPU, but uses the wrapper function directly for
+demonstration purposes.
+
+```nix
+{ config, lib, pkgs, nixGL, ... }:
+{
+  nixGL.packages = nixGL.packages;
+  nixGL.defaultWrapper = "mesa";
+  nixGL.offloadWrapper = "nvidiaPrime";
+  nixGL.installScripts = [ "mesa" "nvidiaPrime" ];
+
+  programs.mpv = {
+    enable = true;
+    package = config.lib.nixGL.wrap pkgs.mpv;
+  };
+
+  home.packages = [
+    (config.lib.nixGL.wrapOffload pkgs.freecad)
+    (config.lib.nixGL.wrappers.nvidiaPrime pkgs.xonotic)
+  ];
+}
+```

modules/misc/nixgl.nix

@@ -1,45 +1,166 @@
 { config, lib, pkgs, ... }:
 
-let cfg = config.nixGL;
+let
+  cfg = config.nixGL;
+  wrapperListMarkdown = with builtins;
+    foldl' (list: name:
+      list + ''
+        - ${name}
+      '') "" (attrNames config.lib.nixGL.wrappers);
 in {
   meta.maintainers = [ lib.maintainers.smona ];
 
-  options.nixGL.prefix = lib.mkOption {
+  options.nixGL = {
-    type = lib.types.str;
+    packages = lib.mkOption {
-    default = "";
+      type = with lib.types; nullOr attrs;
-    example = lib.literalExpression
+      default = null;
-      ''"''${inputs.nixGL.packages.x86_64-linux.nixGLIntel}/bin/nixGLIntel"'';
+      example = lib.literalExpression "inputs.nixGL.packages";
       description = ''
-      The [nixGL](https://github.com/nix-community/nixGL) command that `lib.nixGL.wrap` should prefix
+        The nixGL package set containing GPU library wrappers. This can be used
-      package binaries with. nixGL provides your system's version of libGL to applications, enabling
+        to provide OpenGL and Vulkan access to applications on non-NixOS systems
-      them to access the GPU on non-NixOS systems.
+        by using `(config.lib.nixGL.wrap <package>)` for the default wrapper, or
+        `(config.lib.nixGL.wrappers.<wrapper> <package>)` for any available
+        wrapper.
 
-      Wrap individual packages which require GPU access with the function like so: `(config.lib.nixGL.wrap <package>)`.
+        The wrapper functions are always available. If this option is empty (the
-      The returned package can be used just like the original one, but will have access to libGL. For example:
+        default), they are a no-op. This is useful on NixOS where the wrappers
+        are unnecessary.
 
-      ```nix
+        Note that using any Nvidia wrapper requires building the configuration
-      # If you're using a Home Manager module to configure the package,
+        with the `--impure` option.
-      # pass it into the module's package argument:
-      programs.kitty = {
-        enable = true;
-        package = (config.lib.nixGL.wrap pkgs.kitty);
-      };
-
-      # Otherwise, pass it to any option where a package is expected:
-      home.packages = [ (config.lib.nixGL.wrap pkgs.hello) ];
-      ```
-
-      If this option is empty (the default), then `lib.nixGL.wrap` is a no-op. This is useful for sharing your Home Manager
-      configurations between NixOS and non-NixOS systems, since NixOS already provides libGL to applications without the
-      need for nixGL.
       '';
     };
 
-  config = {
+    defaultWrapper = lib.mkOption {
-    lib.nixGL.wrap = # Wrap a single package with the configured nixGL wrapper
+      type = lib.types.enum (builtins.attrNames config.lib.nixGL.wrappers);
-      pkg:
+      default = "mesa";
+      description = ''
+        The package wrapper function available for use as `(config.lib.nixGL.wrap
+        <package>)`. Intended to start programs on the main GPU.
 
-      if cfg.prefix == "" then
+        Wrapper functions can be found under `config.lib.nixGL.wrappers`. They
+        can be used directly, however, setting this option provides a convenient
+        shorthand.
+
+        The following wrappers are available:
+        ${wrapperListMarkdown}
+      '';
+    };
+
+    offloadWrapper = lib.mkOption {
+      type = lib.types.enum (builtins.attrNames config.lib.nixGL.wrappers);
+      default = "mesaPrime";
+      description = ''
+        The package wrapper function available for use as
+        `(config.lib.nixGL.wrapOffload <package>)`. Intended to start programs
+        on the secondary GPU.
+
+        Wrapper functions can be found under `config.lib.nixGL.wrappers`. They
+        can be used directly, however, setting this option provides a convenient
+        shorthand.
+
+        The following wrappers are available:
+        ${wrapperListMarkdown}
+      '';
+    };
+
+    prime.card = lib.mkOption {
+      type = lib.types.str;
+      default = "1";
+      example = "pci-0000_06_00_0";
+      description = ''
+        Selects the non-default graphics card used for PRIME render offloading.
+        The value can be:
+
+        - a number, selecting the n-th non-default GPU;
+        - a PCI bus id in the form `pci-XXX_YY_ZZ_U`;
+        - a PCI id in the form `vendor_id:device_id`
+
+        For more information, consult the Mesa documentation on the `DRI_PRIME`
+        environment variable.
+      '';
+    };
+
+    prime.nvidiaProvider = lib.mkOption {
+      type = with lib.types; nullOr str;
+      default = null;
+      example = "NVIDIA-G0";
+      description = ''
+        If this option is set, it overrides the offload provider for Nvidia
+        PRIME offloading. Consult the proprietary Nvidia driver documentation
+        on the `__NV_PRIME_RENDER_OFFLOAD_PROVIDER` environment variable.
+      '';
+    };
+
+    prime.installScript = lib.mkOption {
+      type = with lib.types; nullOr (enum [ "mesa" "nvidia" ]);
+      default = null;
+      example = "mesa";
+      description = ''
+        If this option is set, the wrapper script `prime-offload` is installed
+        into the environment. It allows starting programs on the secondary GPU
+        selected by the `nixGL.prime.card` option. This makes sense when the
+        program is not already using one of nixGL PRIME wrappers, or for
+        programs not installed from Nixpkgs.
+
+        This option can be set to either "mesa" or "nvidia", making the script
+        use one or the other graphics library.
+      '';
+    };
+
+    installScripts = lib.mkOption {
+      type = with lib.types;
+        nullOr (listOf (enum (builtins.attrNames config.lib.nixGL.wrappers)));
+      default = null;
+      example = [ "mesa" "mesaPrime" ];
+      description = ''
+        For each wrapper `wrp` named in the provided list, a wrapper script
+        named `nixGLWrp` is installed into the environment. These scripts are
+        useful for running programs not installed via Home Manager.
+
+        The following wrappers are available:
+        ${wrapperListMarkdown}
+      '';
+    };
+
+    vulkan.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = ''
+        Whether to enable Vulkan in nixGL wrappers.
+
+        This is disabled by default bacause Vulkan brings in several libraries
+        that can cause symbol version conflicts in wrapped programs. Your
+        mileage may vary.
+      '';
+    };
+  };
+
+  config = let
+    getWrapperExe = vendor:
+      let
+        glPackage = cfg.packages.${pkgs.system}."nixGL${vendor}";
+        glExe = lib.getExe glPackage;
+        vulkanPackage = cfg.packages.${pkgs.system}."nixVulkan${vendor}";
+        vulkanExe = if cfg.vulkan.enable then lib.getExe vulkanPackage else "";
+      in "${glExe} ${vulkanExe}";
+
+    mesaOffloadEnv = { "DRI_PRIME" = "${cfg.prime.card}"; };
+
+    nvOffloadEnv = {
+      "DRI_PRIME" = "${cfg.prime.card}";
+      "__NV_PRIME_RENDER_OFFLOAD" = "1";
+      "__GLX_VENDOR_LIBRARY_NAME" = "nvidia";
+      "__VK_LAYER_NV_optimus" = "NVIDIA_only";
+    } // (let provider = cfg.prime.nvidiaProvider;
+    in if !isNull provider then {
+      "__NV_PRIME_RENDER_OFFLOAD_PROVIDER" = "${provider}";
+    } else
+      { });
+
+    makePackageWrapper = vendor: environment: pkg:
+      if builtins.isNull cfg.packages then
         pkg
       else
       # Wrap the package's binaries with nixGL, while preserving the rest of
@@ -53,11 +174,17 @@ in {
           separateDebugInfo = false;
           nativeBuildInputs = old.nativeBuildInputs or [ ]
             ++ [ pkgs.makeWrapper ];
-          buildCommand = ''
+          buildCommand = let
+            # We need an intermediate wrapper package because makeWrapper
+            # requires a single executable as the wrapper.
+            combinedWrapperPkg =
+              pkgs.writeShellScriptBin "nixGLCombinedWrapper-${vendor}" ''
+                exec ${getWrapperExe vendor} "$@"
+              '';
+          in ''
             set -eo pipefail
 
-            ${
+            ${ # Heavily inspired by https://stackoverflow.com/a/68523368/6259505
-            # Heavily inspired by https://stackoverflow.com/a/68523368/6259505
             lib.concatStringsSep "\n" (map (outputName: ''
               echo "Copying output ${outputName}"
               set -x
@@ -72,10 +199,14 @@ in {
             for file in ${pkg.out}/bin/*; do
               local prog="$(basename "$file")"
               makeWrapper \
-                "${cfg.prefix}" \
+                "${lib.getExe combinedWrapperPkg}" \
                 "$out/bin/$prog" \
                 --argv0 "$prog" \
-                --add-flags "$file"
+                --add-flags "$file" \
+                ${
+                  lib.concatStringsSep " " (lib.attrsets.mapAttrsToList
+                    (var: val: "--set '${var}' '${val}'") environment)
+                }
             done
 
             # If .desktop files refer to the old package, replace the references
@@ -91,5 +222,56 @@ in {
             shopt -u nullglob # Revert nullglob back to its normal default state
           '';
         }));
+
+    wrappers = {
+      mesa = makePackageWrapper "Intel" { };
+      mesaPrime = makePackageWrapper "Intel" mesaOffloadEnv;
+      nvidia = makePackageWrapper "Nvidia" { };
+      nvidiaPrime = makePackageWrapper "Nvidia" nvOffloadEnv;
+    };
+  in {
+    lib.nixGL.wrap = wrappers.${cfg.defaultWrapper};
+    lib.nixGL.wrapOffload = wrappers.${cfg.offloadWrapper};
+    lib.nixGL.wrappers = wrappers;
+
+    home.packages = let
+      wantsPrimeWrapper = (!isNull cfg.prime.installScript);
+      wantsWrapper = wrapper:
+        (!isNull cfg.packages) && (!isNull cfg.installScripts)
+        && (builtins.elem wrapper cfg.installScripts);
+      envVarsAsScript = environment:
+        lib.concatStringsSep "\n"
+        (lib.attrsets.mapAttrsToList (var: val: "export ${var}=${val}")
+          environment);
+    in [
+      (lib.mkIf wantsPrimeWrapper (pkgs.writeShellScriptBin "prime-offload" ''
+        ${if cfg.prime.installScript == "mesa" then
+          (envVarsAsScript mesaOffloadEnv)
+        else
+          (envVarsAsScript nvOffloadEnv)}
+        exec "$@"
+      ''))
+
+      (lib.mkIf (wantsWrapper "mesa") (pkgs.writeShellScriptBin "nixGLMesa" ''
+        exec ${getWrapperExe "Intel"} "$@"
+      ''))
+
+      (lib.mkIf (wantsWrapper "mesaPrime")
+        (pkgs.writeShellScriptBin "nixGLMesaPrime" ''
+          ${envVarsAsScript mesaOffloadEnv}
+          exec ${getWrapperExe "Intel"} "$@"
+        ''))
+
+      (lib.mkIf (wantsWrapper "nvidia")
+        (pkgs.writeShellScriptBin "nixGLNvidia" ''
+          exec ${getWrapperExe "Nvidia"} "$@"
+        ''))
+
+      (lib.mkIf (wantsWrapper "nvidia")
+        (pkgs.writeShellScriptBin "nixGLNvidiaPrime" ''
+          ${envVarsAsScript nvOffloadEnv}
+          exec ${getWrapperExe "Nvidia"} "$@"
+        ''))
+    ];
   };
 }