From bf6012863f229a246e5c791419c084609ad75b47 Mon Sep 17 00:00:00 2001 From: Phillip Seeber Date: Wed, 27 Jul 2022 22:14:50 +0200 Subject: [PATCH] nix overlay and flake to build tensorflow fix permissions --- flake.lock | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 72 +++++++++++++++++++++++++++++++++++++++++++++ nix/overlay.nix | 78 +++++++++++++++++++++++++++++++++++++++++++++++++ nix/src.json | 6 ---- nix/update | 26 ----------------- 5 files changed, 227 insertions(+), 32 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/overlay.nix delete mode 100644 nix/src.json delete mode 100755 nix/update diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..f0f2197 --- /dev/null +++ b/flake.lock @@ -0,0 +1,77 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1656928814, + "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1658850041, + "narHash": "sha256-HA4koSVBPEERWSrH0LVBy28y2FGJnOeUmGDEyeQPmDY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3bbb296d9a0088c314ce83038b896753bbe33acb", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixpkgs-unstable", + "type": "indirect" + } + }, + "root": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "tensorflowSubModule": "tensorflowSubModule" + } + }, + "tensorflowSubModule": { + "flake": false, + "locked": { + "lastModified": 1595549353, + "narHash": "sha256-4UgDtFLqSJ1tNGLx30x2oVxlsSCb7Cj88c5/Ff1zpbU=", + "owner": "tensorflow", + "repo": "tensorflow", + "rev": "b36436b087bd8e8701ef51718179037cccdfc26e", + "type": "github" + }, + "original": { + "owner": "tensorflow", + "repo": "tensorflow", + "rev": "b36436b087bd8e8701ef51718179037cccdfc26e", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..13c2207 --- /dev/null +++ b/flake.nix @@ -0,0 +1,72 @@ +{ + description = "Haskell bindings for TensorFlow"; + + inputs = { + nixpkgs.url = "nixpkgs/nixpkgs-unstable"; + + flake-utils.url = "github:numtide/flake-utils"; + + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; + + tensorflowSubModule = { + url = + "github:tensorflow/tensorflow/b36436b087bd8e8701ef51718179037cccdfc26e"; + flake = false; + }; + }; + + outputs = { self, nixpkgs, flake-utils, tensorflowSubModule, ... }: + flake-utils.lib.eachSystem [ "x86_64-linux" ] (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ (import ./nix/overlay.nix tensorflowSubModule) ]; + }; + + tfPackages = [ + "tensorflow" + "tensorflow-core-ops" + "tensorflow-logging" + "tensorflow-opgen" + "tensorflow-ops" + "tensorflow-proto" + "tensorflow-records" + "tensorflow-records-conduit" + "tensorflow-test" + ]; + + in { + packages = builtins.listToAttrs (builtins.map (p: { + name = p; + value = pkgs.haskellPackages."${p}"; + }) tfPackages); + + devShells = let + devDeps = with pkgs; [ + cabal-install + haskell-language-server + hlint + ormolu + ]; + + devPkgs = builtins.listToAttrs (builtins.map (p: { + name = p; + value = pkgs.mkShell { + buildInputs = let + pkgEnv = pkgs.haskellPackages.ghcWithPackages (ghcPkgs: + ghcPkgs."${p}".buildInputs + ++ ghcPkgs."${p}".propagatedBuildInputs + ++ ghcPkgs."${p}".propagatedNativeBuildInputs + ++ ghcPkgs."${p}".nativeBuildInputs ++ devDeps); + in [ pkgEnv ]; + }; + }) tfPackages); + + in devPkgs // { default = devPkgs.tensorflow; }; + }) // { + overlays.default = import ./nix/overlay.nix tensorflowSubModule; + }; +} diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 0000000..3a0e765 --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,78 @@ +tensorflowSubModule: final: prev: + +let + /* The tensorflow submodule in third_party/tensorflow and the symlinks in the + Haskell-package directories will always be removed by Nix. + Thus construct a source here where the submodule is a normal directory + whenever required. + */ + tfSrc = prev.runCommand "prepTensorflowSrc" { + src = prev.nix-gitignore.gitignoreSource [ + "nix/" + "flake.nix" + "flake.lock" + "result" + ] ./..; + } '' + cp -r $src/* . + + chmod -R +rwx ./third_party + cp -r ${tensorflowSubModule} ./third_party/tensorflow + + chmod -R +rwx tensorflow + rm -rf tensorflow/third_party + cp -r ${tensorflowSubModule} ./tensorflow/third_party + + for d in tensorflow-opgen tensorflow-proto; do + chmod +rwx $d $d/third_party + rm -rf $d/third_party + mkdir $d/third_party + cp -r ${tensorflowSubModule} $d/third_party/tensorflow + done + cp -r . $out + ''; + + haskellOverrides = final: prev: hfinal: hprev: { + tensorflow = hfinal.callCabal2nix "tensorflow" "${tfSrc}/tensorflow" { }; + + tensorflow-core-ops = + hfinal.callCabal2nix "tensorflow-core-ops" "${tfSrc}/tensorflow-core-ops" + { }; + + tensorflow-logging = + hfinal.callCabal2nix "tensorflow-logging" "${tfSrc}/tensorflow-logging" + { }; + + tensorflow-opgen = + hfinal.callCabal2nix "tensorflow-opgen" "${tfSrc}/tensorflow-opgen" { }; + + tensorflow-ops = + hfinal.callCabal2nix "tensorflow-ops" "${tfSrc}/tensorflow-ops" { }; + + tensorflow-proto = let + c2n = + hfinal.callCabal2nix "tensorflow-proto" "${tfSrc}/tensorflow-proto" { }; + in prev.haskell.lib.overrideCabal c2n (drv: { + libraryToolDepends = drv.libraryToolDepends ++ [ prev.protobuf ]; + }); + + tensorflow-records = + hfinal.callCabal2nix "tensorflow-records" "${tfSrc}/tensorflow-records" + { }; + + tensorflow-records-conduit = + hfinal.callCabal2nix "tensorflow-records-conduit" + "${tfSrc}/tensorflow-records-conduit" { }; + + tensorflow-test = + hfinal.callCabal2nix "tensorflow-test" "${tfSrc}/tensorflow-test" { }; + }; + +in { + haskell = prev.haskell // { + packages = prev.haskell.packages // (builtins.mapAttrs + (key: val: val.override { overrides = haskellOverrides final prev; }) { + inherit (prev.haskell.packages) ghc8107 ghc902 ghc923; + }); + }; +} diff --git a/nix/src.json b/nix/src.json deleted file mode 100644 index d677276..0000000 --- a/nix/src.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "8731aaaf8b30888bc24994096db830993090d7c4", - "sha256": "1hcc89rxi47nb0mpk05nl9rbbb04kfw97xfydhpmmgh57yrp3zqa" -} diff --git a/nix/update b/nix/update deleted file mode 100755 index 3e70253..0000000 --- a/nix/update +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p bash nix curl jq -# vim: filetype=sh -# Updates the nixpkgs.json to the latest release -# See https://gist.github.com/zimbatm/de5350245874361762b6a4dfe5366530 -set -euo pipefail - -cd "$(dirname "$0")" || exit 1 - -branch=release-19.09 - -owner=NixOS -repo=nixpkgs -rev=$(curl -sfL https://api.github.com/repos/$owner/$repo/git/refs/heads/$branch | jq -r .object.sha) -url=https://github.com/$owner/$repo/archive/$rev.tar.gz - -release_sha256=$(nix-prefetch-url --unpack "$url") - -cat <