Fix grpc-haskell tests for Nix (#80)

This updates the `release.nix` to now correctly run the test suite.  You can
now build and test `grpc-haskell` by running:

```
$ nix-build -A grpc-haskell release.nix
```

... and `nix-shell` has been updated, too, so that you can do `cabal`
development inside of a `nix-shell` by running:

```
$ nix-shell -A grpc-haskell.env release.nix
```

For people who prefer to use `stack` you can still just build the `grpc`
library by running:

```
$ nix-build -A grpc release.nix
```

... then pass that library as input to `stack`

default-tests.nix

@@ -1,29 +0,0 @@
-{ grpc-haskell-no-tests
-, ghc
-, python
-}:
-
-attrs@
-{ mkDerivation, async, base, bytestring, c2hs, clock, containers
-, grpc, managed, pipes, proto3-wire, protobuf-wire, random, safe
-, sorted-list, stdenv, stm, tasty, tasty-hunit, tasty-quickcheck
-, text, time, transformers, turtle, unix, vector
-}:
-
-let
-  mkDerivation' = oldAttrs: mkDerivation (oldAttrs // {
-    patches = [ tests/tests.patch ];
-
-    postPatch = ''
-      substituteInPlace tests/simple-server.sh --replace @ghc@ ${ghc} --replace @shell@ ${stdenv.shell}
-      substituteInPlace tests/protoc.sh --replace @python@ ${python} --replace @shell@ ${stdenv.shell}
-    '';
-
-    testHaskellDepends = oldAttrs.testHaskellDepends ++ [
-      ghc grpc-haskell-no-tests
-    ];
-
-    doCheck = true;
-  });
-
-in import ./default.nix (attrs // { mkDerivation = mkDerivation'; })

default.nix

@@ -1,7 +1,7 @@
 { mkDerivation, async, base, bytestring, c2hs, clock, containers
-, grpc, managed, pipes, proto3-wire, protobuf-wire, random, safe
+, grpc, managed, pipes, proto3-wire, protobuf-wire, QuickCheck
-, sorted-list, stdenv, stm, tasty, tasty-hunit, tasty-quickcheck
+, random, safe, sorted-list, stdenv, stm, tasty, tasty-hunit
-, text, time, transformers, turtle, unix, vector
+, tasty-quickcheck, text, time, transformers, turtle, unix, vector
 }:
 mkDerivation {
   pname = "grpc-haskell";
@@ -22,11 +22,10 @@ mkDerivation {
   ];
   testHaskellDepends = [
     async base bytestring clock containers managed pipes protobuf-wire
-    safe tasty tasty-hunit tasty-quickcheck text time transformers
+    QuickCheck safe tasty tasty-hunit tasty-quickcheck text time
-    turtle unix
+    transformers turtle unix
   ];
   homepage = "http://github.com/aloiscochard/grpc-haskell";
   description = "Haskell implementation of gRPC layered on shared C library";
   license = stdenv.lib.licenses.asl20;
-  doCheck = false;
 }

nix/grpc.nix

@@ -0,0 +1,23 @@
+{ pkgs, stdenv, lib, fetchgit, autoconf, automake, libtool, which, zlib, openssl
+}:
+
+stdenv.mkDerivation rec {
+  name    = "grpc-${version}";
+  version = "0.15-${lib.strings.substring 0 7 rev}";
+  rev     = "03efbd34ce64615f58007eae667b375accc6c8e6";
+  src = fetchgit {
+    inherit rev;
+    url    = "https://github.com/grpc/grpc.git";
+    sha256 = "1pac3jby5p5a6p6vpqc5whkgy36hnn2ph2jbckg3w73hrxrnwmdh";
+  };
+  preInstall  = "export prefix";
+  buildInputs =
+    (if stdenv.isDarwin then [ pkgs.darwin.cctools ] else []) ++ [
+    autoconf
+    automake
+    libtool
+    which
+    zlib
+    openssl
+  ];
+}

nix/proto3-wire.nix

@@ -0,0 +1,22 @@
+{ mkDerivation, base, bytestring, cereal, containers, deepseq
+, fetchgit, QuickCheck, safe, stdenv, tasty, tasty-hunit
+, tasty-quickcheck, text
+}:
+mkDerivation {
+  pname = "proto3-wire";
+  version = "1.0.0";
+  src = fetchgit {
+    url = "https://github.com/awakenetworks/proto3-wire.git";
+    sha256 = "1v1jsgsdrhaz3ddbil09yqimnps3svgqbjvdk7hil4irpgqkfs98";
+    rev = "1b88bf24aad15db1f59a00d201d609fa308157f7";
+  };
+  libraryHaskellDepends = [
+    base bytestring cereal containers deepseq QuickCheck safe text
+  ];
+  testHaskellDepends = [
+    base bytestring cereal QuickCheck tasty tasty-hunit
+    tasty-quickcheck text
+  ];
+  description = "A low-level implementation of the Protocol Buffers (version 3) wire format";
+  license = stdenv.lib.licenses.asl20;
+}

nix/protobuf-wire.nix

@@ -0,0 +1,25 @@
+{ mkDerivation, base, bytestring, cereal, containers, deepseq
+, fetchgit, filepath, haskell-src, mtl, parsec, parsers, pipes
+, pretty, proto3-wire, QuickCheck, safe, semigroups, stdenv, tasty
+, tasty-hunit, tasty-quickcheck, text, transformers, turtle, vector
+}:
+mkDerivation {
+  pname = "protobuf-wire";
+  version = "0.1.0.0";
+  src = fetchgit {
+    url = "git@github.mv.awakenetworks.net:awakenetworks/protobuf-wire.git";
+    sha256 = "1d52hd7wq8cfxsp35mmamj0m3mr4705bc76344rhjmsi055r70bc";
+    rev = "927c61bbb1002a9278b8a7cbe3968be059f9ff30";
+  };
+  libraryHaskellDepends = [
+    base bytestring cereal containers deepseq filepath haskell-src mtl
+    parsec parsers pipes pretty proto3-wire QuickCheck safe semigroups
+    text transformers vector
+  ];
+  testHaskellDepends = [
+    base bytestring cereal proto3-wire QuickCheck semigroups tasty
+    tasty-hunit tasty-quickcheck text transformers turtle
+  ];
+  description = "A low level library for writing out data in the Protocol Buffers wire format";
+  license = stdenv.lib.licenses.unfree;
+}

release.nix

@@ -1,97 +1,241 @@
-# To build this repository with `nix` you run:
+# If you would like to test and build changes quickly using `cabal`, run:
 #
-#     $ nix-build release.nix -I ssh-config-file=/path/to/ssh/config
+#     $ # Consider adding the following command to your `~/.profile`
+#     $ NIX_PATH="${NIX_PATH}:ssh-config-file=${HOME}/.ssh/config:ssh-auth-sock=${SSH_AUTH_SOCK}"
+#     $ nix-shell -A grpc-haskell.env release.nix
+#     [nix-shell]$ cabal configure --with-gcc=clang --enable tests
 #
-# ... where the `ssh-config-file` is an SSH configuration file with enough
+# This will open up a Nix shell where all of your Haskell tools will work like
-# information to authenticate (i.e. an `IdentityFile` readable by the user that
+# normal, except that all dependencies (including C libraries) are managed by
-# builds this expression, for example).
+# Nix.  The only thing that won't work is running tests inside this shell
+# (although you can still build them).  Fixing the test suite requires
+# extensive patching of the test scripts (see `postPatch` below)
 #
-# If you update the `.cabal` file (such as changing dependencies or adding new
+# Note that this will compile the library once without tests using Nix.  This
-# library/executable/test/benchmark sections), then update the `default.nix`
+# is due to the fact that `grpc-haskell`'s test suite cannot test code
-# expression by running:
+# generation without the library being built at least once.
+#
+# If you want to build and test this repository using `nix`, you can run the
+# following command:
+#
+#     $ nix-build -A grpc-haskell release.nix
+#
+# ... but this is not recommended for normal development because this will
+# rebuild the repository from scratch every time, which is extremely slow.  Only
+# do this if you want to exactly reproduce our continuous integration build.
+#
+# If you update the `grpc-haskell.cabal` file (such as changing dependencies or
+# adding new library/executable/test/benchmark sections), then update the
+# `default.nix` expression by running:
 #
 #     $ cabal2nix . > default.nix
 #
-# If you want to update a dependency like `proto3-wire` to the latest git
+# By default, Nix will pick a version for each one of your Haskell dependencies.
-# revision, then run:
+# If you would like to select a different version then, run:
 #
-#     $ nix-prefetch-git https://github.com/awakenetworks/proto3-wire.git
+#     $ cabal2nix cabal://${package-name}-${version} > nix/${package-name}.nix
 #
-# ... and modify the `rev` and `sha256` fields of the corresponding `fetchgit`
+# ... and then add this line below in the Haskell package overrides section:
-# expression below using the output of the `nix-prefetch-git` command.
 #
-# If you want to test a local `proto3-wire` repository, then replace the
+#     ${package-name} =
-# `fetchgit { ... }` expression with the relative path to the source repository
+#       haskellPackagesNew.callPackage ./nix/${package-name}.nix { };
-# such as:
 #
-#     let proto3-wire-src = ../proto3-wire;
+# ... replacing `${package-name}` with the name of the package that you would
-#     in
+# like to upgrade and `${version}` with the version you want to upgrade to.
-#     ...
+#
-
+# You can also add private Git dependencies in the same way, except supplying
+# the `git` URL to clone:
+#
+#     $ cabal2nix git@github.mv.awakenetworks.net:awakenetworks/${package-name}.git > ./nix/${package-name}.nix
+#
+# `cabal2nix` also takes an optional `--revision` flag if you want to pick a
+# revision other than the latest one to depend on.
+#
+# If you want to test a local source checkout of a dependency, then run:
+#
+#     $ cabal2nix path/to/dependency/repo > nix/${package-name}.nix
 let
   config = {
     packageOverrides = pkgs: rec {
-      grpc = pkgs.stdenv.mkDerivation rec {
+      cython = pkgs.buildPythonPackage rec {
-        name    = "grpc-${version}";
+        name = "Cython-${version}";
-        version = "0.15-${pkgs.lib.strings.substring 0 7 rev}";
+        version = "0.24.1";
-        rev     = "03efbd34ce64615f58007eae667b375accc6c8e6";
+
-        src = pkgs.fetchgit {
+        src = pkgs.fetchurl {
-          inherit rev;
+          url = "mirror://pypi/C/Cython/${name}.tar.gz";
-          url    = "https://github.com/grpc/grpc.git";
+          sha256 = "84808fda00508757928e1feadcf41c9f78e9a9b7167b6649ab0933b76f75e7b9";
-          sha256 = "1pac3jby5p5a6p6vpqc5whkgy36hnn2ph2jbckg3w73hrxrnwmdh";
         };
-        preInstall  = "export prefix";
+
+        # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729
+        # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround
+        # when that fix is stabilized
+        NIX_CFLAGS_COMPILE =
+          pkgs.stdenv.lib.optionalString (pkgs.stdenv.cc.isClang or false)
+            "-I${pkgs.libcxx}/include/c++/v1";
+
         buildInputs =
-          (if pkgs.stdenv.isDarwin then [ pkgs.darwin.cctools ] else []) ++ [
+              pkgs.stdenv.lib.optional (pkgs.stdenv.cc.isClang or false) pkgs.libcxx
-          pkgs.autoconf
+          ++  [ pkgs.pkgconfig pkgs.gdb ];
-          pkgs.automake
+
-          pkgs.libtool
+        doCheck = false;
-          pkgs.which
+
-          pkgs.zlib
+        doHaddock = false;
-          pkgs.openssl
+
+        doHoogle = false;
+
+        meta = {
+          description = "An optimising static compiler for both the Python programming language and the extended Cython programming language";
+          platforms = pkgs.stdenv.lib.platforms.all;
+          homepage = http://cython.org;
+          license = pkgs.stdenv.lib.licenses.asl20;
+          maintainers = with pkgs.stdenv.lib.maintainers; [ fridh ];
+        };
+      };
+
+      grpc = pkgs.callPackage ./nix/grpc.nix { };
+
+      grpcio = pkgs.pythonPackages.buildPythonPackage rec {
+        name = "grpc-${version}";
+
+        version = "1.0";
+
+        src = pkgs.fetchgit {
+          url    = "https://github.com/grpc/grpc.git";
+          rev    = "c204647295437b01337ad8e6c17c4296609c7a13";
+          sha256 = "0ifg5acwcb0qyf5g5mrja8ab4x3f1fxdw80rkmhn3kyjkhjzm9ik";
+        };
+
+        preConfigure = ''
+          export GRPC_PYTHON_BUILD_WITH_CYTHON=1
+        '';
+
+        # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729
+        # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround
+        # when that fix is stabilized
+        NIX_CFLAGS_COMPILE =
+          pkgs.stdenv.lib.optionalString (pkgs.stdenv.cc.isClang or false)
+            "-I${pkgs.libcxx}/include/c++/v1";
+
+        buildInputs =
+          pkgs.stdenv.lib.optional (pkgs.stdenv.cc.isClang or false) pkgs.libcxx;
+
+        propagatedBuildInputs = [
+          cython
+          pkgs.pythonPackages.futures
+          pkgs.pythonPackages.protobuf3_0
+          pkgs.pythonPackages.enum34
+        ];
+      };
+
+      grpcio-tools = pkgs.pythonPackages.buildPythonPackage rec {
+        name = "grpc-${version}";
+
+        version = "1.0";
+
+        src = pkgs.fetchgit {
+          url    = "https://github.com/grpc/grpc.git";
+          rev    = "c204647295437b01337ad8e6c17c4296609c7a13";
+          sha256 = "0ifg5acwcb0qyf5g5mrja8ab4x3f1fxdw80rkmhn3kyjkhjzm9ik";
+        };
+
+        preConfigure = ''
+          export GRPC_PYTHON_BUILD_WITH_CYTHON=1
+          cd tools/distrib/python/grpcio_tools
+          python ../make_grpcio_tools.py
+        '';
+
+        # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729
+        # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround
+        # when that fix is stabilized
+        NIX_CFLAGS_COMPILE =
+          pkgs.stdenv.lib.optionalString (pkgs.stdenv.cc.isClang or false)
+            "-I${pkgs.libcxx}/include/c++/v1";
+
+        buildInputs =
+          pkgs.stdenv.lib.optional (pkgs.stdenv.cc.isClang or false) pkgs.libcxx;
+
+        propagatedBuildInputs = [
+          cython
+          pkgs.pythonPackages.futures
+          pkgs.pythonPackages.protobuf3_0
+          pkgs.pythonPackages.enum34
+          grpcio
         ];
       };
 
       haskellPackages = pkgs.haskell.packages.ghc7103.override {
         overrides = haskellPackagesNew: haskellPackagesOld: rec {
           proto3-wire =
-            let proto3-wire-src = pkgs.fetchgit {
+            haskellPackagesNew.callPackage ./nix/proto3-wire.nix { };
-              url    = "https://github.com/awakenetworks/proto3-wire.git";
-              rev    = "1b88bf24aad15db1f59a00d201d609fa308157f7";
-              sha256 = "02gsj0qyqqnqawm7s2h4y2510j82jv4jq2gsyadmck1ihlc9pfvl";
-            };
-            in
-            haskellPackagesNew.callPackage proto3-wire-src { };
 
           protobuf-wire =
-            let protobuf-wire-src = pkgs.fetchgitPrivate {
+            pkgs.haskell.lib.dontCheck
-              url    = "git@github.mv.awakenetworks.net:awakenetworks/protobuf-wire.git";
+              (haskellPackagesNew.callPackage ./nix/protobuf-wire.nix {
-              rev    = "927c61bbb1002a9278b8a7cbe3968be059f9ff30";
+                fetchgit = pkgs.fetchgitPrivate;
-              sha256 = "091nhscrdp10cbm1v08m0i2s46qfnyd7c5j0hhbkyi5z50gzi0sk";
+              });
-            };
-            in
-            haskellPackagesNew.callPackage protobuf-wire-src { };
 
           grpc-haskell-no-tests =
-            haskellPackagesNew.callPackage ./default.nix { };
+            pkgs.haskell.lib.overrideCabal
+              (haskellPackagesNew.callPackage ./default.nix { })
+              (oldDerivation: {
+                doCheck = false;
+
+                preBuild =
+                  pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+                    export DYLD_LIBRARY_PATH=${grpc}/lib''${DYLD_LIBRARY_PATH:+:}$DYLD_LIBRARY_PATH
+                  '';
+              });
 
           grpc-haskell =
-            haskellPackagesNew.callPackage (import ./default-tests.nix {
+            pkgs.haskell.lib.overrideCabal
-              inherit grpc-haskell-no-tests;
+              (haskellPackagesNew.callPackage ./default.nix { })
-              inherit (pkgs) ghc python;
+              (oldDerivation:
-            }) { };
+                let
+                  ghc =
+                    haskellPackagesNew.ghcWithPackages (pkgs: [
+                      pkgs.grpc-haskell-no-tests
+                    ]);
 
-          sorted-list = haskellPackagesNew.callPackage
+                  python = pkgs.python.withPackages (pkgs: [
-            ({ mkDerivation, base, deepseq }:
+                    # pkgs.protobuf3_0
-             mkDerivation {
+                    grpcio-tools
-               pname = "sorted-list";
+                  ]);
-               version = "0.2.0.0";
+
-               sha256 = "cc52c787b056f4d3a9ecc59f06701695602558a4233042ff8f613cdd4985d138";
+                in rec {
-               libraryHaskellDepends = [ base deepseq ];
+                  buildDepends = [ pkgs.makeWrapper ];
-               homepage = "https://github.com/Daniel-Diaz/sorted-list/blob/master/README.md";
+
-               description = "Type-enforced sorted lists and related functions";
+                  patches = [ tests/tests.patch ];
-               license = pkgs.stdenv.lib.licenses.bsd3;
+
-             }) {};
+                  preBuild =
+                    pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+                      export DYLD_LIBRARY_PATH=${grpc}/lib''${DYLD_LIBRARY_PATH:+:}$DYLD_LIBRARY_PATH
+                    '';
+
+                  postPatch = ''
+                    patchShebangs tests
+                    substituteInPlace tests/simple-client.sh \
+                      --replace @makeWrapper@ ${pkgs.makeWrapper} \
+                      --replace @grpc@ ${grpc}
+                    substituteInPlace tests/simple-server.sh \
+                      --replace @makeWrapper@ ${pkgs.makeWrapper} \
+                      --replace @grpc@ ${grpc}
+                    wrapProgram tests/protoc.sh \
+                      --prefix PATH : ${python}/bin
+                    wrapProgram tests/test-client.sh \
+                      --prefix PATH : ${python}/bin
+                    wrapProgram tests/test-server.sh \
+                      --prefix PATH : ${python}/bin
+                    wrapProgram tests/simple-client.sh \
+                      --prefix PATH : ${ghc}/bin
+                    wrapProgram tests/simple-server.sh \
+                      --prefix PATH : ${ghc}/bin
+                  '';
+
+                shellHook =
+                  pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+                    export DYLD_LIBRARY_PATH=${grpc}/lib''${DYLD_LIBRARY_PATH:+:}$DYLD_LIBRARY_PATH
+                  '';
+                });
         };
       };
     };
@@ -100,7 +244,17 @@ let
   };
 
 in
-{ pkgs ? import <nixpkgs> { inherit config; } }:
+
-# Disable tests for now, since they don't pass yet
+let
-{ grpc-haskell = pkgs.haskell.lib.dontCheck pkgs.haskellPackages.grpc-haskell;
+   linuxPkgs = import <nixpkgs> { inherit config; system = "x86_64-linux" ; };
-}
+  darwinPkgs = import <nixpkgs> { inherit config; system = "x86_64-darwin"; };
+        pkgs = import <nixpkgs> { inherit config; };
+
+in
+  { grpc-haskell-linux  =  linuxPkgs.haskellPackages.grpc-haskell;
+    grpc-haskell-darwin = darwinPkgs.haskellPackages.grpc-haskell;
+    grpc-haskell        =       pkgs.haskellPackages.grpc-haskell;
+    grpc-linux          =  linuxPkgs.grpc;
+    grpc-darwin         = darwinPkgs.grpc;
+    grpc                =       pkgs.grpc;
+  }

tests/GeneratedTests.hs

@@ -32,7 +32,7 @@ testServerGeneration = testCase "server generation" $ do
   runManaged $ do
     serverExitCodeA <- fork (shell (hsTmpDir <> "/simple-server") empty)
     clientExitCodeA <- fork
-      (export "PYTHONPATH" pyTmpDir >> shell "python tests/test-client.py" empty)
+      (export "PYTHONPATH" pyTmpDir >> shell "tests/test-client.sh" empty)
 
     liftIO $ do
       serverExitCode <- liftIO (wait serverExitCodeA)
@@ -59,7 +59,7 @@ testClientGeneration = testCase "client generation" $ do
 
   runManaged $ do
     serverExitCodeA <- fork
-      (export "PYTHONPATH" pyTmpDir >> shell "python tests/test-server.py" empty)
+      (export "PYTHONPATH" pyTmpDir >> shell "tests/test-server.sh" empty)
     clientExitCodeA <- fork (shell (hsTmpDir <> "/simple-client") empty)
 
     liftIO $ do

tests/test-client.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+
+python tests/test-client.py

tests/test-server.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+
+python tests/test-server.py

tests/tests.patch

@@ -1,30 +1,42 @@
-diff --git a/tests/protoc.sh b/tests/protoc.sh
+diff --git a/tests/simple-client.sh b/tests/simple-client.sh
-index 054c8b6..4bf0893 100644
+index 8feb1f4..ca925b1 100755
---- a/tests/protoc.sh
+--- a/tests/simple-client.sh
-+++ b/tests/protoc.sh
++++ b/tests/simple-client.sh
-@@ -1,8 +1,8 @@
+@@ -2,7 +2,7 @@
--#!/bin/bash -eu
-+#! @shell@ -eu
- 
- pyTmpDir=$1
- 
--python                          \
-+@python@/bin/python             \
-     -m grpc.tools.protoc        \
-     -I tests                    \
-     --python_out=$pyTmpDir      \
-diff --git a/tests/simple-server.sh b/tests/simple-server.sh
-index 37a0a63..8e8fb66 100755
---- a/tests/simple-server.sh
-+++ b/tests/simple-server.sh
-@@ -1,8 +1,8 @@
--#!/bin/bash -eu
-+#! @shell@ -eu
  
  hsTmpDir=$1
  
 -stack ghc --                   \
-+@ghc@/bin/ghc                  \
++ghc                            \
      --make                     \
      -threaded                  \
      -odir $hsTmpDir            \
+@@ -11,3 +11,7 @@ stack ghc --                   \
+     $hsTmpDir/Simple.hs        \
+     tests/TestClient.hs        \
+     > /dev/null
++
++. @makeWrapper@/nix-support/setup-hook
++
++flagsBefore="" wrapProgram "${hsTmpDir}/simple-client" --prefix DYLD_LIBRARY_PATH : @grpc@/lib
+diff --git a/tests/simple-server.sh b/tests/simple-server.sh
+index 37a0a63..eb37776 100755
+--- a/tests/simple-server.sh
++++ b/tests/simple-server.sh
+@@ -2,7 +2,7 @@
+ 
+ hsTmpDir=$1
+ 
+-stack ghc --                   \
++ghc                            \
+     --make                     \
+     -threaded                  \
+     -odir $hsTmpDir            \
+@@ -11,3 +11,7 @@ stack ghc --                   \
+     $hsTmpDir/Simple.hs        \
+     tests/TestServer.hs        \
+     > /dev/null
++
++. @makeWrapper@/nix-support/setup-hook
++
++flagsBefore="" wrapProgram "${hsTmpDir}/simple-server" --prefix DYLD_LIBRARY_PATH : @grpc@/lib