Upgrade gRPC C library (#117)

core/cbits/grpc_haskell.c

@@ -505,7 +505,7 @@ grpc_auth_metadata_processor* mk_auth_metadata_processor(
 grpc_call_credentials* grpc_metadata_credentials_create_from_plugin_(
   grpc_metadata_credentials_plugin* plugin){
 
-  return grpc_metadata_credentials_create_from_plugin(*plugin, NULL);
+  return grpc_metadata_credentials_create_from_plugin(*plugin, GRPC_PRIVACY_AND_INTEGRITY, NULL);
 }
 
 //This is a hack to work around GHC being unable to deal with raw struct params.

core/src/Network/GRPC/Unsafe.chs

@@ -219,9 +219,6 @@ castPeek p = do
     unTag `Tag'}
   -> `()'#}
 
-{#fun grpc_channel_ping as ^
-  {`Channel', `CompletionQueue', unTag `Tag',unReserved `Reserved'} -> `()' #}
-
 {#fun grpc_channel_destroy as ^ {`Channel'} -> `()'#}
 
 -- | Starts executing a batch of ops in the given 'OpArray'. Does not block.

core/tests/LowLevelTests/Op.hs

@@ -1,60 +1,77 @@
-{-# LANGUAGE LambdaCase        #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE RecordWildCards   #-}
+{-# LANGUAGE RecordWildCards #-}
 
 module LowLevelTests.Op where
 
-import           Data.ByteString                (ByteString)
+import Control.Concurrent
-import           Test.Tasty
+import Control.Exception
-import           Test.Tasty.HUnit               as HU (testCase, (@?=))
+import Control.Monad
-
+import Data.ByteString (ByteString)
-import           Network.GRPC.LowLevel
+import Network.GRPC.LowLevel
-import           Network.GRPC.LowLevel.Call
+import Network.GRPC.LowLevel.Call
-import           Network.GRPC.LowLevel.Client
+import Network.GRPC.LowLevel.Client
-import           Network.GRPC.LowLevel.Server
+import Network.GRPC.LowLevel.Op
-import           Network.GRPC.LowLevel.Op
+import Network.GRPC.LowLevel.Server
+import Test.Tasty
+import Test.Tasty.HUnit as HU (testCase, (@?=))
 
 lowLevelOpTests :: TestTree
-lowLevelOpTests = testGroup "Synchronous unit tests of low-level Op interface"
+lowLevelOpTests =
-  [testCancelFromServer]
+  testGroup
+    "Synchronous unit tests of low-level Op interface"
+    [testCancelFromServer]
 
 testCancelFromServer :: TestTree
 testCancelFromServer =
   testCase "Client/Server - client receives server cancellation" $
-  runSerialTest $ \grpc ->
+    runSerialTest $ \grpc ->
-    withClientServerUnaryCall grpc $
+      withClientServerUnaryCall grpc $
-    \(Client{..}, Server{..}, ClientCall{..}, sc@ServerCall{..}) -> do
+        \(Client {..}, Server {..}, ClientCall {..}, sc@ServerCall {..}) -> do
-      serverCallCancel sc StatusPermissionDenied "TestStatus"
+          serverCallCancel sc StatusPermissionDenied "TestStatus"
-      clientRes <- runOps unsafeCC clientCQ clientRecvOps
+          clientRes <- runOps unsafeCC clientCQ clientRecvOps
-      case clientRes of
+          case clientRes of
-        Left x -> error $ "Client recv error: " ++ show x
+            Left x -> error $ "Client recv error: " ++ show x
-        Right [_,_,OpRecvStatusOnClientResult _ code _details] -> do
+            Right [_, _, OpRecvStatusOnClientResult _ code _details] -> do
-          code @?= StatusPermissionDenied
+              code @?= StatusPermissionDenied
-          return $ Right ()
+              return $ Right ()
-        wrong -> error $ "Unexpected op results: " ++ show wrong
+            wrong -> error $ "Unexpected op results: " ++ show wrong
-
 
 runSerialTest :: (GRPC -> IO (Either GRPCIOError ())) -> IO ()
 runSerialTest f =
-  withGRPC f >>= \case Left x -> error $ show x
+  withGRPC f >>= \case
-                       Right () -> return ()
+    Left x -> error $ show x
+    Right () -> return ()
 
-withClientServerUnaryCall :: GRPC
+withClientServerUnaryCall ::
-                             -> ((Client, Server, ClientCall,
+  GRPC ->
-                                  ServerCall ByteString)
+  ( ( Client,
-                                 -> IO (Either GRPCIOError a))
+      Server,
-                             -> IO (Either GRPCIOError a)
+      ClientCall,
+      ServerCall ByteString
+    ) ->
+    IO (Either GRPCIOError a)
+  ) ->
+  IO (Either GRPCIOError a)
 withClientServerUnaryCall grpc f = do
   withClient grpc clientConf $ \c -> do
     crm <- clientRegisterMethodNormal c "/foo"
-    withServer grpc serverConf $ \s ->
+    withServer grpc serverConf $ \s -> do
-      withClientCall c crm 10 $ \cc -> do
+      ccVar <- newEmptyMVar
+      bracket newEmptyMVar (\v -> putMVar v ()) $ \finished -> do
+        _ <- forkIO $
+          void $
+            withClientCall c crm 10 $ \cc -> do
+              putMVar ccVar cc
+              -- NOTE: We need to send client ops here or else `withServerCall` hangs,
+              -- because registered methods try to do recv ops immediately when
+              -- created. If later we want to send payloads or metadata, we'll need
+              -- to tweak this.
+              _clientRes <- runOps (unsafeCC cc) (clientCQ c) clientEmptySendOps
+              takeMVar finished
+              pure (Right ())
         let srm = head (normalMethods s)
-        -- NOTE: We need to send client ops here or else `withServerCall` hangs,
+        cc <- takeMVar ccVar
-        -- because registered methods try to do recv ops immediately when
-        -- created. If later we want to send payloads or metadata, we'll need
-        -- to tweak this.
-        _clientRes <- runOps (unsafeCC cc) (clientCQ c) clientEmptySendOps
         withServerCall s srm $ \sc ->
           f (c, s, cc, sc)
 
@@ -65,16 +82,22 @@ clientConf :: ClientConfig
 clientConf = ClientConfig "localhost" 50051 [] Nothing Nothing
 
 clientEmptySendOps :: [Op]
-clientEmptySendOps = [OpSendInitialMetadata mempty,
+clientEmptySendOps =
-                      OpSendMessage "",
+  [ OpSendInitialMetadata mempty,
-                      OpSendCloseFromClient]
+    OpSendMessage "",
+    OpSendCloseFromClient
+  ]
 
 clientRecvOps :: [Op]
-clientRecvOps = [OpRecvInitialMetadata,
+clientRecvOps =
-                 OpRecvMessage,
+  [ OpRecvInitialMetadata,
-                 OpRecvStatusOnClient]
+    OpRecvMessage,
+    OpRecvStatusOnClient
+  ]
 
 serverEmptyRecvOps :: [Op]
-serverEmptyRecvOps = [OpSendInitialMetadata mempty,
+serverEmptyRecvOps =
-                      OpRecvMessage,
+  [ OpSendInitialMetadata mempty,
-                      OpRecvCloseOnServer]
+    OpRecvMessage,
+    OpRecvCloseOnServer
+  ]

examples/hellos/hellos-client/Main.hs

@@ -31,7 +31,7 @@ instance Message CSRpy
 data BiRqtRpy = BiRqtRpy { biMessage :: T.Text } deriving (Show, Eq, Ord, Generic)
 instance Message BiRqtRpy
 
-expect :: (Eq a, Monad m, Show a) => String -> a -> a -> m ()
+expect :: (Eq a, MonadFail m, Show a) => String -> a -> a -> m ()
 expect ctx ex got
   | ex /= got = fail $ ctx ++ " error: expected " ++ show ex ++ ", got " ++ show got
   | otherwise = return ()

examples/hellos/hellos-server/Main.hs

@@ -32,7 +32,7 @@ instance Message CSRpy
 data BiRqtRpy = BiRqtRpy { biMessage :: T.Text } deriving (Show, Eq, Ord, Generic)
 instance Message BiRqtRpy
 
-expect :: (Eq a, Monad m, Show a) => String -> a -> a -> m ()
+expect :: (Eq a, MonadFail m, Show a) => String -> a -> a -> m ()
 expect ctx ex got
   | ex /= got = fail $ ctx ++ " error: expected " ++ show ex ++ ", got " ++ show got
   | otherwise = return ()

fetch-nixpkgs.nix

@@ -1,60 +0,0 @@
-{ rev                             # The Git revision of nixpkgs to fetch
-, sha256                          # The SHA256 of the downloaded data
-, outputSha256 ? null             # The SHA256 output hash
-, system ? builtins.currentSystem # This is overridable if necessary
-}:
-
-with {
-  ifThenElse = { bool, thenValue, elseValue }: (
-    if bool then thenValue else elseValue);
-};
-
-ifThenElse {
-  bool = (0 <= builtins.compareVersions builtins.nixVersion "1.12");
-
-  # In Nix 1.12, we can just give a `sha256` to `builtins.fetchTarball`.
-  thenValue = (
-    builtins.fetchTarball {
-      url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
-
-      # builtins.fetchTarball does not need the sha256 hash of the
-      # packed and compressed tarball but it _does_ need the
-      # fixed-output sha256 hash.
-      sha256 = outputSha256;
-    });
-
-  # This hack should at least work for Nix 1.11
-  elseValue = (
-    (rec {
-      tarball = import <nix/fetchurl.nix> {
-        url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
-        inherit sha256;
-      };
-
-      builtin-paths = import <nix/config.nix>;
-      
-      script = builtins.toFile "nixpkgs-unpacker" ''
-        "$coreutils/mkdir" "$out"
-        cd "$out"
-        "$gzip" --decompress < "$tarball" | "$tar" -x --strip-components=1
-      '';
-
-      nixpkgs = builtins.derivation ({
-        name = "nixpkgs-${builtins.substring 0 6 rev}";
-
-        builder = builtins.storePath builtin-paths.shell;
-
-        args = [ script ];
-
-        inherit tarball system;
-
-        tar       = builtins.storePath builtin-paths.tar;
-        gzip      = builtins.storePath builtin-paths.gzip;
-        coreutils = builtins.storePath builtin-paths.coreutils;
-      } // (if null == outputSha256 then { } else {
-        outputHashMode = "recursive";
-        outputHashAlgo = "sha256";
-        outputHash = outputSha256;
-      }));
-    }).nixpkgs);
-}

nix/grpc.nix

@@ -1,16 +1,28 @@
-{ stdenv, fetchFromGitHub, cmake, zlib, c-ares, pkgconfig, openssl, protobuf, gflags }:
+{ lib, stdenv, fetchFromGitHub, fetchpatch, cmake, zlib, c-ares, pkg-config, openssl, protobuf
+, gflags, abseil-cpp, libnsl
+}:
 
 stdenv.mkDerivation rec {
-  version = "1.22.1";
+  version = "1.34.1"; # N.B: if you change this, change pythonPackages.grpcio-tools to a matching version too
-  name = "grpc-${version}";
+  pname = "grpc";
   src = fetchFromGitHub {
     owner = "grpc";
     repo = "grpc";
     rev = "v${version}";
-    sha256 = "1ci3v8xrr8iy65ixx2j0aw1i7cmmrm6pll1dnnbvndmjq573qiyk";
+    sha256 = "0p6si9i0gg885ag2x87a7jyzhgd5lhx2bh2vjj2ra1jn6y3vg6qk";
+    fetchSubmodules = true;
   };
-  nativeBuildInputs = [ cmake pkgconfig ];
+  patches = [
-  buildInputs = [ zlib c-ares c-ares.cmake-config openssl protobuf gflags ];
+    # Fix build on armv6l (https://github.com/grpc/grpc/pull/21341)
+    (fetchpatch {
+      url = "https://github.com/grpc/grpc/commit/2f4cf1d9265c8e10fb834f0794d0e4f3ec5ae10e.patch";
+      sha256 = "0ams3jmgh9yzwmxcg4ifb34znamr7pb4qm0609kvil9xqvkqz963";
+    })
+  ];
+
+  nativeBuildInputs = [ cmake pkg-config ];
+  buildInputs = [ zlib c-ares c-ares.cmake-config openssl protobuf gflags abseil-cpp ]
+    ++ lib.optionals stdenv.isLinux [ libnsl ];
 
   cmakeFlags =
     [ "-DgRPC_ZLIB_PROVIDER=package"
@@ -18,6 +30,7 @@ stdenv.mkDerivation rec {
       "-DgRPC_SSL_PROVIDER=package"
       "-DgRPC_PROTOBUF_PROVIDER=package"
       "-DgRPC_GFLAGS_PROVIDER=package"
+      "-DgRPC_ABSL_PROVIDER=package"
       "-DBUILD_SHARED_LIBS=ON"
       "-DCMAKE_SKIP_BUILD_RPATH=OFF"
     ];
@@ -29,17 +42,19 @@ stdenv.mkDerivation rec {
   '';
 
   preBuild = ''
-    export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH=$(pwd)''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH
   '';
 
-  NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.cc.isClang "-Wno-error=unknown-warning-option";
+  NIX_CFLAGS_COMPILE = lib.optionalString stdenv.cc.isClang "-Wno-error=unknown-warning-option";
 
   enableParallelBuilds = true;
 
-  meta = with stdenv.lib; {
+  meta = with lib; {
     description = "The C based gRPC (C++, Python, Ruby, Objective-C, PHP, C#)";
     license = licenses.asl20;
-    maintainers = [ maintainers.lnl7 ];
+    maintainers = [ maintainers.lnl7 maintainers.marsam ];
-    homepage = https://grpc.io/;
+    homepage = "https://grpc.io/";
+    platforms = platforms.all;
+    changelog = "https://github.com/grpc/grpc/releases/tag/v${version}";
   };
 }

nixpkgs.nix

@@ -6,8 +6,7 @@
 #
 # The SHA256 will be printed as the last line of stdout.
 
-import ./fetch-nixpkgs.nix {
+import (builtins.fetchTarball {
-    rev          = "fa12335f425808f53121713f501f3335878e6901";
+    url    = "https://github.com/NixOS/nixpkgs/archive/dd9f73e7d34486b09b966738ace161e621a0480b.tar.gz";
-    sha256       = "1fjyvjxvymz8yd65ahgm798jp9vdcfy7s58zb5ns2iq2ak0h9j8p";
+    sha256 = "0s674386v5b24a9fia26439gw9wsyhif85k2nzpxkp61293v3n3h";
-    outputSha256 = "1qkihrm8xfrh93c7wh1d1x01p7mgv82b2ycpmn9jm5l7976g31vr";
+  })
-}

release.nix

@@ -68,115 +68,14 @@ let
 
   overlay = pkgsNew: pkgsOld: {
 
-    cython = pkgsNew.pythonPackages.buildPythonPackage rec {
-      name = "Cython-${version}";
-      version = "0.24.1";
-
-      src = pkgsNew.fetchurl {
-        url = "mirror://pypi/C/Cython/${name}.tar.gz";
-        sha256 = "84808fda00508757928e1feadcf41c9f78e9a9b7167b6649ab0933b76f75e7b9";
-      };
-
-      # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729
-      # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround
-      # when that fix is stabilized
-      NIX_CFLAGS_COMPILE =
-        pkgsNew.stdenv.lib.optionalString (pkgsNew.stdenv.cc.isClang or false)
-          "-I${pkgsNew.libcxx}/include/c++/v1";
-
-      buildInputs =
-            pkgsNew.stdenv.lib.optional (pkgsNew.stdenv.cc.isClang or false) pkgsNew.libcxx
-        ++  [ pkgsNew.pkgconfig pkgsNew.gdb ];
-
-      doCheck = false;
-
-      doHaddock = false;
-
-      doHoogle = false;
-
-      meta = {
-        description = "An optimising static compiler for both the Python programming language and the extended Cython programming language";
-        platforms = pkgsNew.stdenv.lib.platforms.all;
-        homepage = http://cython.org;
-        license = pkgsNew.stdenv.lib.licenses.asl20;
-        maintainers = with pkgsNew.stdenv.lib.maintainers; [ fridh ];
-      };
-    };
-
     grpc = pkgsNew.callPackage ./nix/grpc.nix { };
 
-    grpcio = pkgsNew.pythonPackages.buildPythonPackage rec {
-      name = "grpc-${version}";
-
-      version = "1.0";
-
-      src = pkgsNew.fetchgit {
-        url    = "https://github.com/grpc/grpc.git";
-        rev    = "e2cfe9df79c4eda4e376222df064c4c65e616352";
-        sha256 = "19ldbjlnbc287hkaylsigm8w9fai2bjdbfxk6315kl75cq54iprr";
-      };
-
-      preConfigure = ''
-        export GRPC_PYTHON_BUILD_WITH_CYTHON=1
-      '';
-
-      # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729
-      # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround
-      # when that fix is stabilized
-      NIX_CFLAGS_COMPILE =
-        pkgsNew.stdenv.lib.optionalString (pkgsNew.stdenv.cc.isClang or false)
-          "-I${pkgsNew.libcxx}/include/c++/v1";
-
-      buildInputs =
-        pkgsNew.stdenv.lib.optional (pkgsNew.stdenv.cc.isClang or false) pkgsNew.libcxx;
-
-      propagatedBuildInputs = [
-        pkgsNew.cython
-        pkgsNew.pythonPackages.futures
-        pkgsNew.protobuf3_2NoCheck
-        pkgsNew.pythonPackages.enum34
-      ];
-    };
-
-    grpcio-tools = pkgsNew.pythonPackages.buildPythonPackage rec {
-      name = "grpc-${version}";
-
-      version = "1.0";
-
-      src = pkgsNew.fetchgit {
-        url    = "https://github.com/grpc/grpc.git";
-        rev    = "e2cfe9df79c4eda4e376222df064c4c65e616352";
-        sha256 = "19ldbjlnbc287hkaylsigm8w9fai2bjdbfxk6315kl75cq54iprr";
-      };
-
-      preConfigure = ''
-        export GRPC_PYTHON_BUILD_WITH_CYTHON=1
-        cd tools/distrib/python/grpcio_tools
-        python ../make_grpcio_tools.py
-      '';
-
-      # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729
-      # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround
-      # when that fix is stabilized
-      NIX_CFLAGS_COMPILE =
-        pkgsNew.stdenv.lib.optionalString (pkgsNew.stdenv.cc.isClang or false)
-          "-I${pkgsNew.libcxx}/include/c++/v1";
-
-      buildInputs =
-        pkgsNew.stdenv.lib.optional (pkgsNew.stdenv.cc.isClang or false) pkgsNew.libcxx;
-
-      propagatedBuildInputs = [
-        pkgsNew.cython
-        pkgsNew.pythonPackages.futures
-        pkgsNew.protobuf3_2NoCheck
-        pkgsNew.pythonPackages.enum34
-        pkgsNew.grpcio
-      ];
-    };
-
     haskellPackages = pkgsOld.haskellPackages.override {
       overrides = haskellPackagesNew: haskellPackagesOld: rec {
 
+        haskell-src =
+          haskellPackagesNew.callHackage "haskell-src" "1.0.3.1" {};
+
         proto3-wire =
           haskellPackagesNew.callPackage ./nix/proto3-wire.nix { };
 
@@ -213,7 +112,7 @@ let
 
                   python = pkgsNew.python.withPackages (pkgs: [
                     # pkgs.protobuf3_0
-                    pkgsNew.grpcio-tools
+                    pkgs.grpcio-tools
                   ]);
 
                 in rec {
@@ -307,9 +206,10 @@ let
 in
 
 let
-   linuxPkgs = import nixpkgs { inherit config overlays; system = "x86_64-linux" ; };
+   nixpkgs = import ./nixpkgs.nix;
-  darwinPkgs = import nixpkgs { inherit config overlays; system = "x86_64-darwin"; };
+   linuxPkgs = nixpkgs { inherit config overlays; system = "x86_64-linux" ; };
-        pkgs = import nixpkgs { inherit config overlays; };
+  darwinPkgs = nixpkgs { inherit config overlays; system = "x86_64-darwin"; };
+        pkgs = nixpkgs { inherit config overlays; };
 
 in
   {