From 0c57ab0785d34afcbb7c7832bf836f1a10cb450c Mon Sep 17 00:00:00 2001 From: Tim McGilchrist Date: Tue, 9 Mar 2021 08:44:36 +1100 Subject: [PATCH] Upgrade gRPC C library (#117) --- core/cbits/grpc_haskell.c | 2 +- core/src/Network/GRPC/Unsafe.chs | 3 - core/tests/LowLevelTests/Op.hs | 119 +++++++++++++++----------- examples/hellos/hellos-client/Main.hs | 2 +- examples/hellos/hellos-server/Main.hs | 2 +- fetch-nixpkgs.nix | 60 ------------- nix/grpc.nix | 37 +++++--- nixpkgs.nix | 9 +- release.nix | 116 ++----------------------- 9 files changed, 112 insertions(+), 238 deletions(-) delete mode 100644 fetch-nixpkgs.nix diff --git a/core/cbits/grpc_haskell.c b/core/cbits/grpc_haskell.c index ebbcbb1..d383983 100644 --- a/core/cbits/grpc_haskell.c +++ b/core/cbits/grpc_haskell.c @@ -505,7 +505,7 @@ grpc_auth_metadata_processor* mk_auth_metadata_processor( grpc_call_credentials* grpc_metadata_credentials_create_from_plugin_( grpc_metadata_credentials_plugin* plugin){ - return grpc_metadata_credentials_create_from_plugin(*plugin, NULL); + return grpc_metadata_credentials_create_from_plugin(*plugin, GRPC_PRIVACY_AND_INTEGRITY, NULL); } //This is a hack to work around GHC being unable to deal with raw struct params. diff --git a/core/src/Network/GRPC/Unsafe.chs b/core/src/Network/GRPC/Unsafe.chs index d11f533..6eb08b8 100644 --- a/core/src/Network/GRPC/Unsafe.chs +++ b/core/src/Network/GRPC/Unsafe.chs @@ -219,9 +219,6 @@ castPeek p = do unTag `Tag'} -> `()'#} -{#fun grpc_channel_ping as ^ - {`Channel', `CompletionQueue', unTag `Tag',unReserved `Reserved'} -> `()' #} - {#fun grpc_channel_destroy as ^ {`Channel'} -> `()'#} -- | Starts executing a batch of ops in the given 'OpArray'. Does not block. diff --git a/core/tests/LowLevelTests/Op.hs b/core/tests/LowLevelTests/Op.hs index 52b6d55..dae4ba7 100644 --- a/core/tests/LowLevelTests/Op.hs +++ b/core/tests/LowLevelTests/Op.hs @@ -1,60 +1,77 @@ -{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE RecordWildCards #-} module LowLevelTests.Op where -import Data.ByteString (ByteString) -import Test.Tasty -import Test.Tasty.HUnit as HU (testCase, (@?=)) - -import Network.GRPC.LowLevel -import Network.GRPC.LowLevel.Call -import Network.GRPC.LowLevel.Client -import Network.GRPC.LowLevel.Server -import Network.GRPC.LowLevel.Op +import Control.Concurrent +import Control.Exception +import Control.Monad +import Data.ByteString (ByteString) +import Network.GRPC.LowLevel +import Network.GRPC.LowLevel.Call +import Network.GRPC.LowLevel.Client +import Network.GRPC.LowLevel.Op +import Network.GRPC.LowLevel.Server +import Test.Tasty +import Test.Tasty.HUnit as HU (testCase, (@?=)) lowLevelOpTests :: TestTree -lowLevelOpTests = testGroup "Synchronous unit tests of low-level Op interface" - [testCancelFromServer] +lowLevelOpTests = + testGroup + "Synchronous unit tests of low-level Op interface" + [testCancelFromServer] testCancelFromServer :: TestTree testCancelFromServer = testCase "Client/Server - client receives server cancellation" $ - runSerialTest $ \grpc -> - withClientServerUnaryCall grpc $ - \(Client{..}, Server{..}, ClientCall{..}, sc@ServerCall{..}) -> do - serverCallCancel sc StatusPermissionDenied "TestStatus" - clientRes <- runOps unsafeCC clientCQ clientRecvOps - case clientRes of - Left x -> error $ "Client recv error: " ++ show x - Right [_,_,OpRecvStatusOnClientResult _ code _details] -> do - code @?= StatusPermissionDenied - return $ Right () - wrong -> error $ "Unexpected op results: " ++ show wrong - + runSerialTest $ \grpc -> + withClientServerUnaryCall grpc $ + \(Client {..}, Server {..}, ClientCall {..}, sc@ServerCall {..}) -> do + serverCallCancel sc StatusPermissionDenied "TestStatus" + clientRes <- runOps unsafeCC clientCQ clientRecvOps + case clientRes of + Left x -> error $ "Client recv error: " ++ show x + Right [_, _, OpRecvStatusOnClientResult _ code _details] -> do + code @?= StatusPermissionDenied + return $ Right () + wrong -> error $ "Unexpected op results: " ++ show wrong runSerialTest :: (GRPC -> IO (Either GRPCIOError ())) -> IO () runSerialTest f = - withGRPC f >>= \case Left x -> error $ show x - Right () -> return () + withGRPC f >>= \case + Left x -> error $ show x + Right () -> return () -withClientServerUnaryCall :: GRPC - -> ((Client, Server, ClientCall, - ServerCall ByteString) - -> IO (Either GRPCIOError a)) - -> IO (Either GRPCIOError a) +withClientServerUnaryCall :: + GRPC -> + ( ( Client, + Server, + ClientCall, + ServerCall ByteString + ) -> + IO (Either GRPCIOError a) + ) -> + IO (Either GRPCIOError a) withClientServerUnaryCall grpc f = do withClient grpc clientConf $ \c -> do crm <- clientRegisterMethodNormal c "/foo" - withServer grpc serverConf $ \s -> - withClientCall c crm 10 $ \cc -> do + withServer grpc serverConf $ \s -> do + ccVar <- newEmptyMVar + bracket newEmptyMVar (\v -> putMVar v ()) $ \finished -> do + _ <- forkIO $ + void $ + withClientCall c crm 10 $ \cc -> do + putMVar ccVar cc + -- NOTE: We need to send client ops here or else `withServerCall` hangs, + -- because registered methods try to do recv ops immediately when + -- created. If later we want to send payloads or metadata, we'll need + -- to tweak this. + _clientRes <- runOps (unsafeCC cc) (clientCQ c) clientEmptySendOps + takeMVar finished + pure (Right ()) let srm = head (normalMethods s) - -- NOTE: We need to send client ops here or else `withServerCall` hangs, - -- because registered methods try to do recv ops immediately when - -- created. If later we want to send payloads or metadata, we'll need - -- to tweak this. - _clientRes <- runOps (unsafeCC cc) (clientCQ c) clientEmptySendOps + cc <- takeMVar ccVar withServerCall s srm $ \sc -> f (c, s, cc, sc) @@ -65,16 +82,22 @@ clientConf :: ClientConfig clientConf = ClientConfig "localhost" 50051 [] Nothing Nothing clientEmptySendOps :: [Op] -clientEmptySendOps = [OpSendInitialMetadata mempty, - OpSendMessage "", - OpSendCloseFromClient] +clientEmptySendOps = + [ OpSendInitialMetadata mempty, + OpSendMessage "", + OpSendCloseFromClient + ] clientRecvOps :: [Op] -clientRecvOps = [OpRecvInitialMetadata, - OpRecvMessage, - OpRecvStatusOnClient] +clientRecvOps = + [ OpRecvInitialMetadata, + OpRecvMessage, + OpRecvStatusOnClient + ] serverEmptyRecvOps :: [Op] -serverEmptyRecvOps = [OpSendInitialMetadata mempty, - OpRecvMessage, - OpRecvCloseOnServer] +serverEmptyRecvOps = + [ OpSendInitialMetadata mempty, + OpRecvMessage, + OpRecvCloseOnServer + ] diff --git a/examples/hellos/hellos-client/Main.hs b/examples/hellos/hellos-client/Main.hs index 5cfea55..5a36bd4 100644 --- a/examples/hellos/hellos-client/Main.hs +++ b/examples/hellos/hellos-client/Main.hs @@ -31,7 +31,7 @@ instance Message CSRpy data BiRqtRpy = BiRqtRpy { biMessage :: T.Text } deriving (Show, Eq, Ord, Generic) instance Message BiRqtRpy -expect :: (Eq a, Monad m, Show a) => String -> a -> a -> m () +expect :: (Eq a, MonadFail m, Show a) => String -> a -> a -> m () expect ctx ex got | ex /= got = fail $ ctx ++ " error: expected " ++ show ex ++ ", got " ++ show got | otherwise = return () diff --git a/examples/hellos/hellos-server/Main.hs b/examples/hellos/hellos-server/Main.hs index 7fdc02a..4512938 100644 --- a/examples/hellos/hellos-server/Main.hs +++ b/examples/hellos/hellos-server/Main.hs @@ -32,7 +32,7 @@ instance Message CSRpy data BiRqtRpy = BiRqtRpy { biMessage :: T.Text } deriving (Show, Eq, Ord, Generic) instance Message BiRqtRpy -expect :: (Eq a, Monad m, Show a) => String -> a -> a -> m () +expect :: (Eq a, MonadFail m, Show a) => String -> a -> a -> m () expect ctx ex got | ex /= got = fail $ ctx ++ " error: expected " ++ show ex ++ ", got " ++ show got | otherwise = return () diff --git a/fetch-nixpkgs.nix b/fetch-nixpkgs.nix deleted file mode 100644 index bf12e8b..0000000 --- a/fetch-nixpkgs.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ rev # The Git revision of nixpkgs to fetch -, sha256 # The SHA256 of the downloaded data -, outputSha256 ? null # The SHA256 output hash -, system ? builtins.currentSystem # This is overridable if necessary -}: - -with { - ifThenElse = { bool, thenValue, elseValue }: ( - if bool then thenValue else elseValue); -}; - -ifThenElse { - bool = (0 <= builtins.compareVersions builtins.nixVersion "1.12"); - - # In Nix 1.12, we can just give a `sha256` to `builtins.fetchTarball`. - thenValue = ( - builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz"; - - # builtins.fetchTarball does not need the sha256 hash of the - # packed and compressed tarball but it _does_ need the - # fixed-output sha256 hash. - sha256 = outputSha256; - }); - - # This hack should at least work for Nix 1.11 - elseValue = ( - (rec { - tarball = import { - url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz"; - inherit sha256; - }; - - builtin-paths = import ; - - script = builtins.toFile "nixpkgs-unpacker" '' - "$coreutils/mkdir" "$out" - cd "$out" - "$gzip" --decompress < "$tarball" | "$tar" -x --strip-components=1 - ''; - - nixpkgs = builtins.derivation ({ - name = "nixpkgs-${builtins.substring 0 6 rev}"; - - builder = builtins.storePath builtin-paths.shell; - - args = [ script ]; - - inherit tarball system; - - tar = builtins.storePath builtin-paths.tar; - gzip = builtins.storePath builtin-paths.gzip; - coreutils = builtins.storePath builtin-paths.coreutils; - } // (if null == outputSha256 then { } else { - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = outputSha256; - })); - }).nixpkgs); -} diff --git a/nix/grpc.nix b/nix/grpc.nix index 729e95b..92cf619 100644 --- a/nix/grpc.nix +++ b/nix/grpc.nix @@ -1,16 +1,28 @@ -{ stdenv, fetchFromGitHub, cmake, zlib, c-ares, pkgconfig, openssl, protobuf, gflags }: +{ lib, stdenv, fetchFromGitHub, fetchpatch, cmake, zlib, c-ares, pkg-config, openssl, protobuf +, gflags, abseil-cpp, libnsl +}: stdenv.mkDerivation rec { - version = "1.22.1"; - name = "grpc-${version}"; + version = "1.34.1"; # N.B: if you change this, change pythonPackages.grpcio-tools to a matching version too + pname = "grpc"; src = fetchFromGitHub { owner = "grpc"; repo = "grpc"; rev = "v${version}"; - sha256 = "1ci3v8xrr8iy65ixx2j0aw1i7cmmrm6pll1dnnbvndmjq573qiyk"; + sha256 = "0p6si9i0gg885ag2x87a7jyzhgd5lhx2bh2vjj2ra1jn6y3vg6qk"; + fetchSubmodules = true; }; - nativeBuildInputs = [ cmake pkgconfig ]; - buildInputs = [ zlib c-ares c-ares.cmake-config openssl protobuf gflags ]; + patches = [ + # Fix build on armv6l (https://github.com/grpc/grpc/pull/21341) + (fetchpatch { + url = "https://github.com/grpc/grpc/commit/2f4cf1d9265c8e10fb834f0794d0e4f3ec5ae10e.patch"; + sha256 = "0ams3jmgh9yzwmxcg4ifb34znamr7pb4qm0609kvil9xqvkqz963"; + }) + ]; + + nativeBuildInputs = [ cmake pkg-config ]; + buildInputs = [ zlib c-ares c-ares.cmake-config openssl protobuf gflags abseil-cpp ] + ++ lib.optionals stdenv.isLinux [ libnsl ]; cmakeFlags = [ "-DgRPC_ZLIB_PROVIDER=package" @@ -18,6 +30,7 @@ stdenv.mkDerivation rec { "-DgRPC_SSL_PROVIDER=package" "-DgRPC_PROTOBUF_PROVIDER=package" "-DgRPC_GFLAGS_PROVIDER=package" + "-DgRPC_ABSL_PROVIDER=package" "-DBUILD_SHARED_LIBS=ON" "-DCMAKE_SKIP_BUILD_RPATH=OFF" ]; @@ -29,17 +42,19 @@ stdenv.mkDerivation rec { ''; preBuild = '' - export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=$(pwd)''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH ''; - NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.cc.isClang "-Wno-error=unknown-warning-option"; + NIX_CFLAGS_COMPILE = lib.optionalString stdenv.cc.isClang "-Wno-error=unknown-warning-option"; enableParallelBuilds = true; - meta = with stdenv.lib; { + meta = with lib; { description = "The C based gRPC (C++, Python, Ruby, Objective-C, PHP, C#)"; license = licenses.asl20; - maintainers = [ maintainers.lnl7 ]; - homepage = https://grpc.io/; + maintainers = [ maintainers.lnl7 maintainers.marsam ]; + homepage = "https://grpc.io/"; + platforms = platforms.all; + changelog = "https://github.com/grpc/grpc/releases/tag/v${version}"; }; } diff --git a/nixpkgs.nix b/nixpkgs.nix index b08c8a1..1c22faf 100644 --- a/nixpkgs.nix +++ b/nixpkgs.nix @@ -6,8 +6,7 @@ # # The SHA256 will be printed as the last line of stdout. -import ./fetch-nixpkgs.nix { - rev = "fa12335f425808f53121713f501f3335878e6901"; - sha256 = "1fjyvjxvymz8yd65ahgm798jp9vdcfy7s58zb5ns2iq2ak0h9j8p"; - outputSha256 = "1qkihrm8xfrh93c7wh1d1x01p7mgv82b2ycpmn9jm5l7976g31vr"; -} +import (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/dd9f73e7d34486b09b966738ace161e621a0480b.tar.gz"; + sha256 = "0s674386v5b24a9fia26439gw9wsyhif85k2nzpxkp61293v3n3h"; + }) diff --git a/release.nix b/release.nix index 70457c9..405c87a 100644 --- a/release.nix +++ b/release.nix @@ -68,115 +68,14 @@ let overlay = pkgsNew: pkgsOld: { - cython = pkgsNew.pythonPackages.buildPythonPackage rec { - name = "Cython-${version}"; - version = "0.24.1"; - - src = pkgsNew.fetchurl { - url = "mirror://pypi/C/Cython/${name}.tar.gz"; - sha256 = "84808fda00508757928e1feadcf41c9f78e9a9b7167b6649ab0933b76f75e7b9"; - }; - - # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729 - # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround - # when that fix is stabilized - NIX_CFLAGS_COMPILE = - pkgsNew.stdenv.lib.optionalString (pkgsNew.stdenv.cc.isClang or false) - "-I${pkgsNew.libcxx}/include/c++/v1"; - - buildInputs = - pkgsNew.stdenv.lib.optional (pkgsNew.stdenv.cc.isClang or false) pkgsNew.libcxx - ++ [ pkgsNew.pkgconfig pkgsNew.gdb ]; - - doCheck = false; - - doHaddock = false; - - doHoogle = false; - - meta = { - description = "An optimising static compiler for both the Python programming language and the extended Cython programming language"; - platforms = pkgsNew.stdenv.lib.platforms.all; - homepage = http://cython.org; - license = pkgsNew.stdenv.lib.licenses.asl20; - maintainers = with pkgsNew.stdenv.lib.maintainers; [ fridh ]; - }; - }; - grpc = pkgsNew.callPackage ./nix/grpc.nix { }; - grpcio = pkgsNew.pythonPackages.buildPythonPackage rec { - name = "grpc-${version}"; - - version = "1.0"; - - src = pkgsNew.fetchgit { - url = "https://github.com/grpc/grpc.git"; - rev = "e2cfe9df79c4eda4e376222df064c4c65e616352"; - sha256 = "19ldbjlnbc287hkaylsigm8w9fai2bjdbfxk6315kl75cq54iprr"; - }; - - preConfigure = '' - export GRPC_PYTHON_BUILD_WITH_CYTHON=1 - ''; - - # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729 - # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround - # when that fix is stabilized - NIX_CFLAGS_COMPILE = - pkgsNew.stdenv.lib.optionalString (pkgsNew.stdenv.cc.isClang or false) - "-I${pkgsNew.libcxx}/include/c++/v1"; - - buildInputs = - pkgsNew.stdenv.lib.optional (pkgsNew.stdenv.cc.isClang or false) pkgsNew.libcxx; - - propagatedBuildInputs = [ - pkgsNew.cython - pkgsNew.pythonPackages.futures - pkgsNew.protobuf3_2NoCheck - pkgsNew.pythonPackages.enum34 - ]; - }; - - grpcio-tools = pkgsNew.pythonPackages.buildPythonPackage rec { - name = "grpc-${version}"; - - version = "1.0"; - - src = pkgsNew.fetchgit { - url = "https://github.com/grpc/grpc.git"; - rev = "e2cfe9df79c4eda4e376222df064c4c65e616352"; - sha256 = "19ldbjlnbc287hkaylsigm8w9fai2bjdbfxk6315kl75cq54iprr"; - }; - - preConfigure = '' - export GRPC_PYTHON_BUILD_WITH_CYTHON=1 - cd tools/distrib/python/grpcio_tools - python ../make_grpcio_tools.py - ''; - - # This workaround was taken from https://github.com/NixOS/nixpkgs/issues/18729 - # This was fixed in `nixpkgs-unstable` so we can get rid of this workaround - # when that fix is stabilized - NIX_CFLAGS_COMPILE = - pkgsNew.stdenv.lib.optionalString (pkgsNew.stdenv.cc.isClang or false) - "-I${pkgsNew.libcxx}/include/c++/v1"; - - buildInputs = - pkgsNew.stdenv.lib.optional (pkgsNew.stdenv.cc.isClang or false) pkgsNew.libcxx; - - propagatedBuildInputs = [ - pkgsNew.cython - pkgsNew.pythonPackages.futures - pkgsNew.protobuf3_2NoCheck - pkgsNew.pythonPackages.enum34 - pkgsNew.grpcio - ]; - }; - haskellPackages = pkgsOld.haskellPackages.override { overrides = haskellPackagesNew: haskellPackagesOld: rec { + haskell-src = + haskellPackagesNew.callHackage "haskell-src" "1.0.3.1" {}; + proto3-wire = haskellPackagesNew.callPackage ./nix/proto3-wire.nix { }; @@ -213,7 +112,7 @@ let python = pkgsNew.python.withPackages (pkgs: [ # pkgs.protobuf3_0 - pkgsNew.grpcio-tools + pkgs.grpcio-tools ]); in rec { @@ -307,9 +206,10 @@ let in let - linuxPkgs = import nixpkgs { inherit config overlays; system = "x86_64-linux" ; }; - darwinPkgs = import nixpkgs { inherit config overlays; system = "x86_64-darwin"; }; - pkgs = import nixpkgs { inherit config overlays; }; + nixpkgs = import ./nixpkgs.nix; + linuxPkgs = nixpkgs { inherit config overlays; system = "x86_64-linux" ; }; + darwinPkgs = nixpkgs { inherit config overlays; system = "x86_64-darwin"; }; + pkgs = nixpkgs { inherit config overlays; }; in {