From c66c912c327810796b6282ae77c33e336c1dadea Mon Sep 17 00:00:00 2001 From: jcmartin Date: Fri, 6 Nov 2020 11:32:21 -0800 Subject: [PATCH] Tensorflow 2.3.0 Support (#267) * Tensorflow 2.3.0 building and passing tests. * Added einsum and test. * Added ByteString as a possible argument to a function. * Support more data types for Adam. * Move to later version of LTS on stackage. * Added a wrapper module for convolution functions. * Update ci build to use a later version of stack. * Removed a deprecated import in GradientTest. --- .gitignore | 1 + ci_build/Dockerfile | 14 +- docker/Dockerfile | 10 +- docker/gpu/Dockerfile | 10 +- stack.yaml | 2 +- tensorflow-core-ops/Setup.hs | 11 + tensorflow-core-ops/tensorflow-core-ops.cabal | 4 +- tensorflow-logging/tensorflow-logging.cabal | 2 +- tensorflow-mnist/tensorflow-mnist.cabal | 2 +- .../src/TensorFlow/OpGen/ParsedOp.hs | 2 +- tensorflow-opgen/tensorflow-opgen.cabal | 2 +- tensorflow-ops/src/TensorFlow/Convolution.hs | 300 ++++++++++++++++++ tensorflow-ops/src/TensorFlow/Gradient.hs | 27 +- tensorflow-ops/src/TensorFlow/Minimize.hs | 29 +- tensorflow-ops/src/TensorFlow/Ops.hs | 2 + tensorflow-ops/tensorflow-ops.cabal | 3 +- tensorflow-ops/tests/GradientTest.hs | 25 +- tensorflow-ops/tests/OpsTest.hs | 15 + tensorflow-proto/tensorflow-proto.cabal | 19 +- tensorflow/tensorflow.cabal | 2 +- third_party/tensorflow | 2 +- 21 files changed, 409 insertions(+), 75 deletions(-) create mode 100644 tensorflow-ops/src/TensorFlow/Convolution.hs diff --git a/.gitignore b/.gitignore index c9f81e4..7195879 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .stack/ tensorflow-mnist-input-data/data/*.gz .DS_Store +stack.yaml.lock diff --git a/ci_build/Dockerfile b/ci_build/Dockerfile index 2c26d3e..fcf33f0 100644 --- a/ci_build/Dockerfile +++ b/ci_build/Dockerfile @@ -3,7 +3,7 @@ # stack to be installed on the host. This comes at the expense of # flexibility. -FROM tensorflow/tensorflow:1.14.0 +FROM tensorflow/tensorflow:2.3.0 LABEL maintainer="TensorFlow authors " # The build context directory is the top of the tensorflow-haskell @@ -30,14 +30,14 @@ RUN \ netbase \ && \ # Installs stack. - curl -O -L https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-x86_64.tar.gz && \ - tar zxf stack-2.1.3-linux-x86_64.tar.gz -C /usr/local/bin stack-2.1.3-linux-x86_64/stack --strip 1 && \ + curl -O -L https://github.com/commercialhaskell/stack/releases/download/v2.5.1/stack-2.5.1-linux-x86_64.tar.gz && \ + tar zxf stack-2.5.1-linux-x86_64.tar.gz -C /usr/local/bin stack-2.5.1-linux-x86_64/stack --strip 1 && \ # Installs protoc and the libraries. - curl -O -L https://github.com/google/protobuf/releases/download/v3.9.1/protoc-3.9.1-linux-x86_64.zip && \ - unzip -d /usr/local protoc-3.9.1-linux-x86_64.zip bin/protoc && \ + curl -O -L https://github.com/google/protobuf/releases/download/v3.13.0/protoc-3.13.0-linux-x86_64.zip && \ + unzip -d /usr/local protoc-3.13.0-linux-x86_64.zip bin/protoc && \ chmod 755 /usr/local/bin/protoc && \ - curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz && \ - tar zxf libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz -C /usr/local && \ + curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.3.0.tar.gz && \ + tar zxf libtensorflow-cpu-linux-x86_64-2.3.0.tar.gz -C /usr/local && \ ldconfig && \ stack setup && \ stack test --only-dependencies diff --git a/docker/Dockerfile b/docker/Dockerfile index 25f9d24..c0a1f16 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # Prepare the image with: # docker build -t tensorflow/haskell:v0 docker -FROM tensorflow/tensorflow:1.14.0 +FROM tensorflow/tensorflow:2.3.0 LABEL maintainer="TensorFlow authors " RUN apt-get update @@ -28,11 +28,11 @@ RUN dpkg-reconfigure locales && \ # Installs protoc and the libraries. RUN \ - curl -O -L https://github.com/google/protobuf/releases/download/v3.9.1/protoc-3.9.1-linux-x86_64.zip && \ - unzip -d /usr/local protoc-3.9.1-linux-x86_64.zip bin/protoc && \ + curl -O -L https://github.com/google/protobuf/releases/download/v3.13.0/protoc-3.13.0-linux-x86_64.zip && \ + unzip -d /usr/local protoc-3.13.0-linux-x86_64.zip bin/protoc && \ chmod 755 /usr/local/bin/protoc && \ - curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz && \ - tar zxf libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz -C /usr/local && \ + curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.3.0.tar.gz && \ + tar zxf libtensorflow-cpu-linux-x86_64-2.3.0.tar.gz -C /usr/local && \ ldconfig ENV LANG en_US.UTF-8 diff --git a/docker/gpu/Dockerfile b/docker/gpu/Dockerfile index 81272bd..dfc22c7 100644 --- a/docker/gpu/Dockerfile +++ b/docker/gpu/Dockerfile @@ -1,6 +1,6 @@ # Prepare the image with: # docker build -t tensorflow/haskell:1.14.0-gpu docker/gpu -FROM tensorflow/tensorflow:1.14.0-gpu +FROM tensorflow/tensorflow:2.3.0-gpu LABEL maintainer="TensorFlow authors " RUN apt-get update @@ -28,11 +28,11 @@ RUN dpkg-reconfigure locales && \ # Installs protoc and the libraries. RUN \ - curl -O -L https://github.com/google/protobuf/releases/download/v3.9.1/protoc-3.9.1-linux-x86_64.zip && \ - unzip -d /usr/local protoc-3.9.1-linux-x86_64.zip bin/protoc && \ + curl -O -L https://github.com/google/protobuf/releases/download/v3.13.0/protoc-3.13.0-linux-x86_64.zip && \ + unzip -d /usr/local protoc-3.13.0-linux-x86_64.zip bin/protoc && \ chmod 755 /usr/local/bin/protoc && \ - curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz && \ - tar zxf libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz -C /usr/local && \ + curl -O https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.3.0.tar.gz && \ + tar zxf libtensorflow-gpu-linux-x86_64-2.3.0.tar.gz -C /usr/local && \ ldconfig ENV LANG en_US.UTF-8 diff --git a/stack.yaml b/stack.yaml index 1eeb681..a169b6c 100644 --- a/stack.yaml +++ b/stack.yaml @@ -1,4 +1,4 @@ -resolver: lts-15.11 +resolver: lts-16.18 packages: - tensorflow diff --git a/tensorflow-core-ops/Setup.hs b/tensorflow-core-ops/Setup.hs index 7d478dd..aebd49d 100644 --- a/tensorflow-core-ops/Setup.hs +++ b/tensorflow-core-ops/Setup.hs @@ -105,6 +105,8 @@ blackList = , "GroupByWindowDataset" , "If" , "InterleaveDataset" + , "LegacyParallelInterleaveDatasetV2" + , "LoadDataset" , "MapAndBatchDataset" , "MapAndBatchDatasetV2" , "MapDataset" @@ -112,16 +114,24 @@ blackList = , "OneShotIterator" , "ParallelInterleaveDataset" , "ParallelInterleaveDatasetV2" + , "ParallelInterleaveDatasetV3" + , "ParallelInterleaveDatasetV4" , "ParallelMapDataset" + , "ParallelMapDatasetV2" , "ParseSequenceExample" + , "ParseSequenceExampleV2" + , "ParseSingleSequenceExample" , "PartitionedCall" , "ReduceDataset" , "RemoteCall" + , "SaveDataset" , "ScanDataset" + , "SnapshotDatasetV2" , "StatefulPartitionedCall" , "StatelessIf" , "StatelessWhile" , "SymbolicGradient" + , "TakeWhileDataset" , "TPUPartitionedCall" , "TPUReplicate" , "While" @@ -130,6 +140,7 @@ blackList = , "XlaReduce" , "XlaReduceWindow" , "XlaSelectAndScatter" + , "XlaScatter" , "XlaWhile" , "_If" , "_TPUReplicate" diff --git a/tensorflow-core-ops/tensorflow-core-ops.cabal b/tensorflow-core-ops/tensorflow-core-ops.cabal index 497d7a4..1347346 100644 --- a/tensorflow-core-ops/tensorflow-core-ops.cabal +++ b/tensorflow-core-ops/tensorflow-core-ops.cabal @@ -16,7 +16,7 @@ library exposed-modules: TensorFlow.GenOps.Core autogen-modules: TensorFlow.GenOps.Core build-depends: bytestring - , proto-lens == 0.6.* + , proto-lens == 0.7.* , tensorflow == 0.2.* , base >= 4.7 && < 5 , lens-family == 2.* @@ -27,7 +27,7 @@ custom-setup setup-depends: Cabal , bytestring , directory - , proto-lens == 0.6.* + , proto-lens == 0.7.* , tensorflow-opgen == 0.2.* , tensorflow == 0.2.* , base >= 4.7 && < 5 diff --git a/tensorflow-logging/tensorflow-logging.cabal b/tensorflow-logging/tensorflow-logging.cabal index b79cf98..e3756cc 100644 --- a/tensorflow-logging/tensorflow-logging.cabal +++ b/tensorflow-logging/tensorflow-logging.cabal @@ -24,7 +24,7 @@ library , filepath , hostname , lens-family == 2.* - , proto-lens == 0.6.* + , proto-lens == 0.7.* , resourcet , stm , stm-chans diff --git a/tensorflow-mnist/tensorflow-mnist.cabal b/tensorflow-mnist/tensorflow-mnist.cabal index f628140..52f2500 100644 --- a/tensorflow-mnist/tensorflow-mnist.cabal +++ b/tensorflow-mnist/tensorflow-mnist.cabal @@ -20,7 +20,7 @@ library exposed-modules: TensorFlow.Examples.MNIST.Parse , TensorFlow.Examples.MNIST.TrainedGraph other-modules: Paths_tensorflow_mnist - build-depends: proto-lens == 0.6.* + build-depends: proto-lens == 0.7.* , base >= 4.7 && < 5 , binary , bytestring diff --git a/tensorflow-opgen/src/TensorFlow/OpGen/ParsedOp.hs b/tensorflow-opgen/src/TensorFlow/OpGen/ParsedOp.hs index 2649d86..e0fba89 100644 --- a/tensorflow-opgen/src/TensorFlow/OpGen/ParsedOp.hs +++ b/tensorflow-opgen/src/TensorFlow/OpGen/ParsedOp.hs @@ -266,7 +266,7 @@ getExplicitInputAttr o implicitAttrs a , a ^. maybe'defaultValue == Nothing , t <- parseAttrType o (a ^. type') , t `elem` map AttrSingle - [AttrBool, AttrInt64, AttrFloat, AttrType, AttrShape] + [AttrBool, AttrInt64, AttrFloat, AttrType, AttrShape, AttrBytes] ++ [AttrList AttrType] = Just t | otherwise = Nothing diff --git a/tensorflow-opgen/tensorflow-opgen.cabal b/tensorflow-opgen/tensorflow-opgen.cabal index e867121..3223256 100644 --- a/tensorflow-opgen/tensorflow-opgen.cabal +++ b/tensorflow-opgen/tensorflow-opgen.cabal @@ -16,7 +16,7 @@ library hs-source-dirs: src exposed-modules: TensorFlow.OpGen.ParsedOp , TensorFlow.OpGen - build-depends: proto-lens == 0.6.* + build-depends: proto-lens == 0.7.* , tensorflow-proto == 0.2.* , base >= 4.7 && < 5 , bytestring diff --git a/tensorflow-ops/src/TensorFlow/Convolution.hs b/tensorflow-ops/src/TensorFlow/Convolution.hs new file mode 100644 index 0000000..94c9c8a --- /dev/null +++ b/tensorflow-ops/src/TensorFlow/Convolution.hs @@ -0,0 +1,300 @@ +-- Copyright 2020 TensorFlow authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE OverloadedStrings #-} + +module TensorFlow.Convolution + ( Padding(..) + , DataFormat(..) + , conv2D + , conv2D' + , conv2DBackpropFilter + , conv2DBackpropFilter' + , conv2DBackpropInput + , conv2DBackpropInput' + , conv3D + , conv3D' + , conv3DBackpropFilter + , conv3DBackpropFilter' + , conv3DBackpropFilterV2 + , conv3DBackpropFilterV2' + , conv3DBackpropInput + , conv3DBackpropInput' + , conv3DBackpropInputV2 + , conv3DBackpropInputV2' + , depthwiseConv2dNative + , depthwiseConv2dNative' + , depthwiseConv2dNativeBackpropFilter + , depthwiseConv2dNativeBackpropFilter' + , depthwiseConv2dNativeBackpropInput + , depthwiseConv2dNativeBackpropInput' + ) where + +import Data.Word (Word16) +import Data.Int (Int32,Int64) +import Data.ByteString (ByteString) +import Lens.Family2 ((.~)) + +import qualified TensorFlow.BuildOp as TF +import qualified TensorFlow.Core as TF +import qualified TensorFlow.GenOps.Core as TF + +-- TODO: Support other convolution parameters such as stride. + +-- | Convolution padding. +data Padding = + -- | output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides[i]) + PaddingValid + -- | output_spatial_shape[i] = ceil( + -- (input_spatial_shape[i] - + -- (spatial_filter_shape[i]-1) * dilation_rate[i]) / strides[i]) + | PaddingSame + +paddingToByteString :: Padding -> ByteString +paddingToByteString x = case x of + PaddingValid -> "VALID" + PaddingSame -> "SAME" + +-- | Matrix format. +data DataFormat = ChannelLast -- ^ Channel is the last dimension (e.g. NWC, NHWC, NDHWC) + | ChannelFirst -- ^ Channel is the first dimension after N (e.g. NCW, NCHW, NCDHW) + +-- TODO: Address 1D convolution. + +dataFormat2D :: DataFormat -> ByteString +dataFormat2D x = case x of + ChannelLast -> "NHWC" + ChannelFirst -> "NCHW" + +dataFormat3D :: DataFormat -> ByteString +dataFormat3D x = case x of + ChannelLast -> "NDHWC" + ChannelFirst -> "NCDHW" + +-- | 2D Convolution with default parameters. +conv2D :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor TF.Build t -- ^ output +conv2D = conv2D' id PaddingValid ChannelLast + +conv2D' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor TF.Build t -- ^ output +conv2D' params padding dataformat = TF.conv2D' + (params . (TF.opAttr "data_format" .~ dataFormat2D dataformat)) + (paddingToByteString padding) + +-- | 2D convolution backpropagation filter with default parameters. +conv2DBackpropFilter :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 Int32 -- ^ filter_sizes + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv2DBackpropFilter = conv2DBackpropFilter' id PaddingValid ChannelLast + +conv2DBackpropFilter' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 Int32 -- ^ filter_sizes + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv2DBackpropFilter' params padding dataformat = TF.conv2DBackpropFilter' + (params . (TF.opAttr "data_format" .~ dataFormat2D dataformat)) + (paddingToByteString padding) + +-- | 2D convolution backpropagation input with default parameters. +conv2DBackpropInput :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 Int32 -- ^ input_sizes + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv2DBackpropInput = conv2DBackpropInput' id PaddingValid ChannelLast + +conv2DBackpropInput' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 Int32 -- ^ input_sizes + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv2DBackpropInput' params padding dataformat = TF.conv2DBackpropInput' + (params . (TF.opAttr "data_format" .~ dataFormat2D dataformat)) + (paddingToByteString padding) + +-- | 3D Convolution with default parameters. +conv3D :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor TF.Build t -- ^ output +conv3D = conv3D' id PaddingValid ChannelLast + +conv3D' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor TF.Build t -- ^ output +conv3D' params padding dataformat = TF.conv3D' + (params . (TF.opAttr "data_format" .~ dataFormat3D dataformat)) + (paddingToByteString padding) + +-- | 3D convolution backpropagation filter with default parameters. +conv3DBackpropFilter :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropFilter = conv3DBackpropFilter' id PaddingValid ChannelLast + +conv3DBackpropFilter' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropFilter' params padding dataformat = TF.conv3DBackpropFilter' + (params . (TF.opAttr "data_format" .~ dataFormat3D dataformat)) + (paddingToByteString padding) + +-- | 3D convolution backpropagation filter with default parameters. +conv3DBackpropFilterV2 :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 Int32 -- ^ filter_sizes + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropFilterV2 = conv3DBackpropFilterV2' id PaddingValid ChannelLast + +conv3DBackpropFilterV2' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 Int32 -- ^ filter_sizes + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropFilterV2' params padding dataformat = TF.conv3DBackpropFilterV2' + (params . (TF.opAttr "data_format" .~ dataFormat3D dataformat)) + (paddingToByteString padding) + +-- | 3D convolution backpropagation input with default parameters. +conv3DBackpropInput :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropInput = conv3DBackpropInput' id PaddingValid ChannelLast + +conv3DBackpropInput' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropInput' params padding dataformat = TF.conv3DBackpropInput' + (params . (TF.opAttr "data_format" .~ dataFormat3D dataformat)) + (paddingToByteString padding) + +-- | 3D convolution backpropagation input with default parameters. +conv3DBackpropInputV2 :: (TF.OneOf '[Word16, Double, Float] t, TF.OneOf '[Int32, Int64] tshape) + => TF.Tensor v1 tshape -- ^ input_sizes + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropInputV2 = conv3DBackpropInputV2' id PaddingValid ChannelLast + +conv3DBackpropInputV2' :: (TF.OneOf '[Word16, Double, Float] t, TF.OneOf '[Int32, Int64] tshape) + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 tshape -- ^ input_sizes + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +conv3DBackpropInputV2' params padding dataformat = TF.conv3DBackpropInputV2' + (params . (TF.opAttr "data_format" .~ dataFormat3D dataformat)) + (paddingToByteString padding) + +-- | Depth-wise 2D convolution native with default parameters. +depthwiseConv2dNative :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor TF.Build t -- ^ output +depthwiseConv2dNative = depthwiseConv2dNative' id PaddingValid ChannelLast + +depthwiseConv2dNative' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 t -- ^ filter + -> TF.Tensor TF.Build t -- ^ output +depthwiseConv2dNative' params padding dataformat = TF.depthwiseConv2dNative' + (params . (TF.opAttr "data_format" .~ dataFormat2D dataformat)) + (paddingToByteString padding) + +-- | Depth-wise 2D convolution native backpropagation filter with default parameters. +depthwiseConv2dNativeBackpropFilter :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 Int32 -- ^ filter_sizes + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +depthwiseConv2dNativeBackpropFilter = depthwiseConv2dNativeBackpropFilter' id PaddingValid ChannelLast + +depthwiseConv2dNativeBackpropFilter' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 t -- ^ input + -> TF.Tensor v2 Int32 -- ^ filter_sizes + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +depthwiseConv2dNativeBackpropFilter' params padding dataformat = TF.depthwiseConv2dNativeBackpropFilter' + (params . (TF.opAttr "data_format" .~ dataFormat2D dataformat)) + (paddingToByteString padding) + +-- | Depth-wise 2D convolution native backpropagation input with default parameters. +depthwiseConv2dNativeBackpropInput :: TF.OneOf '[Word16, Double, Float] t + => TF.Tensor v1 Int32 -- ^ input_sizes + -> TF.Tensor v2 t -- ^ input + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +depthwiseConv2dNativeBackpropInput = depthwiseConv2dNativeBackpropInput' id PaddingValid ChannelLast + + +depthwiseConv2dNativeBackpropInput' :: TF.OneOf '[Word16, Double, Float] t + => TF.OpParams + -> Padding + -> DataFormat + -> TF.Tensor v1 Int32 -- ^ input_sizes + -> TF.Tensor v2 t -- ^ input + -> TF.Tensor v3 t -- ^ out_backprop + -> TF.Tensor TF.Build t -- ^ output +depthwiseConv2dNativeBackpropInput' params padding dataformat = TF.depthwiseConv2dNativeBackpropInput' + (params . (TF.opAttr "data_format" .~ dataFormat2D dataformat)) + (paddingToByteString padding) diff --git a/tensorflow-ops/src/TensorFlow/Gradient.hs b/tensorflow-ops/src/TensorFlow/Gradient.hs index 7f716e5..0e80737 100644 --- a/tensorflow-ops/src/TensorFlow/Gradient.hs +++ b/tensorflow-ops/src/TensorFlow/Gradient.hs @@ -678,16 +678,14 @@ opGrad "Transpose" _ [_, toT -> p] [dz] = opGrad "Conv2D" nodeDef [toT -> x, toT -> y] [dz] = [ Just $ CoreOps.conv2DBackpropInput' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "use_cudnn_on_gpu" .~ useCudnnOnGpu) . (opAttr "data_format" .~ dataFormat)) - (shape x) y dz + padding (shape x) y dz , Just $ CoreOps.conv2DBackpropFilter' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "use_cudnn_on_gpu" .~ useCudnnOnGpu) . (opAttr "data_format" .~ dataFormat)) - x (shape y) dz + padding x (shape y) dz ] where strides = lookupAttr nodeDef "strides" :: [Int64] @@ -699,16 +697,14 @@ opGrad "Conv2DBackpropInput" nodeDef [_, toT -> x, toT -> y] [dz] = [ Nothing , Just $ CoreOps.conv2DBackpropFilter' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "use_cudnn_on_gpu" .~ useCudnnOnGpu) . (opAttr "data_format" .~ dataFormat)) - dz (shape x) y + padding dz (shape x) y , Just $ CoreOps.conv2D' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "use_cudnn_on_gpu" .~ useCudnnOnGpu) . (opAttr "data_format" .~ dataFormat)) - dz x + padding dz x ] where strides = lookupAttr nodeDef "strides" :: [Int64] @@ -719,14 +715,12 @@ opGrad "Conv2DBackpropInput" nodeDef [_, toT -> x, toT -> y] [dz] = opGrad "DepthwiseConv2dNative" nodeDef [toT -> x, toT -> y] [dz] = [ Just $ CoreOps.depthwiseConv2dNativeBackpropInput' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "data_format" .~ dataFormat)) - (shape x) y dz + padding (shape x) y dz , Just $ CoreOps.depthwiseConv2dNativeBackpropFilter' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "data_format" .~ dataFormat)) - x (shape y) dz + padding x (shape y) dz ] where strides = lookupAttr nodeDef "strides" :: [Int64] @@ -737,14 +731,12 @@ opGrad "DepthwiseConv2dNativeBackpropInput" nodeDef [_, toT -> x, toT -> y] [dz] [ Nothing , Just $ CoreOps.depthwiseConv2dNativeBackpropFilter' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "data_format" .~ dataFormat)) - dz (shape x) y + padding dz (shape x) y , Just $ CoreOps.depthwiseConv2dNative' ((opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "data_format" .~ dataFormat)) - dz x + padding dz x ] where strides = lookupAttr nodeDef "strides" :: [Int64] @@ -755,9 +747,8 @@ opGrad "MaxPool" nodeDef [toT -> x] [dz] = [ Just $ CoreOps.maxPoolGrad' ((opAttr "ksize" .~ ksize) . (opAttr "strides" .~ strides) - . (opAttr "padding" .~ padding) . (opAttr "data_format" .~ dataFormat)) - x output dz + padding x output dz ] where output :: Tensor Build a diff --git a/tensorflow-ops/src/TensorFlow/Minimize.hs b/tensorflow-ops/src/TensorFlow/Minimize.hs index 0062d54..ab12f50 100644 --- a/tensorflow-ops/src/TensorFlow/Minimize.hs +++ b/tensorflow-ops/src/TensorFlow/Minimize.hs @@ -12,6 +12,8 @@ -- See the License for the specific language governing permissions and -- limitations under the License. +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE DataKinds #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} @@ -22,11 +24,15 @@ module TensorFlow.Minimize ( Minimizer , minimizeWith , gradientDescent + , OneOfAdamDataTypes , AdamConfig(..) , adam , adam' ) where +import Data.Complex (Complex) +import Data.Int (Int8,Int16,Int32,Int64) +import Data.Word (Word8,Word16,Word32,Word64) import Control.Monad (zipWithM) import Data.Default (Default(..)) import Data.List (zipWith4) @@ -65,16 +71,19 @@ gradientDescent learningRate params grads = TF.withNameScope "gradientDescent" $ TF.assignAdd param (TF.scalar (-learningRate) `TF.mul` grad) TF.group =<< zipWithM applyGrad params grads --- TODO: Support more than Float in adam. - -data AdamConfig = AdamConfig - { adamLearningRate :: Float - , adamBeta1 :: Float - , adamBeta2 :: Float - , adamEpsilon :: Float +data AdamConfig t = AdamConfig + { adamLearningRate :: t + , adamBeta1 :: t + , adamBeta2 :: t + , adamEpsilon :: t } -instance Default AdamConfig where +type OneOfAdamDataTypes t = + TF.OneOf '[ Complex Double, Complex Float + , Int16, Int32, Int64, Int8, Word16, Word32, Word64, Word8 + , Double, Float] t + +instance Fractional t => Default (AdamConfig t) where -- Recommended defaults from the adam paper. def = AdamConfig 0.001 0.9 0.999 1e-8 @@ -83,10 +92,10 @@ instance Default AdamConfig where -- See https://arxiv.org/abs/1412.6980. -- -- NOTE: Currently requires all 'TF.Variable's to have an 'TF.initializedValue'. -adam :: Minimizer Float +adam :: (OneOfAdamDataTypes t, Fractional t) => Minimizer t adam = adam' def -adam' :: AdamConfig -> Minimizer Float +adam' :: OneOfAdamDataTypes t => AdamConfig t -> Minimizer t adam' config params grads = TF.withNameScope "adam" $ do let lr = TF.scalar (adamLearningRate config) beta1 = TF.scalar (adamBeta1 config) diff --git a/tensorflow-ops/src/TensorFlow/Ops.hs b/tensorflow-ops/src/TensorFlow/Ops.hs index 9fcad6e..a583f6e 100644 --- a/tensorflow-ops/src/TensorFlow/Ops.hs +++ b/tensorflow-ops/src/TensorFlow/Ops.hs @@ -89,6 +89,8 @@ module TensorFlow.Ops , CoreOps.identity' , CoreOps.matMul , CoreOps.matMul' + , CoreOps.einsum + , CoreOps.einsum' , matTranspose , matTranspose' , CoreOps.mean diff --git a/tensorflow-ops/tensorflow-ops.cabal b/tensorflow-ops/tensorflow-ops.cabal index e707ae6..d4a68f3 100644 --- a/tensorflow-ops/tensorflow-ops.cabal +++ b/tensorflow-ops/tensorflow-ops.cabal @@ -16,12 +16,13 @@ library hs-source-dirs: src exposed-modules: TensorFlow.Gradient , TensorFlow.Ops + , TensorFlow.Convolution , TensorFlow.EmbeddingOps , TensorFlow.Minimize , TensorFlow.NN , TensorFlow.Queue , TensorFlow.Variable - build-depends: proto-lens == 0.6.* + build-depends: proto-lens == 0.7.* , base >= 4.7 && < 5 , bytestring , fgl diff --git a/tensorflow-ops/tests/GradientTest.hs b/tensorflow-ops/tests/GradientTest.hs index 20f3b01..931c8ee 100644 --- a/tensorflow-ops/tests/GradientTest.hs +++ b/tensorflow-ops/tests/GradientTest.hs @@ -32,7 +32,8 @@ import Control.Monad(forM_, replicateM, zipWithM) import Control.Monad.IO.Class (liftIO) import qualified TensorFlow.Core as TF -import qualified TensorFlow.GenOps.Core as TF (conv2DBackpropInput', max, maximum, resizeBilinear', tile, pad, batchToSpaceND, spaceToBatchND, squeeze, sqrt, slice, shape, diag, depthwiseConv2dNative', depthwiseConv2dNativeBackpropInput', batchMatMul, batchMatMul', conjugateTranspose) +import qualified TensorFlow.GenOps.Core as TF (max, maximum, resizeBilinear', tile, pad, batchToSpaceND, spaceToBatchND, squeeze, sqrt, slice, shape, diag, batchMatMul, batchMatMul', conjugateTranspose) +import qualified TensorFlow.Convolution as TF import qualified TensorFlow.Gradient as TF import qualified TensorFlow.Ops as TF hiding (zeroInitializedVariable, shape) import qualified TensorFlow.Output as TF @@ -42,7 +43,6 @@ import qualified TensorFlow.Variable as TF import Proto.Tensorflow.Core.Framework.Graph_Fields (node) import Proto.Tensorflow.Core.Framework.NodeDef_Fields (op) -import qualified Data.ByteString.Char8 as BS import TensorFlow.Session (SessionT) testGradientSimple :: Test @@ -715,11 +715,8 @@ testConv2DBackpropInputGrad = testCase "testConv2DBackpropInputGrad" $ do let filterShape = TF.vector [2, 2, 1, 1 :: Int32] -- [fh, fw, inc, out] filter' <- TF.render $ TF.fill filterShape (TF.scalar (1::Float)) let y = TF.conv2DBackpropInput' - ( (TF.opAttr "strides" .~ [1::Int64, 1, 1, 1]) - . (TF.opAttr "padding" .~ (BS.pack "VALID")) - . (TF.opAttr "data_format" .~ (BS.pack "NHWC")) - ) - conv_input_shape filter' x + (TF.opAttr "strides" .~ [1::Int64, 1, 1, 1]) + TF.PaddingValid TF.ChannelLast conv_input_shape filter' x [dx] <- TF.gradients y [x] TF.run (dx, TF.shape dx, TF.shape x) @@ -735,11 +732,8 @@ testDepthwiseConv2dGrad = testCase "testDepthwiseConv2dGrad" $ do let filterShape = TF.vector [2, 2, 1, 1 :: Int32] filter' <- TF.render $ TF.fill filterShape (TF.scalar (1 :: Float)) let y = TF.depthwiseConv2dNative' - ( (TF.opAttr "strides" .~ [1 :: Int64, 1, 1, 1]) - . (TF.opAttr "padding" .~ (BS.pack "VALID")) - . (TF.opAttr "data_format" .~ (BS.pack "NHWC")) - ) - x filter' + (TF.opAttr "strides" .~ [1 :: Int64, 1, 1, 1]) + TF.PaddingValid TF.ChannelLast x filter' [dx] <- TF.gradients y [x] TF.run (dx, TF.shape dx, TF.shape x) @@ -757,11 +751,8 @@ testDepthwiseConv2dBackpropInputGrad = testCase "testDepthwiseConv2dBackpropInpu let filterShape = TF.vector [2, 2, 1, 1 :: Int32] filter' <- TF.render $ TF.fill filterShape (TF.scalar (1 :: Float)) let y = TF.depthwiseConv2dNativeBackpropInput' - ( (TF.opAttr "strides" .~ [1 :: Int64, 1, 1, 1]) - . (TF.opAttr "padding" .~ (BS.pack "VALID")) - . (TF.opAttr "data_format" .~ (BS.pack "NHWC")) - ) - conv_input_shape filter' x + (TF.opAttr "strides" .~ [1 :: Int64, 1, 1, 1]) + TF.PaddingValid TF.ChannelLast conv_input_shape filter' x [dx] <- TF.gradients y [x] TF.run (dx, TF.shape dx, TF.shape x) diff --git a/tensorflow-ops/tests/OpsTest.hs b/tensorflow-ops/tests/OpsTest.hs index 6c08c1a..52b5b04 100644 --- a/tensorflow-ops/tests/OpsTest.hs +++ b/tensorflow-ops/tests/OpsTest.hs @@ -104,6 +104,20 @@ testRereadRef = testCase "testReRunAssign" $ TF.runSession $ do f1 <- TF.run w liftIO $ (0.0, 0.1) @=? (TF.unScalar f0, TF.unScalar f1) +-- | Test Einstein summation. +testEinsum :: Test +testEinsum = testCase "testEinsum" $ TF.runSession $ do + -- Matrix multiply + let matA = TF.constant (TF.Shape [3,3]) [1..9 :: Float] + let matB = TF.constant (TF.Shape [3,1]) [1..3 :: Float] + matMulOut <- TF.run $ TF.matMul matA matB + einsumOut <- TF.run $ TF.einsum "ij,jk->ik" [matA,matB] + liftIO $ (matMulOut :: V.Vector Float) @=? einsumOut + -- Hadamard multiply + hadMulOut <- TF.run $ TF.mul matA matA + einsumHad <- TF.run $ TF.einsum "ij,ij->ij" [matA,matA] + liftIO $ (hadMulOut :: V.Vector Float) @=? einsumHad + main :: IO () main = defaultMain [ testSaveRestore @@ -112,4 +126,5 @@ main = defaultMain , testPlaceholderCse , testScalarFeedCse , testRereadRef + , testEinsum ] diff --git a/tensorflow-proto/tensorflow-proto.cabal b/tensorflow-proto/tensorflow-proto.cabal index 5d453b8..3d74db3 100644 --- a/tensorflow-proto/tensorflow-proto.cabal +++ b/tensorflow-proto/tensorflow-proto.cabal @@ -73,6 +73,8 @@ library , Proto.Tensorflow.Core.Protobuf.ControlFlow_Fields , Proto.Tensorflow.Core.Protobuf.Debug , Proto.Tensorflow.Core.Protobuf.Debug_Fields + , Proto.Tensorflow.Core.Protobuf.DeviceFilters + , Proto.Tensorflow.Core.Protobuf.ErrorCodes , Proto.Tensorflow.Core.Protobuf.MetaGraph , Proto.Tensorflow.Core.Protobuf.MetaGraph_Fields , Proto.Tensorflow.Core.Protobuf.NamedTensor @@ -83,12 +85,16 @@ library , Proto.Tensorflow.Core.Protobuf.RewriterConfig_Fields , Proto.Tensorflow.Core.Protobuf.SavedModel , Proto.Tensorflow.Core.Protobuf.SavedModel_Fields + , Proto.Tensorflow.Core.Protobuf.SavedObjectGraph , Proto.Tensorflow.Core.Protobuf.Saver , Proto.Tensorflow.Core.Protobuf.Saver_Fields + , Proto.Tensorflow.Core.Protobuf.Struct , Proto.Tensorflow.Core.Protobuf.TensorBundle , Proto.Tensorflow.Core.Protobuf.TensorBundle_Fields , Proto.Tensorflow.Core.Protobuf.TensorflowServer , Proto.Tensorflow.Core.Protobuf.TensorflowServer_Fields + , Proto.Tensorflow.Core.Protobuf.TrackableObjectGraph + , Proto.Tensorflow.Core.Protobuf.VerifierConfig , Proto.Tensorflow.Core.Util.Event , Proto.Tensorflow.Core.Util.Event_Fields , Proto.Tensorflow.Core.Util.MemmappedFileSystem @@ -125,6 +131,7 @@ library , Proto.Tensorflow.Core.Framework.OpDef_Fields , Proto.Tensorflow.Core.Framework.ResourceHandle , Proto.Tensorflow.Core.Framework.ResourceHandle_Fields + , Proto.Tensorflow.Core.Protobuf.SavedObjectGraph , Proto.Tensorflow.Core.Framework.StepStats , Proto.Tensorflow.Core.Framework.StepStats_Fields , Proto.Tensorflow.Core.Framework.Summary @@ -152,6 +159,8 @@ library , Proto.Tensorflow.Core.Protobuf.ControlFlow_Fields , Proto.Tensorflow.Core.Protobuf.Debug , Proto.Tensorflow.Core.Protobuf.Debug_Fields + , Proto.Tensorflow.Core.Protobuf.DeviceFilters + , Proto.Tensorflow.Core.Protobuf.ErrorCodes , Proto.Tensorflow.Core.Protobuf.MetaGraph , Proto.Tensorflow.Core.Protobuf.MetaGraph_Fields , Proto.Tensorflow.Core.Protobuf.NamedTensor @@ -162,12 +171,16 @@ library , Proto.Tensorflow.Core.Protobuf.RewriterConfig_Fields , Proto.Tensorflow.Core.Protobuf.SavedModel , Proto.Tensorflow.Core.Protobuf.SavedModel_Fields + , Proto.Tensorflow.Core.Protobuf.SavedObjectGraph , Proto.Tensorflow.Core.Protobuf.Saver , Proto.Tensorflow.Core.Protobuf.Saver_Fields + , Proto.Tensorflow.Core.Protobuf.Struct , Proto.Tensorflow.Core.Protobuf.TensorBundle , Proto.Tensorflow.Core.Protobuf.TensorBundle_Fields , Proto.Tensorflow.Core.Protobuf.TensorflowServer , Proto.Tensorflow.Core.Protobuf.TensorflowServer_Fields + , Proto.Tensorflow.Core.Protobuf.TrackableObjectGraph + , Proto.Tensorflow.Core.Protobuf.VerifierConfig , Proto.Tensorflow.Core.Util.Event , Proto.Tensorflow.Core.Util.Event_Fields , Proto.Tensorflow.Core.Util.MemmappedFileSystem @@ -176,9 +189,9 @@ library , Proto.Tensorflow.Core.Util.SavedTensorSlice_Fields , Proto.Tensorflow.Core.Util.TestLog , Proto.Tensorflow.Core.Util.TestLog_Fields - build-depends: proto-lens == 0.6.* - , proto-lens-runtime == 0.6.* - , proto-lens-protobuf-types == 0.6.* + build-depends: proto-lens == 0.7.* + , proto-lens-runtime == 0.7.* + , proto-lens-protobuf-types == 0.7.* , base >= 4.7 && < 5 default-language: Haskell2010 include-dirs: . diff --git a/tensorflow/tensorflow.cabal b/tensorflow/tensorflow.cabal index 0c81268..3f99648 100644 --- a/tensorflow/tensorflow.cabal +++ b/tensorflow/tensorflow.cabal @@ -36,7 +36,7 @@ library , TensorFlow.Types other-modules: TensorFlow.Internal.Raw build-tools: c2hs - build-depends: proto-lens == 0.6.* + build-depends: proto-lens == 0.7.* , tensorflow-proto == 0.2.* , base >= 4.7 && < 5 , async diff --git a/third_party/tensorflow b/third_party/tensorflow index 6612da8..b36436b 160000 --- a/third_party/tensorflow +++ b/third_party/tensorflow @@ -1 +1 @@ -Subproject commit 6612da89516247503f03ef76e974b51a434fb52e +Subproject commit b36436b087bd8e8701ef51718179037cccdfc26e