<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><link rel="stylesheet" type="text/css" href="style.css" /><script type="text/javascript" src="highlight.js"></script></head><body><pre><span class="hs-comment">-- Copyright 2016 TensorFlow authors.</span><span>
</span><a name="line-2"></a><span class="hs-comment">--</span><span>
</span><a name="line-3"></a><span class="hs-comment">-- Licensed under the Apache License, Version 2.0 (the "License");</span><span>
</span><a name="line-4"></a><span class="hs-comment">-- you may not use this file except in compliance with the License.</span><span>
</span><a name="line-5"></a><span class="hs-comment">-- You may obtain a copy of the License at</span><span>
</span><a name="line-6"></a><span class="hs-comment">--</span><span>
</span><a name="line-7"></a><span class="hs-comment">-- http://www.apache.org/licenses/LICENSE-2.0</span><span>
</span><a name="line-8"></a><span class="hs-comment">--</span><span>
</span><a name="line-9"></a><span class="hs-comment">-- Unless required by applicable law or agreed to in writing, software</span><span>
</span><a name="line-10"></a><span class="hs-comment">-- distributed under the License is distributed on an "AS IS" BASIS,</span><span>
</span><a name="line-11"></a><span class="hs-comment">-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span><span>
</span><a name="line-12"></a><span class="hs-comment">-- See the License for the specific language governing permissions and</span><span>
</span><a name="line-13"></a><span class="hs-comment">-- limitations under the License.</span><span>
</span><a name="line-14"></a><span>
</span><a name="line-15"></a><span class="hs-pragma">{-# LANGUAGE FlexibleContexts #-}</span><span>
</span><a name="line-16"></a><span class="hs-pragma">{-# LANGUAGE OverloadedStrings #-}</span><span>
</span><a name="line-17"></a><span class="hs-pragma">{-# LANGUAGE RankNTypes #-}</span><span>
</span><a name="line-18"></a><span class="hs-pragma">{-# LANGUAGE ScopedTypeVariables #-}</span><span>
</span><a name="line-19"></a><span class="hs-pragma">{-# LANGUAGE TypeApplications #-}</span><span>
</span><a name="line-20"></a><span>
</span><a name="line-21"></a><span class="hs-keyword">module</span><span> </span><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">Minimize</span><span>
</span><a name="line-22"></a><span> </span><span class="hs-special">(</span><span> </span><a href="TensorFlow.Minimize.html#Minimizer"><span class="hs-identifier hs-type">Minimizer</span></a><span>
</span><a name="line-23"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Minimize.html#minimizeWith"><span class="hs-identifier hs-var">minimizeWith</span></a><span>
</span><a name="line-24"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Minimize.html#gradientDescent"><span class="hs-identifier hs-var">gradientDescent</span></a><span>
</span><a name="line-25"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Minimize.html#AdamConfig"><span class="hs-identifier hs-type">AdamConfig</span></a><span class="hs-special">(</span><span class="hs-glyph">..</span><span class="hs-special">)</span><span>
</span><a name="line-26"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Minimize.html#adam"><span class="hs-identifier hs-var">adam</span></a><span>
</span><a name="line-27"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Minimize.html#adam%27"><span class="hs-identifier hs-var">adam'</span></a><span>
</span><a name="line-28"></a><span> </span><span class="hs-special">)</span><span> </span><span class="hs-keyword">where</span><span>
</span><a name="line-29"></a><span>
</span><a name="line-30"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Control</span><span class="hs-operator">.</span><span class="hs-identifier">Monad</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">zipWithM</span><span class="hs-special">)</span><span>
</span><a name="line-31"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">Default</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-type">Default</span><span class="hs-special">(</span><span class="hs-glyph">..</span><span class="hs-special">)</span><span class="hs-special">)</span><span>
</span><a name="line-32"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">List</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">zipWith4</span><span class="hs-special">)</span><span>
</span><a name="line-33"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">Maybe</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">fromMaybe</span><span class="hs-special">)</span><span>
</span><a name="line-34"></a><span>
</span><a name="line-35"></a><span class="hs-keyword">import</span><span> </span><span class="hs-keyword">qualified</span><span> </span><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">Core</span><span> </span><span class="hs-keyword">as</span><span> </span><span class="hs-identifier">TF</span><span>
</span><a name="line-36"></a><span class="hs-keyword">import</span><span> </span><span class="hs-keyword">qualified</span><span> </span><a href="TensorFlow.Gradient.html"><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">Gradient</span></a><span> </span><span class="hs-keyword">as</span><span> </span><span class="hs-identifier">TF</span><span>
</span><a name="line-37"></a><span class="hs-keyword">import</span><span> </span><span class="hs-keyword">qualified</span><span> </span><a href="TensorFlow.Ops.html"><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">Ops</span></a><span> </span><span class="hs-keyword">as</span><span> </span><span class="hs-identifier">TF</span><span> </span><span class="hs-keyword">hiding</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">assign</span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Ops.html#initializedVariable"><span class="hs-identifier hs-var">initializedVariable</span></a><span class="hs-special">)</span><span>
</span><a name="line-38"></a><span class="hs-keyword">import</span><span> </span><span class="hs-keyword">qualified</span><span> </span><a href="TensorFlow.Variable.html"><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">Variable</span></a><span> </span><span class="hs-keyword">as</span><span> </span><span class="hs-identifier">TF</span><span>
</span><a name="line-39"></a><span>
</span><a name="line-40"></a><span class="hs-comment">-- | Functions that minimize a loss w.r.t. a set of 'TF.Variable's.</span><span>
</span><a name="line-41"></a><span class="hs-comment">--</span><span>
</span><a name="line-42"></a><span class="hs-comment">-- Generally only performs one step of an iterative algorithm.</span><span>
</span><a name="line-43"></a><span class="hs-comment">--</span><span>
</span><a name="line-44"></a><span class="hs-comment">-- 'Minimizer's are defined as a function of the gradients instead of</span><span>
</span><a name="line-45"></a><span class="hs-comment">-- the loss so that users can apply transformations to the gradients.</span><span>
</span><a name="line-46"></a><span class="hs-keyword">type</span><span> </span><a name="Minimizer"><a href="TensorFlow.Minimize.html#Minimizer"><span class="hs-identifier">Minimizer</span></a></a><span> </span><a name="local-6989586621679101337"><a href="#local-6989586621679101337"><span class="hs-identifier">a</span></a></a><span> </span><span class="hs-glyph">=</span><span>
</span><a name="line-47"></a><span> </span><span class="hs-keyword">forall</span><span> </span><a name="local-6989586621679101338"><a href="#local-6989586621679101338"><span class="hs-identifier">m</span></a></a><span class="hs-operator">.</span><span> </span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">MonadBuild</span><span> </span><a href="#local-6989586621679101338"><span class="hs-identifier hs-type">m</span></a><span> </span><span class="hs-glyph">=></span><span>
</span><a name="line-48"></a><span> </span><span class="hs-special">[</span><a href="TensorFlow.Variable.html#Variable"><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">Variable</span></a><span> </span><a href="#local-6989586621679101337"><span class="hs-identifier hs-type">a</span></a><span class="hs-special">]</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-special">[</span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">Tensor</span><span> </span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">Value</span><span> </span><a href="#local-6989586621679101337"><span class="hs-identifier hs-type">a</span></a><span class="hs-special">]</span><span> </span><span class="hs-glyph">-></span><span> </span><a href="#local-6989586621679101338"><span class="hs-identifier hs-type">m</span></a><span> </span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ControlNode</span><span>
</span><a name="line-49"></a><span>
</span><a name="line-50"></a><span class="hs-comment">-- | Convenience wrapper around 'TF.gradients' and a 'Minimizer'.</span><span>
</span><a name="line-51"></a><span class="hs-identifier">minimizeWith</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">MonadBuild</span><span> </span><a href="#local-6989586621679101340"><span class="hs-identifier hs-type">m</span></a><span class="hs-special">,</span><span> </span><a href="TensorFlow.Gradient.html#GradientCompatible"><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">GradientCompatible</span></a><span> </span><a href="#local-6989586621679101341"><span class="hs-identifier hs-type">a</span></a><span class="hs-special">)</span><span>
</span><a name="line-52"></a><span> </span><span class="hs-glyph">=></span><span> </span><a href="TensorFlow.Minimize.html#Minimizer"><span class="hs-identifier hs-type">Minimizer</span></a><span> </span><a href="#local-6989586621679101341"><span class="hs-identifier hs-type">a</span></a><span>
</span><a name="line-53"></a><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">Tensor</span><span> </span><a href="#local-6989586621679101342"><span class="hs-identifier hs-type">v</span></a><span> </span><a href="#local-6989586621679101341"><span class="hs-identifier hs-type">a</span></a><span> </span><span class="hs-comment">-- ^ Loss.</span><span>
</span><a name="line-54"></a><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-special">[</span><a href="TensorFlow.Variable.html#Variable"><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">Variable</span></a><span> </span><a href="#local-6989586621679101341"><span class="hs-identifier hs-type">a</span></a><span class="hs-special">]</span><span> </span><span class="hs-comment">-- ^ Parameters of the loss function.</span><span>
</span><a name="line-55"></a><span> </span><span class="hs-glyph">-></span><span> </span><a href="#local-6989586621679101340"><span class="hs-identifier hs-type">m</span></a><span> </span><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ControlNode</span><span>
</span><a name="line-56"></a><a name="minimizeWith"><a href="TensorFlow.Minimize.html#minimizeWith"><span class="hs-identifier">minimizeWith</span></a></a><span> </span><a name="local-6989586621679101343"><a href="#local-6989586621679101343"><span class="hs-identifier">minimizer</span></a></a><span> </span><a name="local-6989586621679101344"><a href="#local-6989586621679101344"><span class="hs-identifier">loss</span></a></a><span> </span><a name="local-6989586621679101345"><a href="#local-6989586621679101345"><span class="hs-identifier">params</span></a></a><span> </span><span class="hs-glyph">=</span><span>
</span><a name="line-57"></a><span> </span><a href="TensorFlow.Gradient.html#gradients"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">gradients</span></a><span> </span><a href="#local-6989586621679101344"><span class="hs-identifier hs-var">loss</span></a><span> </span><a href="#local-6989586621679101345"><span class="hs-identifier hs-var">params</span></a><span> </span><span class="hs-operator hs-var">>>=</span><span> </span><a href="#local-6989586621679101343"><span class="hs-identifier hs-var">minimizer</span></a><span> </span><a href="#local-6989586621679101345"><span class="hs-identifier hs-var">params</span></a><span>
</span><a name="line-58"></a><span>
</span><a name="line-59"></a><span class="hs-comment">-- | Perform one step of the gradient descent algorithm.</span><span>
</span><a name="line-60"></a><span class="hs-identifier">gradientDescent</span><span> </span><span class="hs-glyph">::</span><span> </span><a href="TensorFlow.Gradient.html#GradientCompatible"><span class="hs-identifier hs-type">TF</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">GradientCompatible</span></a><span> </span><a href="#local-6989586621679101339"><span class="hs-identifier hs-type">a</span></a><span>
</span><a name="line-61"></a><span> </span><span class="hs-glyph">=></span><span> </span><a href="#local-6989586621679101339"><span class="hs-identifier hs-type">a</span></a><span> </span><span class="hs-comment">-- ^ Learning rate.</span><span>
</span><a name="line-62"></a><span> </span><span class="hs-glyph">-></span><span> </span><a href="TensorFlow.Minimize.html#Minimizer"><span class="hs-identifier hs-type">Minimizer</span></a><span> </span><a href="#local-6989586621679101339"><span class="hs-identifier hs-type">a</span></a><span>
</span><a name="line-63"></a><a name="gradientDescent"><a href="TensorFlow.Minimize.html#gradientDescent"><span class="hs-identifier">gradientDescent</span></a></a><span> </span><a name="local-6989586621679101346"><a href="#local-6989586621679101346"><span class="hs-identifier">learningRate</span></a></a><span> </span><a name="local-6989586621679101347"><a href="#local-6989586621679101347"><span class="hs-identifier">params</span></a></a><span> </span><a name="local-6989586621679101348"><a href="#local-6989586621679101348"><span class="hs-identifier">grads</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">withNameScope</span><span> </span><span class="hs-string">"gradientDescent"</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-keyword">do</span><span>
</span><a name="line-64"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679101349"><a href="#local-6989586621679101349"><span class="hs-identifier">applyGrad</span></a></a><span> </span><a name="local-6989586621679101350"><a href="#local-6989586621679101350"><span class="hs-identifier">param</span></a></a><span> </span><a name="local-6989586621679101351"><a href="#local-6989586621679101351"><span class="hs-identifier">grad</span></a></a><span> </span><span class="hs-glyph">=</span><span>
</span><a name="line-65"></a><span> </span><a href="TensorFlow.Variable.html#assignAdd"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">assignAdd</span></a><span> </span><a href="#local-6989586621679101350"><span class="hs-identifier hs-var">param</span></a><span> </span><span class="hs-special">(</span><a href="TensorFlow.Ops.html#scalar"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">scalar</span></a><span> </span><span class="hs-special">(</span><span class="hs-glyph">-</span><a href="#local-6989586621679101346"><span class="hs-identifier hs-var">learningRate</span></a><span class="hs-special">)</span><span> </span><span class="hs-special">`</span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">mul</span><span class="hs-special">`</span><span> </span><a href="#local-6989586621679101351"><span class="hs-identifier hs-var">grad</span></a><span class="hs-special">)</span><span>
</span><a name="line-66"></a><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">group</span><span> </span><span class="hs-operator hs-var">=<<</span><span> </span><span class="hs-identifier hs-var">zipWithM</span><span> </span><a href="#local-6989586621679101349"><span class="hs-identifier hs-var">applyGrad</span></a><span> </span><a href="#local-6989586621679101347"><span class="hs-identifier hs-var">params</span></a><span> </span><a href="#local-6989586621679101348"><span class="hs-identifier hs-var">grads</span></a><span>
</span><a name="line-67"></a><span>
</span><a name="line-68"></a><span class="hs-comment">-- TODO: Support more than Float in adam.</span><span>
</span><a name="line-69"></a><span>
</span><a name="line-70"></a><span class="hs-keyword">data</span><span> </span><a name="AdamConfig"><a href="TensorFlow.Minimize.html#AdamConfig"><span class="hs-identifier">AdamConfig</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a name="AdamConfig"><a href="TensorFlow.Minimize.html#AdamConfig"><span class="hs-identifier">AdamConfig</span></a></a><span>
</span><a name="line-71"></a><span> </span><span class="hs-special">{</span><span> </span><a name="adamLearningRate"><a href="TensorFlow.Minimize.html#adamLearningRate"><span class="hs-identifier">adamLearningRate</span></a></a><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Float</span><span>
</span><a name="line-72"></a><span> </span><span class="hs-special">,</span><span> </span><a name="adamBeta1"><a href="TensorFlow.Minimize.html#adamBeta1"><span class="hs-identifier">adamBeta1</span></a></a><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Float</span><span>
</span><a name="line-73"></a><span> </span><span class="hs-special">,</span><span> </span><a name="adamBeta2"><a href="TensorFlow.Minimize.html#adamBeta2"><span class="hs-identifier">adamBeta2</span></a></a><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Float</span><span>
</span><a name="line-74"></a><span> </span><span class="hs-special">,</span><span> </span><a name="adamEpsilon"><a href="TensorFlow.Minimize.html#adamEpsilon"><span class="hs-identifier">adamEpsilon</span></a></a><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Float</span><span>
</span><a name="line-75"></a><span> </span><span class="hs-special">}</span><span>
</span><a name="line-76"></a><span>
</span><a name="line-77"></a><span class="hs-keyword">instance</span><span> </span><span class="hs-identifier hs-type">Default</span><span> </span><a href="TensorFlow.Minimize.html#AdamConfig"><span class="hs-identifier hs-type">AdamConfig</span></a><span> </span><span class="hs-keyword">where</span><span>
</span><a name="line-78"></a><span> </span><span class="hs-comment">-- Recommended defaults from the adam paper.</span><span>
</span><a name="line-79"></a><span> </span><a name="local-8214565720323802837"><span class="hs-identifier">def</span></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Minimize.html#AdamConfig"><span class="hs-identifier hs-var">AdamConfig</span></a><span> </span><span class="hs-number">0.001</span><span> </span><span class="hs-number">0.9</span><span> </span><span class="hs-number">0.999</span><span> </span><span class="hs-number">1e-8</span><span>
</span><a name="line-80"></a><span>
</span><a name="line-81"></a><span class="hs-comment">-- | Perform one step of the adam algorithm.</span><span>
</span><a name="line-82"></a><span class="hs-comment">--</span><span>
</span><a name="line-83"></a><span class="hs-comment">-- See https://arxiv.org/abs/1412.6980.</span><span>
</span><a name="line-84"></a><span class="hs-comment">--</span><span>
</span><a name="line-85"></a><span class="hs-comment">-- NOTE: Currently requires all 'TF.Variable's to have an 'TF.initializedValue'.</span><span>
</span><a name="line-86"></a><span class="hs-identifier">adam</span><span> </span><span class="hs-glyph">::</span><span> </span><a href="TensorFlow.Minimize.html#Minimizer"><span class="hs-identifier hs-type">Minimizer</span></a><span> </span><span class="hs-identifier hs-type">Float</span><span>
</span><a name="line-87"></a><a name="adam"><a href="TensorFlow.Minimize.html#adam"><span class="hs-identifier">adam</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Minimize.html#adam%27"><span class="hs-identifier hs-var">adam'</span></a><span> </span><span class="hs-identifier hs-var">def</span><span>
</span><a name="line-88"></a><span>
</span><a name="line-89"></a><span class="hs-identifier">adam'</span><span> </span><span class="hs-glyph">::</span><span> </span><a href="TensorFlow.Minimize.html#AdamConfig"><span class="hs-identifier hs-type">AdamConfig</span></a><span> </span><span class="hs-glyph">-></span><span> </span><a href="TensorFlow.Minimize.html#Minimizer"><span class="hs-identifier hs-type">Minimizer</span></a><span> </span><span class="hs-identifier hs-type">Float</span><span>
</span><a name="line-90"></a><a name="adam%27"><a href="TensorFlow.Minimize.html#adam%27"><span class="hs-identifier">adam'</span></a></a><span> </span><a name="local-6989586621679101352"><a href="#local-6989586621679101352"><span class="hs-identifier">config</span></a></a><span> </span><a name="local-6989586621679101353"><a href="#local-6989586621679101353"><span class="hs-identifier">params</span></a></a><span> </span><a name="local-6989586621679101354"><a href="#local-6989586621679101354"><span class="hs-identifier">grads</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">withNameScope</span><span> </span><span class="hs-string">"adam"</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-keyword">do</span><span>
</span><a name="line-91"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679101355"><a href="#local-6989586621679101355"><span class="hs-identifier">lr</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Ops.html#scalar"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">scalar</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier">adamLearningRate</span><span> </span><a href="#local-6989586621679101352"><span class="hs-identifier hs-var">config</span></a><span class="hs-special">)</span><span>
</span><a name="line-92"></a><span> </span><a name="local-6989586621679101356"><a href="#local-6989586621679101356"><span class="hs-identifier">beta1</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Ops.html#scalar"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">scalar</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier">adamBeta1</span><span> </span><a href="#local-6989586621679101352"><span class="hs-identifier hs-var">config</span></a><span class="hs-special">)</span><span>
</span><a name="line-93"></a><span> </span><a name="local-6989586621679101357"><a href="#local-6989586621679101357"><span class="hs-identifier">beta2</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Ops.html#scalar"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">scalar</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier">adamBeta2</span><span> </span><a href="#local-6989586621679101352"><span class="hs-identifier hs-var">config</span></a><span class="hs-special">)</span><span>
</span><a name="line-94"></a><span> </span><a name="local-6989586621679101358"><a href="#local-6989586621679101358"><span class="hs-identifier">epsilon</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Ops.html#scalar"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">scalar</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier">adamEpsilon</span><span> </span><a href="#local-6989586621679101352"><span class="hs-identifier hs-var">config</span></a><span class="hs-special">)</span><span>
</span><a name="line-95"></a><span> </span><span class="hs-comment">-- Create adam state variables.</span><span>
</span><a name="line-96"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679101359"><a href="#local-6989586621679101359"><span class="hs-identifier">errorMsg</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-string">"TensorFlow.Minimize.adam requires an initial value for all variables"</span><span>
</span><a name="line-97"></a><span> </span><a name="local-6989586621679101360"><a href="#local-6989586621679101360"><span class="hs-identifier">initVal</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">fromMaybe</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">error</span><span> </span><a href="#local-6989586621679101359"><span class="hs-identifier hs-var">errorMsg</span></a><span class="hs-special">)</span><span> </span><span class="hs-operator hs-var">.</span><span> </span><span class="hs-identifier">TF</span><span class="hs-operator">.</span><span class="hs-identifier">initializedValue</span><span>
</span><a name="line-98"></a><span> </span><a name="local-6989586621679101361"><a href="#local-6989586621679101361"><span class="hs-identifier">ms</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">mapM</span><span> </span><span class="hs-special">(</span><a href="TensorFlow.Variable.html#initializedVariable"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">initializedVariable</span></a><span> </span><span class="hs-operator hs-var">.</span><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">zerosLike</span><span> </span><span class="hs-operator hs-var">.</span><span> </span><a href="#local-6989586621679101360"><span class="hs-identifier hs-var">initVal</span></a><span class="hs-special">)</span><span> </span><a href="#local-6989586621679101353"><span class="hs-identifier hs-var">params</span></a><span>
</span><a name="line-99"></a><span> </span><a name="local-6989586621679101362"><a href="#local-6989586621679101362"><span class="hs-identifier">vs</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">mapM</span><span> </span><span class="hs-special">(</span><a href="TensorFlow.Variable.html#initializedVariable"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">initializedVariable</span></a><span> </span><span class="hs-operator hs-var">.</span><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">zerosLike</span><span> </span><span class="hs-operator hs-var">.</span><span> </span><a href="#local-6989586621679101360"><span class="hs-identifier hs-var">initVal</span></a><span class="hs-special">)</span><span> </span><a href="#local-6989586621679101353"><span class="hs-identifier hs-var">params</span></a><span>
</span><a name="line-100"></a><span> </span><a name="local-6989586621679101363"><a href="#local-6989586621679101363"><span class="hs-identifier">beta1Power</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><a href="TensorFlow.Variable.html#initializedVariable"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">initializedVariable</span></a><span> </span><a href="#local-6989586621679101356"><span class="hs-identifier hs-var">beta1</span></a><span>
</span><a name="line-101"></a><span> </span><a name="local-6989586621679101364"><a href="#local-6989586621679101364"><span class="hs-identifier">beta2Power</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><a href="TensorFlow.Variable.html#initializedVariable"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">initializedVariable</span></a><span> </span><a href="#local-6989586621679101357"><span class="hs-identifier hs-var">beta2</span></a><span>
</span><a name="line-102"></a><span> </span><span class="hs-comment">-- Perform adam update.</span><span>
</span><a name="line-103"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679101365"><a href="#local-6989586621679101365"><span class="hs-identifier">applyGrad</span></a></a><span> </span><a name="local-6989586621679101366"><a href="#local-6989586621679101366"><span class="hs-identifier">param</span></a></a><span> </span><a name="local-6989586621679101367"><a href="#local-6989586621679101367"><span class="hs-identifier">m</span></a></a><span> </span><a name="local-6989586621679101368"><a href="#local-6989586621679101368"><span class="hs-identifier">v</span></a></a><span> </span><span class="hs-glyph">=</span><span>
</span><a name="line-104"></a><span> </span><a href="TensorFlow.Variable.html#resourceApplyAdam"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">resourceApplyAdam</span></a><span> </span><a href="#local-6989586621679101366"><span class="hs-identifier hs-var">param</span></a><span> </span><a href="#local-6989586621679101367"><span class="hs-identifier hs-var">m</span></a><span> </span><a href="#local-6989586621679101368"><span class="hs-identifier hs-var">v</span></a><span>
</span><a name="line-105"></a><span> </span><span class="hs-special">(</span><a href="TensorFlow.Variable.html#readValue"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">readValue</span></a><span> </span><a href="#local-6989586621679101363"><span class="hs-identifier hs-var">beta1Power</span></a><span class="hs-special">)</span><span>
</span><a name="line-106"></a><span> </span><span class="hs-special">(</span><a href="TensorFlow.Variable.html#readValue"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">readValue</span></a><span> </span><a href="#local-6989586621679101364"><span class="hs-identifier hs-var">beta2Power</span></a><span class="hs-special">)</span><span>
</span><a name="line-107"></a><span> </span><a href="#local-6989586621679101355"><span class="hs-identifier hs-var">lr</span></a><span> </span><a href="#local-6989586621679101356"><span class="hs-identifier hs-var">beta1</span></a><span> </span><a href="#local-6989586621679101357"><span class="hs-identifier hs-var">beta2</span></a><span> </span><a href="#local-6989586621679101358"><span class="hs-identifier hs-var">epsilon</span></a><span>
</span><a name="line-108"></a><span> </span><a name="local-6989586621679101369"><a href="#local-6989586621679101369"><span class="hs-identifier">updateVars</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">sequence</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-identifier hs-var">zipWith4</span><span> </span><a href="#local-6989586621679101365"><span class="hs-identifier hs-var">applyGrad</span></a><span> </span><a href="#local-6989586621679101353"><span class="hs-identifier hs-var">params</span></a><span> </span><a href="#local-6989586621679101361"><span class="hs-identifier hs-var">ms</span></a><span> </span><a href="#local-6989586621679101362"><span class="hs-identifier hs-var">vs</span></a><span> </span><a href="#local-6989586621679101354"><span class="hs-identifier hs-var">grads</span></a><span>
</span><a name="line-109"></a><span> </span><span class="hs-comment">-- Update beta variables after adam update.</span><span>
</span><a name="line-110"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679101370"><a href="#local-6989586621679101370"><span class="hs-identifier">updateBeta</span></a></a><span> </span><a name="local-6989586621679101371"><a href="#local-6989586621679101371"><span class="hs-identifier">betaPower</span></a></a><span> </span><a name="local-6989586621679101372"><a href="#local-6989586621679101372"><span class="hs-identifier">beta</span></a></a><span> </span><span class="hs-glyph">=</span><span>
</span><a name="line-111"></a><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">withControlDependencies</span><span> </span><a href="#local-6989586621679101369"><span class="hs-identifier hs-var">updateVars</span></a><span>
</span><a name="line-112"></a><span> </span><span class="hs-special">(</span><a href="TensorFlow.Variable.html#assign"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">assign</span></a><span> </span><a href="#local-6989586621679101371"><span class="hs-identifier hs-var">betaPower</span></a><span> </span><span class="hs-special">(</span><a href="TensorFlow.Variable.html#readValue"><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">readValue</span></a><span> </span><a href="#local-6989586621679101371"><span class="hs-identifier hs-var">betaPower</span></a><span> </span><span class="hs-special">`</span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">mul</span><span class="hs-special">`</span><span> </span><a href="#local-6989586621679101372"><span class="hs-identifier hs-var">beta</span></a><span class="hs-special">)</span><span class="hs-special">)</span><span>
</span><a name="line-113"></a><span> </span><a name="local-6989586621679101373"><a href="#local-6989586621679101373"><span class="hs-identifier">updateBeta1</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><a href="#local-6989586621679101370"><span class="hs-identifier hs-var">updateBeta</span></a><span> </span><a href="#local-6989586621679101363"><span class="hs-identifier hs-var">beta1Power</span></a><span> </span><a href="#local-6989586621679101356"><span class="hs-identifier hs-var">beta1</span></a><span>
</span><a name="line-114"></a><span> </span><a name="local-6989586621679101374"><a href="#local-6989586621679101374"><span class="hs-identifier">updateBeta2</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><a href="#local-6989586621679101370"><span class="hs-identifier hs-var">updateBeta</span></a><span> </span><a href="#local-6989586621679101364"><span class="hs-identifier hs-var">beta2Power</span></a><span> </span><a href="#local-6989586621679101357"><span class="hs-identifier hs-var">beta2</span></a><span>
</span><a name="line-115"></a><span> </span><span class="hs-identifier hs-var">TF</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">group</span><span> </span><span class="hs-special">(</span><a href="#local-6989586621679101373"><span class="hs-identifier hs-var">updateBeta1</span></a><span class="hs-glyph">:</span><a href="#local-6989586621679101374"><span class="hs-identifier hs-var">updateBeta2</span></a><span class="hs-glyph">:</span><a href="#local-6989586621679101369"><span class="hs-identifier hs-var">updateVars</span></a><span class="hs-special">)</span><span>
</span><a name="line-116"></a></pre></body></html>