mirror of
https://github.com/tensorflow/haskell.git
synced 2024-06-30 16:38:34 +02:00
135 lines
36 KiB
HTML
135 lines
36 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><link rel="stylesheet" type="text/css" href="style.css" /><script type="text/javascript" src="highlight.js"></script></head><body><pre><span class="hs-comment">-- Copyright 2016 TensorFlow authors.</span><span>
|
|
</span><a name="line-2"></a><span class="hs-comment">--</span><span>
|
|
</span><a name="line-3"></a><span class="hs-comment">-- Licensed under the Apache License, Version 2.0 (the "License");</span><span>
|
|
</span><a name="line-4"></a><span class="hs-comment">-- you may not use this file except in compliance with the License.</span><span>
|
|
</span><a name="line-5"></a><span class="hs-comment">-- You may obtain a copy of the License at</span><span>
|
|
</span><a name="line-6"></a><span class="hs-comment">--</span><span>
|
|
</span><a name="line-7"></a><span class="hs-comment">-- http://www.apache.org/licenses/LICENSE-2.0</span><span>
|
|
</span><a name="line-8"></a><span class="hs-comment">--</span><span>
|
|
</span><a name="line-9"></a><span class="hs-comment">-- Unless required by applicable law or agreed to in writing, software</span><span>
|
|
</span><a name="line-10"></a><span class="hs-comment">-- distributed under the License is distributed on an "AS IS" BASIS,</span><span>
|
|
</span><a name="line-11"></a><span class="hs-comment">-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span><span>
|
|
</span><a name="line-12"></a><span class="hs-comment">-- See the License for the specific language governing permissions and</span><span>
|
|
</span><a name="line-13"></a><span class="hs-comment">-- limitations under the License.</span><span>
|
|
</span><a name="line-14"></a><span>
|
|
</span><a name="line-15"></a><span class="hs-comment">-- | Encoder and decoder for the TensorFlow \"TFRecords\" format.</span><span>
|
|
</span><a name="line-16"></a><span>
|
|
</span><a name="line-17"></a><span class="hs-pragma">{-# LANGUAGE Rank2Types #-}</span><span>
|
|
</span><a name="line-18"></a><span class="hs-keyword">module</span><span> </span><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">Records</span><span>
|
|
</span><a name="line-19"></a><span> </span><span class="hs-special">(</span><span>
|
|
</span><a name="line-20"></a><span> </span><span class="hs-comment">-- * Records</span><span>
|
|
</span><a name="line-21"></a><span> </span><a href="TensorFlow.Records.html#putTFRecord"><span class="hs-identifier hs-var">putTFRecord</span></a><span>
|
|
</span><a name="line-22"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Records.html#getTFRecord"><span class="hs-identifier hs-var">getTFRecord</span></a><span>
|
|
</span><a name="line-23"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Records.html#getTFRecords"><span class="hs-identifier hs-var">getTFRecords</span></a><span>
|
|
</span><a name="line-24"></a><span>
|
|
</span><a name="line-25"></a><span> </span><span class="hs-comment">-- * Implementation</span><span>
|
|
</span><a name="line-26"></a><span>
|
|
</span><a name="line-27"></a><span> </span><span class="hs-comment">-- | These may be useful for encoding or decoding to types other than</span><span>
|
|
</span><a name="line-28"></a><span> </span><span class="hs-comment">-- 'ByteString' that have their own Cereal codecs.</span><span>
|
|
</span><a name="line-29"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Records.html#getTFRecordLength"><span class="hs-identifier hs-var">getTFRecordLength</span></a><span>
|
|
</span><a name="line-30"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Records.html#getTFRecordData"><span class="hs-identifier hs-var">getTFRecordData</span></a><span>
|
|
</span><a name="line-31"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Records.html#putTFRecordLength"><span class="hs-identifier hs-var">putTFRecordLength</span></a><span>
|
|
</span><a name="line-32"></a><span> </span><span class="hs-special">,</span><span> </span><a href="TensorFlow.Records.html#putTFRecordData"><span class="hs-identifier hs-var">putTFRecordData</span></a><span>
|
|
</span><a name="line-33"></a><span> </span><span class="hs-special">)</span><span> </span><span class="hs-keyword">where</span><span>
|
|
</span><a name="line-34"></a><span>
|
|
</span><a name="line-35"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Control</span><span class="hs-operator">.</span><span class="hs-identifier">Exception</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">evaluate</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-36"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Control</span><span class="hs-operator">.</span><span class="hs-identifier">Monad</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">when</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-37"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">ByteString</span><span class="hs-operator">.</span><span class="hs-identifier">Unsafe</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">unsafePackCStringLen</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-38"></a><span class="hs-keyword">import</span><span> </span><span class="hs-keyword">qualified</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">ByteString</span><span class="hs-operator">.</span><span class="hs-identifier">Builder</span><span> </span><span class="hs-keyword">as</span><span> </span><span class="hs-identifier">B</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-type">Builder</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-39"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">ByteString</span><span class="hs-operator">.</span><span class="hs-identifier">Builder</span><span class="hs-operator">.</span><span class="hs-identifier">Extra</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">runBuilder</span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-type">Next</span><span class="hs-special">(</span><span class="hs-glyph">..</span><span class="hs-special">)</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-40"></a><span class="hs-keyword">import</span><span> </span><span class="hs-keyword">qualified</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">ByteString</span><span class="hs-operator">.</span><span class="hs-identifier">Lazy</span><span> </span><span class="hs-keyword">as</span><span> </span><span class="hs-identifier">BL</span><span>
|
|
</span><a name="line-41"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">Serialize</span><span class="hs-operator">.</span><span class="hs-identifier">Get</span><span>
|
|
</span><a name="line-42"></a><span> </span><span class="hs-special">(</span><span> </span><span class="hs-identifier hs-type">Get</span><span>
|
|
</span><a name="line-43"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">getBytes</span><span>
|
|
</span><a name="line-44"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">getWord32le</span><span>
|
|
</span><a name="line-45"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">getWord64le</span><span>
|
|
</span><a name="line-46"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">getLazyByteString</span><span>
|
|
</span><a name="line-47"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">isEmpty</span><span>
|
|
</span><a name="line-48"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">lookAhead</span><span>
|
|
</span><a name="line-49"></a><span> </span><span class="hs-special">)</span><span>
|
|
</span><a name="line-50"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">Serialize</span><span>
|
|
</span><a name="line-51"></a><span> </span><span class="hs-special">(</span><span> </span><span class="hs-identifier hs-type">Put</span><span>
|
|
</span><a name="line-52"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">execPut</span><span>
|
|
</span><a name="line-53"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">putLazyByteString</span><span>
|
|
</span><a name="line-54"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">putWord32le</span><span>
|
|
</span><a name="line-55"></a><span> </span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">putWord64le</span><span>
|
|
</span><a name="line-56"></a><span> </span><span class="hs-special">)</span><span>
|
|
</span><a name="line-57"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Data</span><span class="hs-operator">.</span><span class="hs-identifier">Word</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-type">Word8</span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-type">Word64</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-58"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Foreign</span><span class="hs-operator">.</span><span class="hs-identifier">Marshal</span><span class="hs-operator">.</span><span class="hs-identifier">Alloc</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">allocaBytes</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-59"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">Foreign</span><span class="hs-operator">.</span><span class="hs-identifier">Ptr</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-type">Ptr</span><span class="hs-special">,</span><span> </span><span class="hs-identifier hs-var">castPtr</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-60"></a><span class="hs-keyword">import</span><span> </span><span class="hs-identifier">System</span><span class="hs-operator">.</span><span class="hs-identifier">IO</span><span class="hs-operator">.</span><span class="hs-identifier">Unsafe</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">unsafePerformIO</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-61"></a><span>
|
|
</span><a name="line-62"></a><span class="hs-keyword">import</span><span> </span><a href="TensorFlow.CRC32C.html"><span class="hs-identifier">TensorFlow</span><span class="hs-operator">.</span><span class="hs-identifier">CRC32C</span></a><span> </span><span class="hs-special">(</span><a href="TensorFlow.CRC32C.html#crc32cLBSMasked"><span class="hs-identifier hs-var">crc32cLBSMasked</span></a><span class="hs-special">,</span><span> </span><a href="TensorFlow.CRC32C.html#crc32cUpdate"><span class="hs-identifier hs-var">crc32cUpdate</span></a><span class="hs-special">,</span><span> </span><a href="TensorFlow.CRC32C.html#crc32cMask"><span class="hs-identifier hs-var">crc32cMask</span></a><span class="hs-special">)</span><span>
|
|
</span><a name="line-63"></a><span>
|
|
</span><a name="line-64"></a><span class="hs-comment">-- | Parse one TFRecord.</span><span>
|
|
</span><a name="line-65"></a><span class="hs-identifier">getTFRecord</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Get</span><span> </span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span>
|
|
</span><a name="line-66"></a><a name="getTFRecord"><a href="TensorFlow.Records.html#getTFRecord"><span class="hs-identifier">getTFRecord</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.Records.html#getTFRecordLength"><span class="hs-identifier hs-var">getTFRecordLength</span></a><span> </span><span class="hs-operator hs-var">>>=</span><span> </span><a href="TensorFlow.Records.html#getTFRecordData"><span class="hs-identifier hs-var">getTFRecordData</span></a><span>
|
|
</span><a name="line-67"></a><span>
|
|
</span><a name="line-68"></a><span class="hs-comment">-- | Parse many TFRecords as a list. Note you probably want streaming instead</span><span>
|
|
</span><a name="line-69"></a><span class="hs-comment">-- as provided by the tensorflow-records-conduit package.</span><span>
|
|
</span><a name="line-70"></a><span class="hs-identifier">getTFRecords</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Get</span><span> </span><span class="hs-special">[</span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span class="hs-special">]</span><span>
|
|
</span><a name="line-71"></a><a name="getTFRecords"><a href="TensorFlow.Records.html#getTFRecords"><span class="hs-identifier">getTFRecords</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-keyword">do</span><span>
|
|
</span><a name="line-72"></a><span> </span><a name="local-6989586621679027727"><a href="#local-6989586621679027727"><span class="hs-identifier">e</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">isEmpty</span><span>
|
|
</span><a name="line-73"></a><span> </span><span class="hs-keyword">if</span><span> </span><a href="#local-6989586621679027727"><span class="hs-identifier hs-var">e</span></a><span> </span><span class="hs-keyword">then</span><span> </span><span class="hs-identifier hs-var">return</span><span> </span><span class="hs-special">[</span><span class="hs-special">]</span><span> </span><span class="hs-keyword">else</span><span> </span><span class="hs-special">(</span><span class="hs-glyph">:</span><span class="hs-special">)</span><span> </span><span class="hs-operator hs-var"><$></span><span> </span><a href="TensorFlow.Records.html#getTFRecord"><span class="hs-identifier hs-var">getTFRecord</span></a><span> </span><span class="hs-operator hs-var"><*></span><span> </span><a href="TensorFlow.Records.html#getTFRecords"><span class="hs-identifier hs-var">getTFRecords</span></a><span>
|
|
</span><a name="line-74"></a><span>
|
|
</span><a name="line-75"></a><span class="hs-identifier">getCheckMaskedCRC32C</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">Get</span><span> </span><span class="hs-special">(</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-76"></a><a name="getCheckMaskedCRC32C"><a href="TensorFlow.Records.html#getCheckMaskedCRC32C"><span class="hs-identifier">getCheckMaskedCRC32C</span></a></a><span> </span><a name="local-6989586621679027737"><a href="#local-6989586621679027737"><span class="hs-identifier">bs</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-keyword">do</span><span>
|
|
</span><a name="line-77"></a><span> </span><a name="local-6989586621679027738"><a href="#local-6989586621679027738"><span class="hs-identifier">wireCRC</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">getWord32le</span><span>
|
|
</span><a name="line-78"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679027739"><a href="#local-6989586621679027739"><span class="hs-identifier">maskedCRC</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.CRC32C.html#crc32cLBSMasked"><span class="hs-identifier hs-var">crc32cLBSMasked</span></a><span> </span><a href="#local-6989586621679027737"><span class="hs-identifier hs-var">bs</span></a><span>
|
|
</span><a name="line-79"></a><span> </span><span class="hs-identifier hs-var">when</span><span> </span><span class="hs-special">(</span><a href="#local-6989586621679027739"><span class="hs-identifier hs-var">maskedCRC</span></a><span> </span><span class="hs-operator hs-var">/=</span><span> </span><a href="#local-6989586621679027738"><span class="hs-identifier hs-var">wireCRC</span></a><span class="hs-special">)</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-identifier hs-var">fail</span><span> </span><span class="hs-operator hs-var">$</span><span>
|
|
</span><a name="line-80"></a><span> </span><span class="hs-string">"getCheckMaskedCRC32C: CRC mismatch, computed: "</span><span> </span><span class="hs-operator hs-var">++</span><span> </span><span class="hs-identifier hs-var">show</span><span> </span><a href="#local-6989586621679027739"><span class="hs-identifier hs-var">maskedCRC</span></a><span> </span><span class="hs-operator hs-var">++</span><span>
|
|
</span><a name="line-81"></a><span> </span><span class="hs-string">", expected: "</span><span> </span><span class="hs-operator hs-var">++</span><span> </span><span class="hs-identifier hs-var">show</span><span> </span><a href="#local-6989586621679027738"><span class="hs-identifier hs-var">wireCRC</span></a><span>
|
|
</span><a name="line-82"></a><span>
|
|
</span><a name="line-83"></a><span class="hs-comment">-- | Get a length and verify its checksum.</span><span>
|
|
</span><a name="line-84"></a><span class="hs-identifier">getTFRecordLength</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Get</span><span> </span><span class="hs-identifier hs-type">Word64</span><span>
|
|
</span><a name="line-85"></a><a name="getTFRecordLength"><a href="TensorFlow.Records.html#getTFRecordLength"><span class="hs-identifier">getTFRecordLength</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-keyword">do</span><span>
|
|
</span><a name="line-86"></a><span> </span><a name="local-6989586621679031882"><a href="#local-6989586621679031882"><span class="hs-identifier">buf</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">lookAhead</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">getBytes</span><span> </span><span class="hs-number">8</span><span class="hs-special">)</span><span>
|
|
</span><a name="line-87"></a><span> </span><span class="hs-identifier hs-var">getWord64le</span><span> </span><span class="hs-operator hs-var"><*</span><span> </span><a href="TensorFlow.Records.html#getCheckMaskedCRC32C"><span class="hs-identifier hs-var">getCheckMaskedCRC32C</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">BL</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">fromStrict</span><span> </span><a href="#local-6989586621679031882"><span class="hs-identifier hs-var">buf</span></a><span class="hs-special">)</span><span>
|
|
</span><a name="line-88"></a><span>
|
|
</span><a name="line-89"></a><span class="hs-comment">-- | Get a record payload and verify its checksum.</span><span>
|
|
</span><a name="line-90"></a><span class="hs-identifier">getTFRecordData</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Word64</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">Get</span><span> </span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span>
|
|
</span><a name="line-91"></a><a name="getTFRecordData"><a href="TensorFlow.Records.html#getTFRecordData"><span class="hs-identifier">getTFRecordData</span></a></a><span> </span><a name="local-6989586621679031883"><a href="#local-6989586621679031883"><span class="hs-identifier">len</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-keyword">if</span><span> </span><a href="#local-6989586621679031883"><span class="hs-identifier hs-var">len</span></a><span> </span><span class="hs-operator hs-var">></span><span> </span><span class="hs-number">0x7fffffffffffffff</span><span>
|
|
</span><a name="line-92"></a><span> </span><span class="hs-keyword">then</span><span> </span><span class="hs-identifier hs-var">fail</span><span> </span><span class="hs-string">"getTFRecordData: Record size overflows Int64"</span><span>
|
|
</span><a name="line-93"></a><span> </span><span class="hs-keyword">else</span><span> </span><span class="hs-keyword">do</span><span>
|
|
</span><a name="line-94"></a><span> </span><a name="local-6989586621679032426"><a href="#local-6989586621679032426"><span class="hs-identifier">bs</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">getLazyByteString</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">fromIntegral</span><span> </span><a href="#local-6989586621679031883"><span class="hs-identifier hs-var">len</span></a><span class="hs-special">)</span><span>
|
|
</span><a name="line-95"></a><span> </span><a href="TensorFlow.Records.html#getCheckMaskedCRC32C"><span class="hs-identifier hs-var">getCheckMaskedCRC32C</span></a><span> </span><a href="#local-6989586621679032426"><span class="hs-identifier hs-var">bs</span></a><span>
|
|
</span><a name="line-96"></a><span> </span><span class="hs-identifier hs-var">return</span><span> </span><a href="#local-6989586621679032426"><span class="hs-identifier hs-var">bs</span></a><span>
|
|
</span><a name="line-97"></a><span>
|
|
</span><a name="line-98"></a><span class="hs-identifier">putMaskedCRC32C</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">Put</span><span>
|
|
</span><a name="line-99"></a><a name="putMaskedCRC32C"><a href="TensorFlow.Records.html#putMaskedCRC32C"><span class="hs-identifier">putMaskedCRC32C</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">putWord32le</span><span> </span><span class="hs-operator hs-var">.</span><span> </span><a href="TensorFlow.CRC32C.html#crc32cLBSMasked"><span class="hs-identifier hs-var">crc32cLBSMasked</span></a><span>
|
|
</span><a name="line-100"></a><span>
|
|
</span><a name="line-101"></a><span class="hs-comment">-- Runs a Builder that's known to write a fixed number of bytes on an 'alloca'</span><span>
|
|
</span><a name="line-102"></a><span class="hs-comment">-- buffer, and runs the given IO action on the result. Raises exceptions if</span><span>
|
|
</span><a name="line-103"></a><span class="hs-comment">-- the Builder yields ByteString chunks or attempts to write more bytes than</span><span>
|
|
</span><a name="line-104"></a><span class="hs-comment">-- expected.</span><span>
|
|
</span><a name="line-105"></a><span class="hs-identifier">unsafeWithFixedWidthBuilder</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Int</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">B</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">Builder</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-type">Ptr</span><span> </span><span class="hs-identifier hs-type">Word8</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">IO</span><span> </span><a href="#local-6989586621679027403"><span class="hs-identifier hs-type">r</span></a><span class="hs-special">)</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">IO</span><span> </span><a href="#local-6989586621679027403"><span class="hs-identifier hs-type">r</span></a><span>
|
|
</span><a name="line-106"></a><a name="unsafeWithFixedWidthBuilder"><a href="TensorFlow.Records.html#unsafeWithFixedWidthBuilder"><span class="hs-identifier">unsafeWithFixedWidthBuilder</span></a></a><span> </span><a name="local-6989586621679032427"><a href="#local-6989586621679032427"><span class="hs-identifier">n</span></a></a><span> </span><a name="local-6989586621679032428"><a href="#local-6989586621679032428"><span class="hs-identifier">b</span></a></a><span> </span><a name="local-6989586621679032429"><a href="#local-6989586621679032429"><span class="hs-identifier">act</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">allocaBytes</span><span> </span><a href="#local-6989586621679032427"><span class="hs-identifier hs-var">n</span></a><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-glyph">\</span><a name="local-6989586621679032430"><a href="#local-6989586621679032430"><span class="hs-identifier">ptr</span></a></a><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-keyword">do</span><span>
|
|
</span><a name="line-107"></a><span> </span><span class="hs-special">(</span><span class="hs-identifier">_</span><span class="hs-special">,</span><span> </span><a name="local-6989586621679032431"><a href="#local-6989586621679032431"><span class="hs-identifier">signal</span></a></a><span class="hs-special">)</span><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">runBuilder</span><span> </span><a href="#local-6989586621679032428"><span class="hs-identifier hs-var">b</span></a><span> </span><a href="#local-6989586621679032430"><span class="hs-identifier hs-var">ptr</span></a><span> </span><a href="#local-6989586621679032427"><span class="hs-identifier hs-var">n</span></a><span>
|
|
</span><a name="line-108"></a><span> </span><span class="hs-keyword">case</span><span> </span><a href="#local-6989586621679032431"><span class="hs-identifier hs-var">signal</span></a><span> </span><span class="hs-keyword">of</span><span>
|
|
</span><a name="line-109"></a><span> </span><span class="hs-identifier hs-var">Done</span><span> </span><span class="hs-glyph">-></span><span> </span><a href="#local-6989586621679032429"><span class="hs-identifier hs-var">act</span></a><span> </span><a href="#local-6989586621679032430"><span class="hs-identifier hs-var">ptr</span></a><span>
|
|
</span><a name="line-110"></a><span> </span><span class="hs-identifier hs-var">More</span><span> </span><span class="hs-identifier">_</span><span> </span><span class="hs-identifier">_</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-var">error</span><span> </span><span class="hs-string">"unsafeWithFixedWidthBuilder: Builder returned More."</span><span>
|
|
</span><a name="line-111"></a><span> </span><span class="hs-identifier hs-var">Chunk</span><span> </span><span class="hs-identifier">_</span><span> </span><span class="hs-identifier">_</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-var">error</span><span> </span><span class="hs-string">"unsafeWithFixedWidthBuilder: Builder returned Chunk."</span><span>
|
|
</span><a name="line-112"></a><span>
|
|
</span><a name="line-113"></a><span class="hs-comment">-- | Put a record length and its checksum.</span><span>
|
|
</span><a name="line-114"></a><span class="hs-identifier">putTFRecordLength</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">Word64</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">Put</span><span>
|
|
</span><a name="line-115"></a><a name="putTFRecordLength"><a href="TensorFlow.Records.html#putTFRecordLength"><span class="hs-identifier">putTFRecordLength</span></a></a><span> </span><a name="local-6989586621679032462"><a href="#local-6989586621679032462"><span class="hs-identifier">x</span></a></a><span> </span><span class="hs-glyph">=</span><span>
|
|
</span><a name="line-116"></a><span> </span><span class="hs-keyword">let</span><span> </span><a name="local-6989586621679032463"><a href="#local-6989586621679032463"><span class="hs-identifier">put</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">putWord64le</span><span> </span><a href="#local-6989586621679032462"><span class="hs-identifier hs-var">x</span></a><span>
|
|
</span><a name="line-117"></a><span> </span><a name="local-6989586621679032464"><a href="#local-6989586621679032464"><span class="hs-identifier">len</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-number">8</span><span>
|
|
</span><a name="line-118"></a><span> </span><a name="local-6989586621679032465"><a href="#local-6989586621679032465"><span class="hs-identifier">crc</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><a href="TensorFlow.CRC32C.html#crc32cMask"><span class="hs-identifier hs-var">crc32cMask</span></a><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-identifier hs-var">unsafePerformIO</span><span> </span><span class="hs-operator hs-var">$</span><span>
|
|
</span><a name="line-119"></a><span> </span><span class="hs-comment">-- Serialized Word64 is always 8 bytes, so we can go fast by using</span><span>
|
|
</span><a name="line-120"></a><span> </span><span class="hs-comment">-- alloca.</span><span>
|
|
</span><a name="line-121"></a><span> </span><a href="TensorFlow.Records.html#unsafeWithFixedWidthBuilder"><span class="hs-identifier hs-var">unsafeWithFixedWidthBuilder</span></a><span> </span><a href="#local-6989586621679032464"><span class="hs-identifier hs-var">len</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">execPut</span><span> </span><a href="#local-6989586621679032463"><span class="hs-identifier hs-var">put</span></a><span class="hs-special">)</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-glyph">\</span><a name="local-6989586621679032479"><a href="#local-6989586621679032479"><span class="hs-identifier">ptr</span></a></a><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-keyword">do</span><span>
|
|
</span><a name="line-122"></a><span> </span><a name="local-6989586621679032480"><a href="#local-6989586621679032480"><span class="hs-identifier">str</span></a></a><span> </span><span class="hs-glyph"><-</span><span> </span><span class="hs-identifier hs-var">unsafePackCStringLen</span><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">castPtr</span><span> </span><a href="#local-6989586621679032479"><span class="hs-identifier hs-var">ptr</span></a><span class="hs-special">,</span><span> </span><a href="#local-6989586621679032464"><span class="hs-identifier hs-var">len</span></a><span class="hs-special">)</span><span>
|
|
</span><a name="line-123"></a><span> </span><span class="hs-comment">-- Force the result to ensure it's evaluated before freeing ptr.</span><span>
|
|
</span><a name="line-124"></a><span> </span><span class="hs-identifier hs-var">evaluate</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><a href="TensorFlow.CRC32C.html#crc32cUpdate"><span class="hs-identifier hs-var">crc32cUpdate</span></a><span> </span><span class="hs-number">0</span><span> </span><a href="#local-6989586621679032480"><span class="hs-identifier hs-var">str</span></a><span>
|
|
</span><a name="line-125"></a><span> </span><span class="hs-keyword">in</span><span> </span><a href="#local-6989586621679032463"><span class="hs-identifier hs-var">put</span></a><span> </span><span class="hs-operator hs-var">*></span><span> </span><span class="hs-identifier hs-var">putWord32le</span><span> </span><a href="#local-6989586621679032465"><span class="hs-identifier hs-var">crc</span></a><span>
|
|
</span><a name="line-126"></a><span>
|
|
</span><a name="line-127"></a><span class="hs-comment">-- | Put a record payload and its checksum.</span><span>
|
|
</span><a name="line-128"></a><span class="hs-identifier">putTFRecordData</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">Put</span><span>
|
|
</span><a name="line-129"></a><a name="putTFRecordData"><a href="TensorFlow.Records.html#putTFRecordData"><span class="hs-identifier">putTFRecordData</span></a></a><span> </span><a name="local-6989586621679032594"><a href="#local-6989586621679032594"><span class="hs-identifier">bs</span></a></a><span> </span><span class="hs-glyph">=</span><span> </span><span class="hs-identifier hs-var">putLazyByteString</span><span> </span><a href="#local-6989586621679032594"><span class="hs-identifier hs-var">bs</span></a><span> </span><span class="hs-operator hs-var">*></span><span> </span><a href="TensorFlow.Records.html#putMaskedCRC32C"><span class="hs-identifier hs-var">putMaskedCRC32C</span></a><span> </span><a href="#local-6989586621679032594"><span class="hs-identifier hs-var">bs</span></a><span>
|
|
</span><a name="line-130"></a><span>
|
|
</span><a name="line-131"></a><span class="hs-comment">-- | Put one TFRecord with the given contents.</span><span>
|
|
</span><a name="line-132"></a><span class="hs-identifier">putTFRecord</span><span> </span><span class="hs-glyph">::</span><span> </span><span class="hs-identifier hs-type">BL</span><span class="hs-operator hs-type">.</span><span class="hs-identifier hs-type">ByteString</span><span> </span><span class="hs-glyph">-></span><span> </span><span class="hs-identifier hs-type">Put</span><span>
|
|
</span><a name="line-133"></a><a name="putTFRecord"><a href="TensorFlow.Records.html#putTFRecord"><span class="hs-identifier">putTFRecord</span></a></a><span> </span><a name="local-6989586621679032595"><a href="#local-6989586621679032595"><span class="hs-identifier">bs</span></a></a><span> </span><span class="hs-glyph">=</span><span>
|
|
</span><a name="line-134"></a><span> </span><a href="TensorFlow.Records.html#putTFRecordLength"><span class="hs-identifier hs-var">putTFRecordLength</span></a><span> </span><span class="hs-special">(</span><span class="hs-identifier hs-var">fromIntegral</span><span> </span><span class="hs-operator hs-var">$</span><span> </span><span class="hs-identifier hs-var">BL</span><span class="hs-operator hs-var">.</span><span class="hs-identifier hs-var">length</span><span> </span><a href="#local-6989586621679032595"><span class="hs-identifier hs-var">bs</span></a><span class="hs-special">)</span><span> </span><span class="hs-operator hs-var">*></span><span> </span><a href="TensorFlow.Records.html#putTFRecordData"><span class="hs-identifier hs-var">putTFRecordData</span></a><span> </span><a href="#local-6989586621679032595"><span class="hs-identifier hs-var">bs</span></a><span>
|
|
</span><a name="line-135"></a></pre></body></html> |