From 1ef35343288ae7cb1d074c6a25aa5e47fdc55da9 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Tue, 16 Mar 2021 15:59:50 -0700
Subject: [PATCH] Increase heap space in runtime for benchmarks.
Otherwise we're essentially benchmarking garbage collecting,
which can give very inconsistent results.
---
Makefile | 9 +++++----
benchmark/benchmark-pandoc.hs | 1 +
pandoc.cabal | 1 +
stack.yaml | 1 +
4 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index ac1850c2d..60aaeb43f 100644
--- a/Makefile
+++ b/Makefile
@@ -15,9 +15,10 @@ endif
GHCOPTS=-fdiagnostics-color=always -j4 +RTS -A256m -RTS
WEBSITE=../../web/pandoc.org
REVISION?=1
-BENCHARGS?=--small --time-limit=2 --match=pattern $(PATTERN)
+# For gauge:
+BENCHARGS?=--small --ci=0.90 --match=pattern $(PATTERN) +RTS -T -A256m -I0 -RTS
# For tasty-bench:
-# BENCHARGS?=--csv bench_$(TIMESTAMP).csv --timeout=6 +RTS -T -RTS $(if $(PATTERN),--pattern "$(PATTERN)",)
+# BENCHARGS?=--csv bench_$(TIMESTAMP).csv --timeout=6 +RTS -T -A256m -I0 -RTS $(if $(PATTERN),--pattern "$(PATTERN)",)
quick:
stack install --ghc-options='$(GHCOPTS)' --install-ghc --flag 'pandoc:embed_data_files' --fast --test --ghc-options='$(GHCOPTS)' --test-arguments='-j4 --hide-successes $(TESTARGS)'
@@ -57,8 +58,8 @@ bench:
--ghc-options '-Rghc-timing $(GHCOPTS)' \
--benchmark-arguments='$(BENCHARGS)' 2>&1 | \
tee "bench_latest.txt"
- perl -pe 's/\x1b\[[0-9;]*[mGK]//g;s/\r//;' bench_latest.txt > \
- "bench_$(TIMESTAMP).txt"
+ perl -ne 'if (/\r/) { s/\x1b\[[0-9;]*[mGK]//g;s/^.*\r//;print; }' \
+ bench_latest.txt > "bench_$(TIMESTAMP).txt"
reformat:
for f in $(SOURCEFILES); do echo $$f; stylish-haskell -i $$f ; done
diff --git a/benchmark/benchmark-pandoc.hs b/benchmark/benchmark-pandoc.hs
index f9a872011..111e63274 100644
--- a/benchmark/benchmark-pandoc.hs
+++ b/benchmark/benchmark-pandoc.hs
@@ -23,6 +23,7 @@ import Control.Monad.Except (throwError)
import qualified Text.Pandoc.UTF8 as UTF8
import qualified Data.ByteString as B
import qualified Data.Text as T
+-- import Test.Tasty.Bench
import Gauge
import qualified Data.ByteString.Lazy as BL
import Data.Maybe (mapMaybe)
diff --git a/pandoc.cabal b/pandoc.cabal
index 647505211..2f49f0ac3 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -827,6 +827,7 @@ benchmark benchmark-pandoc
build-depends: bytestring,
containers,
gauge >= 0.2 && < 0.3,
+ -- tasty-bench >= 0.2 && <= 0.3,
mtl >= 2.2 && < 2.3,
text >= 1.1.1.0 && < 1.3,
time,
diff --git a/stack.yaml b/stack.yaml
index 0f21bbb6c..7a6462d51 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -13,6 +13,7 @@ extra-deps:
- doclayout-0.3.0.2
- citeproc-0.3.0.9
- texmath-0.12.2
+- tasty-bench-0.2.2
ghc-options:
"$locals": -fhide-source-paths -Wno-missing-home-modules
resolver: lts-17.5