Rewrote LaTeX reader with proper tokenization.

This rewrite is primarily motivated by the need to
get macros working properly.  A side benefit is that the
reader is significantly faster (27s -> 19s in one
benchmark, and there is a lot of room for further
optimization).

We now tokenize the input text, then parse the token stream.

Macros modify the token stream, so they should now be effective
in any context, including math. Thus, we no longer need the clunky
macro processing capacities of texmath.

A custom state LaTeXState is used instead of ParserState.
This, plus the tokenization, will require some rewriting
of the exported functions rawLaTeXInline, inlineCommand,
rawLaTeXBlock.

* Added Text.Pandoc.Readers.LaTeX.Types (new exported module).
  Exports Macro, Tok, TokType, Line, Column.  [API change]
* Text.Pandoc.Parsing: adjusted type of `insertIncludedFile`
  so it can be used with token parser.
* Removed old texmath macro stuff from Parsing.
  Use Macro from Text.Pandoc.Readers.LaTeX.Types instead.
* Removed texmath macro material from Markdown reader.
* Changed types for Text.Pandoc.Readers.LaTeX's
  rawLaTeXInline and rawLaTeXBlock.  (Both now return a String,
  and they are polymorphic in state.)
* Added orgMacros field to OrgState.  [API change]
* Removed readerApplyMacros from ReaderOptions.
  Now we just check the `latex_macros` reader extension.
* Allow `\newcommand\foo{blah}` without braces.

Fixes #1390.
Fixes #2118.
Fixes #3236.
Fixes #3779.
Fixes #934.
Fixes #982.

pandoc.cabal

@@ -366,6 +366,7 @@ Library
                    Text.Pandoc.Readers,
                    Text.Pandoc.Readers.HTML,
                    Text.Pandoc.Readers.LaTeX,
+                   Text.Pandoc.Readers.LaTeX.Types,
                    Text.Pandoc.Readers.Markdown,
                    Text.Pandoc.Readers.CommonMark,
                    Text.Pandoc.Readers.MediaWiki,

src/Text/Pandoc/App.hs

@@ -308,7 +308,6 @@ convertWithOpts opts = do
                       , readerColumns = optColumns opts
                       , readerTabStop = optTabStop opts
                       , readerIndentedCodeClasses = optIndentedCodeClasses opts
-                      , readerApplyMacros = not laTeXOutput
                       , readerDefaultImageExtension =
                          optDefaultImageExtension opts
                       , readerTrackChanges = optTrackChanges opts

src/Text/Pandoc/Error.hs

@@ -64,6 +64,7 @@ data PandocError = PandocIOError String IOError
                  | PandocTemplateError String
                  | PandocAppError String
                  | PandocEpubSubdirectoryError String
+                 | PandocMacroLoop String
                  deriving (Show, Typeable, Generic)
 
 instance Exception PandocError
@@ -107,6 +108,8 @@ handleError (Left e) =
     PandocAppError s -> err 1 s
     PandocEpubSubdirectoryError s -> err 31 $
       "EPUB subdirectory name '" ++ s ++ "' contains illegal characters"
+    PandocMacroLoop s -> err 91 $
+      "Loop encountered in expanding macro " ++ s
 
 err :: Int -> String -> IO a
 err exitCode msg = do

src/Text/Pandoc/Extensions.hs

@@ -318,6 +318,7 @@ getDefaultExtensions "epub2"           = getDefaultExtensions "epub"
 getDefaultExtensions "epub3"           = getDefaultExtensions "epub"
 getDefaultExtensions "latex"           = extensionsFromList
                                           [Ext_smart,
+                                           Ext_latex_macros,
                                            Ext_auto_identifiers]
 getDefaultExtensions "context"         = extensionsFromList
                                           [Ext_smart,

src/Text/Pandoc/Options.hs

@@ -61,7 +61,6 @@ data ReaderOptions = ReaderOptions{
        , readerStandalone            :: Bool -- ^ Standalone document with header
        , readerColumns               :: Int  -- ^ Number of columns in terminal
        , readerTabStop               :: Int  -- ^ Tab stop
-       , readerApplyMacros           :: Bool -- ^ Apply macros to TeX math
        , readerIndentedCodeClasses   :: [String] -- ^ Default classes for
                                        -- indented code blocks
        , readerAbbreviations         :: Set.Set String -- ^ Strings to treat as abbreviations
@@ -75,7 +74,6 @@ instance Default ReaderOptions
                , readerStandalone            = False
                , readerColumns               = 80
                , readerTabStop               = 4
-               , readerApplyMacros           = True
                , readerIndentedCodeClasses   = []
                , readerAbbreviations         = defaultAbbrevs
                , readerDefaultImageExtension = ""

src/Text/Pandoc/Parsing.hs

@@ -109,8 +109,6 @@ module Text.Pandoc.Parsing ( anyLine,
                              dash,
                              nested,
                              citeKey,
-                             macro,
-                             applyMacros',
                              Parser,
                              ParserT,
                              F,
@@ -130,6 +128,7 @@ module Text.Pandoc.Parsing ( anyLine,
                              runParser,
                              runParserT,
                              parse,
+                             tokenPrim,
                              anyToken,
                              getInput,
                              setInput,
@@ -178,13 +177,16 @@ module Text.Pandoc.Parsing ( anyLine,
                              sourceLine,
                              setSourceColumn,
                              setSourceLine,
-                             newPos
+                             newPos,
+                             Line,
+                             Column
                              )
 where
 
+import Data.Text (Text)
 import Text.Pandoc.Definition
 import Text.Pandoc.Options
-import Text.Pandoc.Builder (Blocks, Inlines, rawBlock, HasMeta(..), trimInlines)
+import Text.Pandoc.Builder (Blocks, Inlines, HasMeta(..), trimInlines)
 import qualified Text.Pandoc.Builder as B
 import Text.Pandoc.XML (fromEntities)
 import qualified Text.Pandoc.UTF8 as UTF8 (putStrLn)
@@ -195,7 +197,7 @@ import Data.Char ( toLower, toUpper, ord, chr, isAscii, isAlphaNum,
 import Data.List ( intercalate, transpose, isSuffixOf )
 import Text.Pandoc.Shared
 import qualified Data.Map as M
-import Text.TeXMath.Readers.TeX.Macros (applyMacros, Macro, pMacroDefinition)
+import Text.Pandoc.Readers.LaTeX.Types (Macro)
 import Text.HTML.TagSoup.Entity ( lookupEntity )
 import Text.Pandoc.Asciify (toAsciiChar)
 import Data.Monoid ((<>))
@@ -994,7 +996,7 @@ data ParserState = ParserState
       stateIdentifiers     :: Set.Set String, -- ^ Header identifiers used
       stateNextExample     :: Int,           -- ^ Number of next example
       stateExamples        :: M.Map String Int, -- ^ Map from example labels to numbers
-      stateMacros          :: [Macro],       -- ^ List of macros defined so far
+      stateMacros          :: M.Map Text Macro, -- ^ Table of macros defined so far
       stateRstDefaultRole  :: String,        -- ^ Current rST default interpreted text role
       stateRstCustomRoles  :: M.Map String (String, Maybe String, Attr), -- ^ Current rST custom text roles
       -- Triple represents: 1) Base role, 2) Optional format (only for :raw:
@@ -1057,8 +1059,8 @@ instance HasIdentifierList ParserState where
   updateIdentifierList f st = st{ stateIdentifiers = f $ stateIdentifiers st }
 
 class HasMacros st where
-  extractMacros         :: st -> [Macro]
+  extractMacros         :: st -> M.Map Text Macro
-  updateMacros          :: ([Macro] -> [Macro]) -> st -> st
+  updateMacros          :: (M.Map Text Macro -> M.Map Text Macro) -> st -> st
 
 instance HasMacros ParserState where
   extractMacros        = stateMacros
@@ -1112,7 +1114,7 @@ defaultParserState =
                   stateIdentifiers     = Set.empty,
                   stateNextExample     = 1,
                   stateExamples        = M.empty,
-                  stateMacros          = [],
+                  stateMacros          = M.empty,
                   stateRstDefaultRole  = "title-reference",
                   stateRstCustomRoles  = M.empty,
                   stateCaption         = Nothing,
@@ -1341,33 +1343,6 @@ token :: (Stream s m t)
       -> ParsecT s st m a
 token pp pos match = tokenPrim pp (\_ t _ -> pos t) match
 
---
--- Macros
---
-
--- | Parse a \newcommand or \newenviroment macro definition.
-macro :: (Stream [Char] m Char, HasMacros st, HasReaderOptions st)
-      => ParserT [Char] st m Blocks
-macro = do
-  apply <- getOption readerApplyMacros
-  (m, def') <- withRaw pMacroDefinition
-  if apply
-     then do
-       updateState $ \st -> updateMacros (m:) st
-       return mempty
-     else return $ rawBlock "latex" def'
-
--- | Apply current macros to string.
-applyMacros' :: (HasReaderOptions st, HasMacros st, Stream [Char] m Char)
-             => String
-             -> ParserT [Char] st m String
-applyMacros' target = do
-  apply <- getOption readerApplyMacros
-  if apply
-     then do macros <- extractMacros <$> getState
-             return $ applyMacros macros target
-     else return target
-
 infixr 5 <+?>
 (<+?>) :: (Monoid a) => ParserT s st m a -> ParserT s st m a -> ParserT s st m a
 a <+?> b = a >>= flip fmap (try b <|> return mempty) . (<>)
@@ -1385,10 +1360,11 @@ extractIdClass (ident, cls, kvs) = (ident', cls', kvs')
 
 insertIncludedFile' :: (PandocMonad m, HasIncludeFiles st,
                         Functor mf, Applicative mf, Monad mf)
-                    => ParserT String st m (mf Blocks)
+                    => ParserT [a] st m (mf Blocks)
+                    -> (String -> [a])
                     -> [FilePath] -> FilePath
-                    -> ParserT String st m (mf Blocks)
+                    -> ParserT [a] st m (mf Blocks)
-insertIncludedFile' blocks dirs f = do
+insertIncludedFile' blocks totoks dirs f = do
   oldPos <- getPosition
   oldInput <- getInput
   containers <- getIncludeFiles <$> getState
@@ -1402,7 +1378,7 @@ insertIncludedFile' blocks dirs f = do
                      report $ CouldNotLoadIncludeFile f oldPos
                      return ""
   setPosition $ newPos f 1 1
-  setInput contents
+  setInput $ totoks contents
   bs <- blocks
   setInput oldInput
   setPosition oldPos
@@ -1412,11 +1388,12 @@ insertIncludedFile' blocks dirs f = do
 -- | Parse content of include file as blocks. Circular includes result in an
 -- @PandocParseError@.
 insertIncludedFile :: (PandocMonad m, HasIncludeFiles st)
-                   => ParserT String st m Blocks
+                   => ParserT [a] st m Blocks
+                   -> (String -> [a])
                    -> [FilePath] -> FilePath
-                   -> ParserT String st m Blocks
+                   -> ParserT [a] st m Blocks
-insertIncludedFile blocks dirs f =
+insertIncludedFile blocks totoks dirs f =
-  runIdentity <$> insertIncludedFile' (Identity <$> blocks) dirs f
+  runIdentity <$> insertIncludedFile' (Identity <$> blocks) totoks dirs f
 
 -- | Parse content of include file as future blocks. Circular includes result in
 -- an @PandocParseError@.
@@ -1424,4 +1401,4 @@ insertIncludedFileF :: (PandocMonad m, HasIncludeFiles st)
                     => ParserT String st m (Future st Blocks)
                     -> [FilePath] -> FilePath
                     -> ParserT String st m (Future st Blocks)
-insertIncludedFileF = insertIncludedFile'
+insertIncludedFileF p = insertIncludedFile' p id

src/Text/Pandoc/Readers/LaTeX/Types.hs

@@ -0,0 +1,48 @@
+{-
+Copyright (C) 2017 John MacFarlane <jgm@berkeley.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+-}
+
+{- |
+   Module      : Text.Pandoc.Readers.LaTeX.Types
+   Copyright   : Copyright (C) 2017 John MacFarlane
+   License     : GNU GPL, version 2 or above
+
+   Maintainer  : John MacFarlane <jgm@berkeley.edu>
+   Stability   : alpha
+   Portability : portable
+
+Types for LaTeX tokens and macros.
+-}
+module Text.Pandoc.Readers.LaTeX.Types ( Tok(..)
+                                       , TokType(..)
+                                       , Macro(..)
+                                       , Line
+                                       , Column )
+where
+import Data.Text (Text)
+import Text.Parsec.Pos (Line, Column)
+
+data TokType = CtrlSeq Text | Spaces | Newline | Symbol | Word | Comment |
+               Esc1    | Esc2   | Arg Int
+     deriving (Eq, Ord, Show)
+
+data Tok = Tok (Line, Column) TokType Text
+     deriving (Eq, Ord, Show)
+
+data Macro = Macro Int (Maybe [Tok]) [Tok]
+     deriving Show
+

src/Text/Pandoc/Readers/Markdown.hs

@@ -61,7 +61,8 @@ import Text.Pandoc.Options
 import Text.Pandoc.Parsing hiding (tableWith)
 import Text.Pandoc.Readers.HTML (htmlInBalanced, htmlTag, isBlockTag,
                                  isCommentTag, isInlineTag, isTextTag)
-import Text.Pandoc.Readers.LaTeX (rawLaTeXBlock, rawLaTeXInline)
+import Text.Pandoc.Readers.LaTeX (rawLaTeXBlock, rawLaTeXInline, applyMacros,
+                                  macro)
 import Text.Pandoc.Shared
 import qualified Text.Pandoc.UTF8 as UTF8
 import Text.Pandoc.XML (fromEntities)
@@ -1105,10 +1106,11 @@ latexMacro = try $ do
 rawTeXBlock :: PandocMonad m => MarkdownParser m (F Blocks)
 rawTeXBlock = do
   guardEnabled Ext_raw_tex
-  result <- (B.rawBlock "latex" . concat <$>
+  result <- (B.rawBlock "context" . concat <$>
-                  rawLaTeXBlock `sepEndBy1` blankline)
-        <|> (B.rawBlock "context" . concat <$>
                   rawConTeXtEnvironment `sepEndBy1` blankline)
+        <|> (B.rawBlock "latex" . concat <$>
+                  rawLaTeXBlock `sepEndBy1` blankline)
+
   spaces
   return $ return result
 
@@ -1553,8 +1555,8 @@ code = try $ do
          Right attr -> B.codeWith attr result
 
 math :: PandocMonad m => MarkdownParser m (F Inlines)
-math =  (return . B.displayMath <$> (mathDisplay >>= applyMacros'))
+math =  (return . B.displayMath <$> (mathDisplay >>= applyMacros))
-     <|> (return . B.math <$> (mathInline >>= applyMacros')) <+?>
+     <|> (return . B.math <$> (mathInline >>= applyMacros)) <+?>
                (guardEnabled Ext_smart *> (return <$> apostrophe)
                 <* notFollowedBy (space <|> satisfy isPunctuation))
 
@@ -1878,9 +1880,8 @@ rawLaTeXInline' = try $ do
   guardEnabled Ext_raw_tex
   lookAhead (char '\\')
   notFollowedBy' rawConTeXtEnvironment
-  RawInline _ s <- rawLaTeXInline
+  s <- rawLaTeXInline
-  return $ return $ B.rawInline "tex" s
+  return $ return $ B.rawInline "tex" s -- "tex" because it might be context
-  -- "tex" because it might be context or latex
 
 rawConTeXtEnvironment :: PandocMonad m => ParserT [Char] st m String
 rawConTeXtEnvironment = try $ do

src/Text/Pandoc/Readers/Muse.hs

@@ -58,7 +58,7 @@ import Text.Pandoc.Definition
 import Text.Pandoc.Logging
 import Text.Pandoc.Options
 import Text.Pandoc.Shared (crFilter)
-import Text.Pandoc.Parsing hiding (macro, nested)
+import Text.Pandoc.Parsing hiding (nested)
 import Text.Pandoc.Readers.HTML (htmlTag)
 import Text.Pandoc.XML (fromEntities)
 import System.FilePath (takeExtension)

src/Text/Pandoc/Readers/Org/Inlines.hs

@@ -826,9 +826,10 @@ maybeRight = either (const Nothing) Just
 inlineLaTeXCommand :: PandocMonad m => OrgParser m String
 inlineLaTeXCommand = try $ do
   rest <- getInput
-  parsed <- (lift . lift) $ runParserT rawLaTeXInline def "source" rest
+  st <- getState
+  parsed <- (lift . lift) $ runParserT rawLaTeXInline st "source" rest
   case parsed of
-    Right (RawInline _ cs) -> do
+    Right cs -> do
       -- drop any trailing whitespace, those are not be part of the command as
       -- far as org mode is concerned.
       let cmdNoSpc = dropWhileEnd isSpace cs

src/Text/Pandoc/Readers/Org/ParserState.hs

@@ -33,6 +33,7 @@ module Text.Pandoc.Readers.Org.ParserState
   , OrgNoteRecord
   , HasReaderOptions (..)
   , HasQuoteContext (..)
+  , HasMacros (..)
   , TodoMarker (..)
   , TodoSequence
   , TodoState (..)
@@ -57,14 +58,17 @@ import Control.Monad.Reader (ReaderT, asks, local)
 import Data.Default (Default (..))
 import qualified Data.Map as M
 import qualified Data.Set as Set
+import Data.Text (Text)
 
 import Text.Pandoc.Builder (Blocks, Inlines)
 import Text.Pandoc.Definition (Meta (..), nullMeta)
 import Text.Pandoc.Logging
 import Text.Pandoc.Options (ReaderOptions (..))
+import Text.Pandoc.Readers.LaTeX.Types (Macro)
 import Text.Pandoc.Parsing (Future, HasHeaderMap (..), HasIdentifierList (..),
                             HasIncludeFiles (..), HasLastStrPosition (..),
                             HasLogMessages (..), HasQuoteContext (..),
+                            HasMacros (..),
                             HasReaderOptions (..), ParserContext (..),
                             QuoteContext (..), SourcePos, askF, asksF, returnF,
                             runF, trimInlinesF)
@@ -118,6 +122,7 @@ data OrgParserState = OrgParserState
   , orgStateParserContext        :: ParserContext
   , orgStateTodoSequences        :: [TodoSequence]
   , orgLogMessages               :: [LogMessage]
+  , orgMacros                    :: M.Map Text Macro
   }
 
 data OrgParserLocal = OrgParserLocal { orgLocalQuoteContext :: QuoteContext }
@@ -148,6 +153,10 @@ instance HasLogMessages OrgParserState where
   addLogMessage msg st = st{ orgLogMessages = msg : orgLogMessages st }
   getLogMessages st = reverse $ orgLogMessages st
 
+instance HasMacros OrgParserState where
+  extractMacros st = orgMacros st
+  updateMacros f st = st{ orgMacros = f (orgMacros st) }
+
 instance HasIncludeFiles OrgParserState where
   getIncludeFiles = orgStateIncludeFiles
   addIncludeFile f st = st { orgStateIncludeFiles = f : orgStateIncludeFiles st }
@@ -178,6 +187,7 @@ defaultOrgParserState = OrgParserState
   , orgStateParserContext = NullState
   , orgStateTodoSequences = []
   , orgLogMessages = []
+  , orgMacros = M.empty
   }
 
 optionsToParserState :: ReaderOptions -> OrgParserState

src/Text/Pandoc/Readers/TWiki.hs

@@ -45,7 +45,7 @@ import qualified Text.Pandoc.Builder as B
 import Text.Pandoc.Class (PandocMonad(..))
 import Text.Pandoc.Definition
 import Text.Pandoc.Options
-import Text.Pandoc.Parsing hiding (enclosed, macro, nested)
+import Text.Pandoc.Parsing hiding (enclosed, nested)
 import Text.Pandoc.Readers.HTML (htmlTag, isCommentTag)
 import Text.Pandoc.XML (fromEntities)
 import Text.Pandoc.Shared (crFilter)

src/Text/Pandoc/Readers/Textile.hs

@@ -573,7 +573,7 @@ rawHtmlInline = B.rawInline "html" . snd <$> htmlTag isInlineTag
 rawLaTeXInline' :: PandocMonad m => ParserT [Char] ParserState m Inlines
 rawLaTeXInline' = try $ do
   guardEnabled Ext_raw_tex
-  B.singleton <$> rawLaTeXInline
+  B.rawInline "latex" <$> rawLaTeXInline
 
 -- | Textile standard link syntax is "label":target. But we
 -- can also have ["label":target].

src/Text/Pandoc/Readers/Txt2Tags.hs

@@ -40,7 +40,7 @@ import Text.Pandoc.Builder (Blocks, Inlines, trimInlines)
 import qualified Text.Pandoc.Builder as B
 import Text.Pandoc.Definition
 import Text.Pandoc.Options
-import Text.Pandoc.Parsing hiding (macro, space, spaces, uri)
+import Text.Pandoc.Parsing hiding (space, spaces, uri)
 import Text.Pandoc.Shared (compactify, compactifyDL, escapeURI, crFilter)
 import Control.Monad (guard, void, when)
 import Control.Monad.Reader (Reader, asks, runReader)

test/Tests/Readers/LaTeX.hs

@@ -58,7 +58,8 @@ tests = [ testGroup "basic"
           , "blank lines + space + comments" =:
             "% my comment\n\n  \n  % another\n\nhi" =?> para "hi"
           , "comment in paragraph" =:
-            "hi % this is a comment\nthere\n" =?> para "hi there"
+            "hi % this is a comment\nthere\n" =?>
+                para ("hi" <> softbreak <> "there")
           ]
 
         , testGroup "code blocks"

test/command/1390.md

@@ -0,0 +1,20 @@
+```
+% pandoc -f latex -t native
+\newcommand\foo{+}
+Testing: $\mu\foo\eta$.
+^D
+[Para [Str "Testing:",Space,Math InlineMath "\\mu+\\eta",Str "."]]
+```
+
+<!-- It would be nice to handle this case, but I don't
+know how:
+
+```
+% pandoc -f latex -t native
+\newcommand{\vecx}{a + b}
+$\hat\vecx$
+^D
+[Para [Math InlineMath "\\hat{a+b}"]]
+```
+-->
+

test/command/2118.md

@@ -0,0 +1,11 @@
+```
+% pandoc -f latex -t native
+\newcommand{\inclgraph}{\includegraphics[width=0.8\textwidth]}
+\begin{figure}[ht]
+  \inclgraph{setminus.png}
+  \caption{Set subtraction}
+  \label{fig:setminus}
+\end{figure}
+^D
+[Para [Image ("",[],[("width","80%")]) [Str "Set",Space,Str "subtraction",Span ("",[],[("data-label","fig:setminus")]) []] ("setminus.png","fig:")]]
+```

test/command/3113.md

@@ -8,6 +8,6 @@ C&=&D,\\
 E&=&F
 \end{eqnarray}
 ^D
-[Para [Math DisplayMath "\\begin{aligned}\nA&=&B,\\\\\nC&=&D,\\\\\nE&=&F\\end{aligned}"]]
+[Para [Math DisplayMath "\\begin{aligned}\nA&=&B,\\\\\nC&=&D,\\\\\n%\\end{eqnarray}\n%\\begin{eqnarray}\nE&=&F\\end{aligned}"]]
 ```
 

test/command/3236.md

@@ -0,0 +1,9 @@
+```
+pandoc -f latex -t native
+\newcommand{\mycolor}{red}
+
+\includegraphics[width=17cm]{\mycolor /header}
+Magnificent \mycolor{} header.
+^D
+[Para [Image ("",[],[("width","17cm")]) [Str "image"] ("red/header",""),SoftBreak,Str "Magnificent",Space,Str "red",Space,Str "header."]]
+```

test/command/3558.md

@@ -1,8 +1,12 @@
 ```
 % pandoc -t native
-\startmulti
+\multi
+
 hello
+
 \endmulti
 ^D
-[Para [RawInline (Format "tex") "\\startmulti\n",Str "hello",SoftBreak,RawInline (Format "tex") "\\endmulti"]]
+[RawBlock (Format "latex") "\\multi"
+,Para [Str "hello"]
+,RawBlock (Format "latex") "\\endmulti"]
 ```

test/command/3779.md

@@ -0,0 +1,28 @@
+```
+% pandoc -f latex -t native
+\newcommand{\fakeitemize}[1]{
+    \begin{itemize}
+        #1
+    \end{itemize}
+}
+\newcommand{\testcmd}[1]{
+    #1
+}
+\fakeitemize{
+    \item Pandoc is 100\% awesome.
+}
+
+\begin{itemize}
+    \item Pandoc is 200\% awesome.
+\end{itemize}
+
+\testcmd{
+    Pandoc is 300\% awesome.
+}
+^D
+[BulletList
+ [[Para [Str "Pandoc",Space,Str "is",Space,Str "100%",Space,Str "awesome."]]]
+,BulletList
+ [[Para [Str "Pandoc",Space,Str "is",Space,Str "200%",Space,Str "awesome."]]]
+,Para [Str "Pandoc",Space,Str "is",Space,Str "300%",Space,Str "awesome."]]
+```

test/command/934.md

@@ -0,0 +1,12 @@
+```
+% pandoc -f latex -t native
+\newcommand{\ddb}[2]{
+  \textit{``#1''}
+
+  \textbf{#2}
+}
+\ddb{This should be italic and in quotes}{And this is the attribution}
+^D
+[Para [Emph [Quoted DoubleQuote [Str "This",Space,Str "should",Space,Str "be",Space,Str "italic",Space,Str "and",Space,Str "in",Space,Str "quotes"]]]
+,Para [Strong [Str "And",Space,Str "this",Space,Str "is",Space,Str "the",Space,Str "attribution"]]]
+```

test/command/982.md

@@ -0,0 +1,11 @@
+```
+% pandoc -f latex -t native
+\newcommand{\BEQ}{\begin{equation}}
+\newcommand{\EEQ}{\end{equation}}
+
+\BEQ
+y=x^2
+\EEQ
+^D
+[Para [Math DisplayMath "y=x^2"]]
+```

test/latex-reader.latex

@@ -4,7 +4,6 @@
 \setlength{\parindent}{0pt}
 \setlength{\parskip}{6pt plus 2pt minus 1pt}
 
-\newcommand{\textsubscript}[1]{\ensuremath{_{\scriptsize\textrm{#1}}}}
 \usepackage[breaklinks=true,unicode=true]{hyperref}
 \usepackage[normalem]{ulem}
 % avoid problems with \sout in headers with hyperref:

test/latex-reader.native

@@ -261,7 +261,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Spa
 ,Header 1 ("latex",[],[]) [Str "LaTeX"]
 ,BulletList
  [[Para [Cite [Citation {citationId = "smith.1899", citationPrefix = [], citationSuffix = [Str "22-23"], citationMode = NormalCitation, citationNoteNum = 0, citationHash = 0}] [RawInline (Format "latex") "\\cite[22-23]{smith.1899}"]]]
- ,[Para [RawInline (Format "latex") "\\doublespacing\n"]]
+ ,[Para [RawInline (Format "latex") "\\doublespacing"]]
  ,[Para [Math InlineMath "2+2=4"]]
  ,[Para [Math InlineMath "x \\in y"]]
  ,[Para [Math InlineMath "\\alpha \\wedge \\omega"]]

test/markdown-reader-more.native

@@ -3,7 +3,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
 ,Header 2 ("blank-line-before-url-in-link-reference",[],[]) [Str "Blank",Space,Str "line",Space,Str "before",Space,Str "URL",Space,Str "in",Space,Str "link",Space,Str "reference"]
 ,Para [Link ("",[],[]) [Str "foo"] ("/url",""),Space,Str "and",Space,Link ("",[],[]) [Str "bar"] ("/url","title")]
 ,Header 2 ("raw-context-environments",[],[]) [Str "Raw",Space,Str "ConTeXt",Space,Str "environments"]
-,Plain [RawInline (Format "tex") "\\placeformula "]
+,RawBlock (Format "latex") "\\placeformula "
 ,RawBlock (Format "context") "\\startformula\n   L_{1} = L_{2}\n   \\stopformula"
 ,RawBlock (Format "context") "\\start[a2]\n\\start[a2]\n\\stop[a2]\n\\stop[a2]"
 ,Header 2 ("raw-latex-environments",[],[]) [Str "Raw",Space,Str "LaTeX",Space,Str "environments"]
@@ -56,7 +56,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S
 ,OrderedList (3,Example,TwoParens)
  [[Plain [Str "Third",Space,Str "example."]]]
 ,Header 2 ("macros",[],[]) [Str "Macros"]
-,Para [Math InlineMath "{\\langle x,y \\rangle}"]
+,Para [Math InlineMath "\\langle x,y \\rangle"]
 ,Header 2 ("case-insensitive-references",[],[]) [Str "Case-insensitive",Space,Str "references"]
 ,Para [Link ("",[],[]) [Str "Fum"] ("/fum","")]
 ,Para [Link ("",[],[]) [Str "FUM"] ("/fum","")]