diff --git a/pandoc.cabal b/pandoc.cabal
index 481e5d076..31cabd52e 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -148,6 +148,7 @@ extra-source-files:
test/command/3510-subdoc.org
test/command/3510-export.latex
test/command/3510-src.hs
+ test/command/3971b.tex
test/docbook-reader.docbook
test/docbook-xref.docbook
test/html-reader.html
diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index 57db33871..8c6c7d0ff 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -71,7 +71,9 @@ import Text.Pandoc.Shared
import Text.Pandoc.Readers.LaTeX.Types (Macro(..), ExpansionPoint(..), Tok(..),
TokType(..))
import Text.Pandoc.Walk
-import Text.Pandoc.Error (PandocError(PandocParsecError, PandocMacroLoop))
+import Text.Pandoc.Error
+ (PandocError(PandocParsecError, PandocParseError, PandocMacroLoop))
+import Text.Parsec.Pos
-- for debugging:
-- import Text.Pandoc.Extensions (getDefaultExtensions)
@@ -85,7 +87,7 @@ readLaTeX :: PandocMonad m
-> m Pandoc
readLaTeX opts ltx = do
parsed <- runParserT parseLaTeX def{ sOptions = opts } "source"
- (tokenize (crFilter ltx))
+ (tokenize "source" (crFilter ltx))
case parsed of
Right result -> return result
Left e -> throwError $ PandocParsecError (T.unpack ltx) e
@@ -127,7 +129,7 @@ resolveRefs _ x = x
-- res <- runIOorExplode (runParserT p defaultLaTeXState{
-- sOptions = def{ readerExtensions =
-- enableExtension Ext_raw_tex $
--- getDefaultExtensions "latex" }} "source" (tokenize t))
+-- getDefaultExtensions "latex" }} "source" (tokenize "source" t))
-- case res of
-- Left e -> error (show e)
-- Right r -> return r
@@ -238,7 +240,7 @@ rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s)
=> LP m a -> ParserT String s m String
rawLaTeXParser parser = do
inp <- getInput
- let toks = tokenize $ T.pack inp
+ let toks = tokenize "source" $ T.pack inp
pstate <- getState
let lstate = def{ sOptions = extractReaderOptions pstate }
res <- lift $ runParserT ((,) <$> try (snd <$> withRaw parser) <*> getState)
@@ -257,7 +259,7 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>
pstate <- getState
let lstate = def{ sOptions = extractReaderOptions pstate
, sMacros = extractMacros pstate }
- res <- runParserT retokenize lstate "math" (tokenize (T.pack s))
+ res <- runParserT retokenize lstate "math" (tokenize "math" (T.pack s))
case res of
Left e -> fail (show e)
Right s' -> return s'
@@ -278,7 +280,7 @@ inlineCommand :: PandocMonad m => ParserT String ParserState m Inlines
inlineCommand = do
lookAhead (try (char '\\' >> letter) <|> char '$')
inp <- getInput
- let toks = tokenize $ T.pack inp
+ let toks = tokenize "chunk" $ T.pack inp
let rawinline = do
(il, raw) <- try $ withRaw (inlineEnvironment <|> inlineCommand')
st <- getState
@@ -294,32 +296,33 @@ inlineCommand = do
takeP (T.length (untokenize raw))
return il
-tokenize :: Text -> [Tok]
-tokenize = totoks (1, 1)
+tokenize :: SourceName -> Text -> [Tok]
+tokenize sourcename = totoks (initialPos sourcename)
-totoks :: (Line, Column) -> Text -> [Tok]
-totoks (lin,col) t =
+totoks :: SourcePos -> Text -> [Tok]
+totoks pos t =
case T.uncons t of
Nothing -> []
Just (c, rest)
| c == '\n' ->
- Tok (lin, col) Newline "\n"
- : totoks (lin + 1,1) rest
+ Tok pos Newline "\n"
+ : totoks (setSourceColumn (incSourceLine pos 1) 1) rest
| isSpaceOrTab c ->
let (sps, rest') = T.span isSpaceOrTab t
- in Tok (lin, col) Spaces sps
- : totoks (lin, col + T.length sps) rest'
+ in Tok pos Spaces sps
+ : totoks (incSourceColumn pos (T.length sps))
+ rest'
| isAlphaNum c ->
let (ws, rest') = T.span isAlphaNum t
- in Tok (lin, col) Word ws
- : totoks (lin, col + T.length ws) rest'
+ in Tok pos Word ws
+ : totoks (incSourceColumn pos (T.length ws)) rest'
| c == '%' ->
let (cs, rest') = T.break (== '\n') rest
- in Tok (lin, col) Comment ("%" <> cs)
- : totoks (lin, col + 1 + T.length cs) rest'
+ in Tok pos Comment ("%" <> cs)
+ : totoks (incSourceColumn pos (1 + T.length cs)) rest'
| c == '\\' ->
case T.uncons rest of
- Nothing -> [Tok (lin, col) Symbol (T.singleton c)]
+ Nothing -> [Tok pos Symbol (T.singleton c)]
Just (d, rest')
| isLetterOrAt d ->
-- \makeatletter is common in macro defs;
@@ -328,24 +331,24 @@ totoks (lin,col) t =
-- probably best for now
let (ws, rest'') = T.span isLetterOrAt rest
(ss, rest''') = T.span isSpaceOrTab rest''
- in Tok (lin, col) (CtrlSeq ws) ("\\" <> ws <> ss)
- : totoks (lin,
- col + 1 + T.length ws + T.length ss) rest'''
+ in Tok pos (CtrlSeq ws) ("\\" <> ws <> ss)
+ : totoks (incSourceColumn pos
+ (1 + T.length ws + T.length ss)) rest'''
| d == '\t' || d == '\n' ->
- Tok (lin, col) Symbol ("\\")
- : totoks (lin, col + 1) rest
+ Tok pos Symbol ("\\")
+ : totoks (incSourceColumn pos 1) rest
| otherwise ->
- Tok (lin, col) (CtrlSeq (T.singleton d)) (T.pack [c,d])
- : totoks (lin, col + 2) rest'
+ Tok pos (CtrlSeq (T.singleton d)) (T.pack [c,d])
+ : totoks (incSourceColumn pos 2) rest'
| c == '#' ->
let (t1, t2) = T.span (\d -> d >= '0' && d <= '9') rest
in case safeRead (T.unpack t1) of
Just i ->
- Tok (lin, col) (Arg i) ("#" <> t1)
- : totoks (lin, col + 1 + T.length t1) t2
+ Tok pos (Arg i) ("#" <> t1)
+ : totoks (incSourceColumn pos (1 + T.length t1)) t2
Nothing ->
- Tok (lin, col) Symbol ("#")
- : totoks (lin, col + 1) t2
+ Tok pos Symbol ("#")
+ : totoks (incSourceColumn pos 1) t2
| c == '^' ->
case T.uncons rest of
Just ('^', rest') ->
@@ -354,20 +357,20 @@ totoks (lin,col) t =
| isLowerHex d ->
case T.uncons rest'' of
Just (e, rest''') | isLowerHex e ->
- Tok (lin, col) Esc2 (T.pack ['^','^',d,e])
- : totoks (lin, col + 4) rest'''
+ Tok pos Esc2 (T.pack ['^','^',d,e])
+ : totoks (incSourceColumn pos 4) rest'''
_ ->
- Tok (lin, col) Esc1 (T.pack ['^','^',d])
- : totoks (lin, col + 3) rest''
+ Tok pos Esc1 (T.pack ['^','^',d])
+ : totoks (incSourceColumn pos 3) rest''
| d < '\128' ->
- Tok (lin, col) Esc1 (T.pack ['^','^',d])
- : totoks (lin, col + 3) rest''
- _ -> [Tok (lin, col) Symbol ("^"),
- Tok (lin, col + 1) Symbol ("^")]
- _ -> Tok (lin, col) Symbol ("^")
- : totoks (lin, col + 1) rest
+ Tok pos Esc1 (T.pack ['^','^',d])
+ : totoks (incSourceColumn pos 3) rest''
+ _ -> [Tok pos Symbol ("^"),
+ Tok (incSourceColumn pos 1) Symbol ("^")]
+ _ -> Tok pos Symbol ("^")
+ : totoks (incSourceColumn pos 1) rest
| otherwise ->
- Tok (lin, col) Symbol (T.singleton c) : totoks (lin, col + 1) rest
+ Tok pos Symbol (T.singleton c) : totoks (incSourceColumn pos 1) rest
where isSpaceOrTab ' ' = True
isSpaceOrTab '\t' = True
@@ -393,8 +396,7 @@ satisfyTok f =
where matcher t | f t = Just t
| otherwise = Nothing
updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos
- updatePos spos _ (Tok (lin,col) _ _ : _) =
- setSourceColumn (setSourceLine spos lin) col
+ updatePos _spos _ (Tok pos _ _ : _) = pos
updatePos spos _ [] = spos
doMacros :: PandocMonad m => Int -> LP m ()
@@ -437,7 +439,7 @@ doMacros n = do
else doMacros (n + 1)
ExpandWhenDefined -> return ()
-setpos :: (Line, Column) -> Tok -> Tok
+setpos :: SourcePos -> Tok -> Tok
setpos spos (Tok _ tt txt) = Tok spos tt txt
anyControlSeq :: PandocMonad m => LP m Tok
@@ -728,15 +730,15 @@ doverb = do
verbTok :: PandocMonad m => Char -> LP m Tok
verbTok stopchar = do
- t@(Tok (lin, col) toktype txt) <- satisfyTok (not . isNewlineTok)
+ t@(Tok pos toktype txt) <- satisfyTok (not . isNewlineTok)
case T.findIndex (== stopchar) txt of
Nothing -> return t
Just i -> do
let (t1, t2) = T.splitAt i txt
inp <- getInput
- setInput $ Tok (lin, col + i) Symbol (T.singleton stopchar)
- : (totoks (lin, col + i + 1) (T.drop 1 t2)) ++ inp
- return $ Tok (lin, col) toktype t1
+ setInput $ Tok (incSourceColumn pos i) Symbol (T.singleton stopchar)
+ : (totoks (incSourceColumn pos (i + 1)) (T.drop 1 t2)) ++ inp
+ return $ Tok pos toktype t1
dolstinline :: PandocMonad m => LP m Inlines
dolstinline = do
@@ -1117,16 +1119,16 @@ tok = grouped inline <|> inlineCommand' <|> singleChar'
singleChar :: PandocMonad m => LP m Tok
singleChar = try $ do
- Tok (lin,col) toktype t <- satisfyTok (tokTypeIn [Word, Symbol])
+ Tok pos toktype t <- satisfyTok (tokTypeIn [Word, Symbol])
guard $ not $ toktype == Symbol &&
T.any (`Set.member` specialChars) t
if T.length t > 1
then do
let (t1, t2) = (T.take 1 t, T.drop 1 t)
inp <- getInput
- setInput $ (Tok (lin, col + 1) toktype t2) : inp
- return $ Tok (lin,col) toktype t1
- else return $ Tok (lin,col) toktype t
+ setInput $ (Tok (incSourceColumn pos 1) toktype t2) : inp
+ return $ Tok pos toktype t1
+ else return $ Tok pos toktype t
opt :: PandocMonad m => LP m Inlines
opt = bracketed inline
@@ -1159,7 +1161,7 @@ withRaw :: PandocMonad m => LP m a -> LP m (a, [Tok])
withRaw parser = do
inp <- getInput
result <- parser
- nxt <- option (Tok (0,0) Word "") (lookAhead anyTok)
+ nxt <- option (Tok (initialPos "source") Word "") (lookAhead anyTok)
let raw = takeWhile (/= nxt) inp
return (result, raw)
@@ -1739,7 +1741,27 @@ include = do
then map (maybeAddExtension ".sty") fs
else map (maybeAddExtension ".tex") fs
dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"
- mconcat <$> mapM (insertIncludedFile blocks (tokenize . T.pack) dirs) fs'
+ mapM_ (insertIncluded dirs) fs'
+ return mempty
+
+insertIncluded :: PandocMonad m
+ => [FilePath]
+ -> FilePath
+ -> LP m ()
+insertIncluded dirs f = do
+ pos <- getPosition
+ containers <- getIncludeFiles <$> getState
+ when (f `elem` containers) $ do
+ throwError $ PandocParseError $ "Include file loop at " ++ show pos
+ updateState $ addIncludeFile f
+ mbcontents <- readFileFromDirs dirs f
+ contents <- case mbcontents of
+ Just s -> return s
+ Nothing -> do
+ report $ CouldNotLoadIncludeFile f pos
+ return ""
+ getInput >>= setInput . (tokenize f (T.pack contents) ++)
+ updateState dropLatestIncludeFile
maybeAddExtension :: String -> FilePath -> FilePath
maybeAddExtension ext fp =
@@ -2394,9 +2416,7 @@ parseTableRow envname prefsufs = do
>> anyTok)
suffpos <- getPosition
option [] (count 1 amp)
- return $ map (setpos (sourceLine prefpos, sourceColumn prefpos)) pref
- ++ contents ++
- map (setpos (sourceLine suffpos, sourceColumn suffpos)) suff
+ return $ map (setpos prefpos) pref ++ contents ++ map (setpos suffpos) suff
rawcells <- sequence (map celltoks prefsufs)
oldInput <- getInput
cells <- sequence $ map (\ts -> setInput ts >> parseTableCell) rawcells
diff --git a/src/Text/Pandoc/Readers/LaTeX/Types.hs b/src/Text/Pandoc/Readers/LaTeX/Types.hs
index 2bef3cb1a..9e441714d 100644
--- a/src/Text/Pandoc/Readers/LaTeX/Types.hs
+++ b/src/Text/Pandoc/Readers/LaTeX/Types.hs
@@ -31,17 +31,17 @@ module Text.Pandoc.Readers.LaTeX.Types ( Tok(..)
, TokType(..)
, Macro(..)
, ExpansionPoint(..)
- , Line
- , Column )
+ , SourcePos
+ )
where
import Data.Text (Text)
-import Text.Parsec.Pos (Line, Column)
+import Text.Parsec.Pos (SourcePos)
data TokType = CtrlSeq Text | Spaces | Newline | Symbol | Word | Comment |
Esc1 | Esc2 | Arg Int
deriving (Eq, Ord, Show)
-data Tok = Tok (Line, Column) TokType Text
+data Tok = Tok SourcePos TokType Text
deriving (Eq, Ord, Show)
data ExpansionPoint = ExpandWhenDefined | ExpandWhenUsed
diff --git a/test/command/3971.md b/test/command/3971.md
new file mode 100644
index 000000000..923609f4e
--- /dev/null
+++ b/test/command/3971.md
@@ -0,0 +1,9 @@
+```
+% TEXINPUTS=command pandoc -f latex -t native
+\documentclass{article}
+\include{3971b}
+\code{f}
+\end{document}
+^D
+[Para [Code ("",[],[]) "f"]]
+```
diff --git a/test/command/3971b.tex b/test/command/3971b.tex
new file mode 100644
index 000000000..0d1dc3e02
--- /dev/null
+++ b/test/command/3971b.tex
@@ -0,0 +1,2 @@
+\newcommand{\code}[1]{\texttt{#1}}
+\begin{document}