diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index 52782653e..e1497dfb1 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -164,6 +164,7 @@ data LaTeXState = LaTeXState{ sOptions       :: ReaderOptions
                             , sInTableCell   :: Bool
                             , sLastHeaderNum :: HeaderNum
                             , sLabels        :: M.Map String [Inline]
+                            , sHasChapters   :: Bool
                             , sToggles       :: M.Map String Bool
                             }
      deriving Show
@@ -183,6 +184,7 @@ defaultLaTeXState = LaTeXState{ sOptions       = def
                               , sInTableCell   = False
                               , sLastHeaderNum = HeaderNum []
                               , sLabels        = M.empty
+                              , sHasChapters   = False
                               , sToggles       = M.empty
                               }
 
@@ -1984,9 +1986,13 @@ section starred (ident, classes, kvs) lvl = do
           try (spaces >> controlSeq "label"
                >> spaces >> toksToString <$> braced)
   let classes' = if starred then "unnumbered" : classes else classes
+  when (lvl == 0) $
+    updateState $ \st -> st{ sHasChapters = True }
   unless starred $ do
     hn <- sLastHeaderNum <$> getState
-    let num = incrementHeaderNum lvl hn
+    hasChapters <- sHasChapters <$> getState
+    let lvl' = lvl + if hasChapters then 1 else 0
+    let num = incrementHeaderNum lvl' hn
     updateState $ \st -> st{ sLastHeaderNum = num }
     updateState $ \st -> st{ sLabels = M.insert lab
                             [Str (renderHeaderNum num)]
diff --git a/test/command/4529.md b/test/command/4529.md
new file mode 100644
index 000000000..4a2125b9c
--- /dev/null
+++ b/test/command/4529.md
@@ -0,0 +1,36 @@
+```
+% pandoc -f latex -t plain
+\chapter{First chapter}\label{sec:chp1}
+The next chapter is Chapter~\ref{sec:chp2}.
+\section{First section}\label{sec:chp1sec1}
+The next section is Section~\ref{sec:chp2sec1}.
+
+\chapter{Second chapter}\label{sec:chp2}
+The previous chapter is Chapter~\ref{sec:chp1}.
+\section{First section}\label{sec:chp2sec1}
+The previous section is Section~\ref{sec:chp1sec1}.
+^D
+
+
+FIRST CHAPTER
+
+
+The next chapter is Chapter 2.
+
+
+First section
+
+The next section is Section 2.1.
+
+
+
+SECOND CHAPTER
+
+
+The previous chapter is Chapter 1.
+
+
+First section
+
+The previous section is Section 1.1.
+```