diff --git a/MANUAL.txt b/MANUAL.txt
index 2ca50e44a..d755c0c00 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -5080,7 +5080,11 @@ Without this extension, you would have to use
 `![image](chap1/spider.jpg)` in `chap1/text.md` and
 `![image](chap2/spider.jpg)` in `chap2/text.md`.  Links with
 relative paths will be rewritten in the same way as images.
-*This option currently only affects Markdown input.*
+
+Note that relative paths in reference links and images will
+be rewritten relative to the file containing the link
+reference definition, not the file containing the reference link
+or image itself, if these differ.
 
 #### Extension: `attributes` ####
 
diff --git a/src/Text/Pandoc/Readers/Markdown.hs b/src/Text/Pandoc/Readers/Markdown.hs
index 968c6c165..e5cbadc94 100644
--- a/src/Text/Pandoc/Readers/Markdown.hs
+++ b/src/Text/Pandoc/Readers/Markdown.hs
@@ -336,7 +336,9 @@ referenceKey = try $ do
                     notFollowedBy' (() <$ reference)
                     many1Char $ notFollowedBy space >> litChar
   let betweenAngles = try $ char '<' >> manyTillChar litChar (char '>')
-  src <- try betweenAngles <|> sourceURL
+  rebase <- option False (True <$ guardEnabled Ext_rebase_relative_paths)
+  src <- (if rebase then rebasePath pos else id) <$>
+             (try betweenAngles <|> sourceURL)
   tit <- option "" referenceTitle
   attr   <- option nullAttr $ try $
               do guardEnabled Ext_link_attributes
@@ -1857,8 +1859,6 @@ referenceLink constructor (lab, raw) = do
                          return (mempty, "")))
       <|>
       try ((guardDisabled Ext_spaced_reference_links <|> spnl) >> reference)
-  rebase <- option False (True <$ guardEnabled Ext_rebase_relative_paths)
-  pos <- getPosition
   when (raw' == "") $ guardEnabled Ext_shortcut_reference_links
   let labIsRef = raw' == "" || raw' == "[]"
   let key = toKey $ if labIsRef then raw else raw'
@@ -1884,8 +1884,7 @@ referenceLink constructor (lab, raw) = do
                    Nothing              -> makeFallback
             else makeFallback
        Just ((src,tit), attr) ->
-          let src' = if rebase then rebasePath pos src else src
-           in constructor attr src' tit <$> lab
+           constructor attr src tit <$> lab
 
 dropBrackets :: Text -> Text
 dropBrackets = dropRB . dropLB
diff --git a/test/command/3752.md b/test/command/3752.md
index 76d51989b..6d31eb450 100644
--- a/test/command/3752.md
+++ b/test/command/3752.md
@@ -2,12 +2,14 @@
 % pandoc command/chap1/text.md command/chap2/text.md -f markdown+rebase_relative_paths --verbose -t docx | pandoc -f docx -t plain
 ^D
 [INFO] Loaded command/chap1/spider.png from ./command/chap1/spider.png
-[INFO] Loaded command/chap1/../../lalune.jpg from ./command/chap1/../../lalune.jpg
 [INFO] Loaded command/chap2/spider.png from ./command/chap2/spider.png
+[INFO] Loaded command/chap1/../../lalune.jpg from ./command/chap1/../../lalune.jpg
 Chapter one
 
 A spider: [spider]
 
+Another spider: [another spider]
+
 The moon: [moon]
 
 Link to spider picture.
@@ -26,6 +28,7 @@ A spider: [spider]
 ^D
 <h1 id="chapter-one">Chapter one</h1>
 <p>A spider: <img src="command/chap1/spider.png" alt="spider" /></p>
+<p>Another spider: <img src="command/chap2/spider.png" alt="another spider" /></p>
 <p>The moon: <img src="command/chap1/../../lalune.jpg" alt="moon" /></p>
 <p>Link to <a href="command/chap1/spider.png">spider picture</a>.</p>
 <p>URL left alone: <a href="https://pandoc.org/MANUAL.html">manual</a>.</p>
diff --git a/test/command/chap1/text.md b/test/command/chap1/text.md
index 88b30313d..d25514241 100644
--- a/test/command/chap1/text.md
+++ b/test/command/chap1/text.md
@@ -2,6 +2,8 @@
 
 A spider: ![spider](spider.png)
 
+Another spider: ![another spider][refspider]
+
 The moon: ![moon](../../lalune.jpg)
 
 Link to [spider picture](spider.png).
diff --git a/test/command/chap2/text.md b/test/command/chap2/text.md
index 435a266d7..082a1d79e 100644
--- a/test/command/chap2/text.md
+++ b/test/command/chap2/text.md
@@ -1,3 +1,5 @@
 # Chapter two
 
 A spider: ![spider](spider.png)
+
+[refspider]: spider.png