diff --git a/modules/home-environment.nix b/modules/home-environment.nix
index bd2fb6ffa..e88ea95d4 100644
--- a/modules/home-environment.nix
+++ b/modules/home-environment.nix
@@ -315,16 +315,20 @@ in
             else
               echo "Checking $targetPath  gone (deleting)"
               $DRY_RUN_CMD rm $VERBOSE_ARG "$targetPath"
-              targetDir="$(dirname "$targetPath")"
 
-              # Recursively remove the containing directory. We only
-              # do this if the containing folder is not $HOME since
-              # running rmdir on $HOME will result in a harmless but
-              # unpleasant error message.
-              if [[ "$targetDir" != "$HOME" ]] ; then
+              # Recursively delete empty parent directories.
+              targetDir="$(dirname "$relativePath")"
+              if [[ "$targetDir" != "." ]] ; then
+                pushd "$HOME" > /dev/null
+
+                # Call rmdir with a relative path excluding $HOME.
+                # Otherwise, it might try to delete $HOME and exit
+                # with a permission error.
                 $DRY_RUN_CMD rmdir $VERBOSE_ARG \
                     -p --ignore-fail-on-non-empty \
                     "$targetDir"
+
+                popd > /dev/null
               fi
             fi
           done