diff --git a/modules/services/gpg-agent.nix b/modules/services/gpg-agent.nix
index 7af58c8df..0f7b82639 100644
--- a/modules/services/gpg-agent.nix
+++ b/modules/services/gpg-agent.nix
@@ -280,8 +280,11 @@ in {
           "pinentry-program ${lib.getExe cfg.pinentryPackage}"
           ++ [ cfg.extraConfig ]);
 
+      # Make sure we export GnuPG agent socket for SSH
+      # https://www.gnupg.org/documentation/manuals/gnupg/Agent-Examples.html#Agent-Examples
       home.sessionVariablesExtra = optionalString cfg.enableSshSupport ''
-        if [[ -z "$SSH_AUTH_SOCK" ]]; then
+        unset SSH_AGENT_PID
+        if [ "''${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then
           export SSH_AUTH_SOCK="$(${gpgPkg}/bin/gpgconf --list-dirs agent-ssh-socket)"
         fi
       '';