diff --git a/desktop_version/src/main.cpp b/desktop_version/src/main.cpp
index 52674f0a..c947536f 100644
--- a/desktop_version/src/main.cpp
+++ b/desktop_version/src/main.cpp
@@ -559,6 +559,9 @@ int main(int argc, char *argv[])
 
     SDL_SetHintWithPriority(SDL_HINT_IME_SHOW_UI, "1", SDL_HINT_OVERRIDE);
 
+    /* We already do the button swapping in ButtonGlyphs, disable SDL's swapping */
+    SDL_SetHintWithPriority(SDL_HINT_GAMECONTROLLER_USE_BUTTON_LABELS, "0", SDL_HINT_OVERRIDE);
+
     if(!FILESYSTEM_init(argv[0], baseDir, assetsPath, langDir, fontsDir))
     {
         vlog_error("Unable to initialize filesystem!");