From 86d90a1296739adef30b224f41e3a6ab55069a48 Mon Sep 17 00:00:00 2001
From: Misa <infoteddy@infoteddy.info>
Date: Mon, 14 Nov 2022 21:57:01 -0800
Subject: [PATCH] Add color support to Windows console output, properly

This adds color support to the output of the console on Windows. Now if
you're using Windows 10 build 1511 or later (I think it's build 1511
anyway; they added more VT sequence support in later versions), you will
see colors by default. This isn't due to Windows helping in any way;
this commit has to specifically enable it with SetConsoleMode() because
by default, Windows won't enable color support unless we enable it. (Or
if it's enabled in the registry, but having to go through the registry
to enable basic shit like that is completely fucking stupid.)

I tested this in my Windows 10 virtual machine and it's completely
working.
---
 desktop_version/src/Vlogging.c | 64 ++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 14 deletions(-)

diff --git a/desktop_version/src/Vlogging.c b/desktop_version/src/Vlogging.c
index a88b9616..a867228e 100644
--- a/desktop_version/src/Vlogging.c
+++ b/desktop_version/src/Vlogging.c
@@ -5,15 +5,8 @@
 #ifdef _WIN32
 #   define WIN32_LEAN_AND_MEAN
 #   include <windows.h>
-#   define STDOUT_IS_TTY 0
-#   define STDERR_IS_TTY 0
 #elif defined(__unix__) || defined(__APPLE__)
 #   include <unistd.h>
-#   define STDOUT_IS_TTY isatty(STDOUT_FILENO)
-#   define STDERR_IS_TTY isatty(STDERR_FILENO)
-#else
-#   define STDOUT_IS_TTY 0
-#   define STDERR_IS_TTY 0
 #endif
 
 #define COLOR(EXPR) (color_enabled && color_supported ? EXPR : "")
@@ -32,13 +25,7 @@ static int info_enabled = 1;
 static int warn_enabled = 1;
 static int error_enabled = 1;
 
-static void check_color_support(void)
-{
-    if (STDOUT_IS_TTY && STDERR_IS_TTY)
-    {
-        color_supported = 1;
-    }
-}
+static void check_color_support(void);
 
 void vlog_init(void)
 {
@@ -196,3 +183,52 @@ void vlog_open_console(void)
     check_color_support();
 }
 #endif /* _WIN32 */
+
+static void check_color_support(void)
+{
+#ifdef _WIN32
+    /* VT100 colors are supported since Windows 10 build 16257,
+     * but it's not enabled by default. So we have to set it. */
+
+    const HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
+    if (hStdout == INVALID_HANDLE_VALUE)
+    {
+        vlog_error(
+            "Could not set color support: GetStdHandle() failed with %d",
+            GetLastError()
+        );
+        return;
+    }
+
+    const HANDLE hStderr = GetStdHandle(STD_ERROR_HANDLE);
+    if (hStderr == INVALID_HANDLE_VALUE)
+    {
+        vlog_error(
+            "Could not enable color support: GetStdHandle() failed with %d",
+            GetLastError()
+        );
+        return;
+    }
+
+    const BOOL success = SetConsoleMode(
+        hStdout, ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING
+    );
+    if (!success)
+    {
+        /* Debug, not error, because it might not be an error.
+         * (E.g. this version of Windows doesn't support VT100 colors.) */
+        vlog_debug(
+            "Could not enable color support: SetConsoleMode() failed with %d",
+            GetLastError()
+        );
+        return;
+    }
+
+    color_supported = 1;
+#elif defined(__unix__) || defined(__APPLE__)
+    if (isatty(STDOUT_FILENO) && isatty(STDERR_FILENO))
+    {
+        color_supported = 1;
+    }
+#endif
+}