Add File.getOrEnableAnsiEscapeSupport and use it

On Windows, the console mode flag `ENABLE_VIRTUAL_TERMINAL_PROCESSING` determines whether or not ANSI escape codes are parsed/acted on. On the newer Windows Terminal, this flag is set by default, but on the older Windows Console, it is not set by default, but *can* be enabled (since Windows 10 RS1 from June 2016).

The new `File.getOrEnableAnsiEscapeSupport` function will get the current status of ANSI escape code support, but will also attempt to enable `ENABLE_VIRTUAL_TERMINAL_PROCESSING` on Windows if necessary which will provide better/more consistent results for things like `std.Progress` and `std.io.tty`.

This type of change was not done previously due to a mistaken assumption (on my part) that the console mode would persist after the run of a program. However, it turns out that the console mode is always reset to the default for each program run in a console session.

Author: squeek502

lib/std/Progress.zig

@@ -377,7 +377,7 @@ pub fn start(options: Options) Node {
             }
             const stderr = std.io.getStdErr();
             global_progress.terminal = stderr;
-            if (stderr.supportsAnsiEscapeCodes()) {
+            if (stderr.getOrEnableAnsiEscapeSupport()) {
                 global_progress.terminal_mode = .ansi_escape_codes;
             } else if (is_windows and stderr.isTty()) {
                 global_progress.terminal_mode = TerminalMode{ .windows_api = .{

lib/std/fs/File.zig

@@ -188,7 +188,7 @@ pub fn sync(self: File) SyncError!void {
 }
 
 /// Test whether the file refers to a terminal.
-/// See also `supportsAnsiEscapeCodes`.
+/// See also `getOrEnableAnsiEscapeSupport` and `supportsAnsiEscapeCodes`.
 pub fn isTty(self: File) bool {
     return posix.isatty(self.handle);
 }
@@ -245,8 +245,16 @@ pub fn isCygwinPty(file: File) bool {
         std.mem.indexOf(u16, name_wide, &[_]u16{ '-', 'p', 't', 'y' }) != null;
 }
 
-/// Test whether ANSI escape codes will be treated as such.
-pub fn supportsAnsiEscapeCodes(self: File) bool {
+/// Returns whether or not ANSI escape codes will be treated as such,
+/// and attempts to enable support for ANSI escape codes if necessary
+/// (on Windows).
+///
+/// Returns `true` if ANSI escape codes are supported or support was
+/// successfully enabled. Returns false if ANSI escape codes are not
+/// supported or support was unable to be enabled.
+///
+/// See also `supportsAnsiEscapeCodes`.
+pub fn getOrEnableAnsiEscapeSupport(self: File) bool {
     if (builtin.os.tag == .windows) {
         var original_console_mode: windows.DWORD = 0;
 
@@ -262,14 +270,31 @@ pub fn supportsAnsiEscapeCodes(self: File) bool {
             var console_mode = original_console_mode | requested_console_modes;
             if (windows.kernel32.SetConsoleMode(self.handle, console_mode) != 0) return true;
 
-            // An application receiving ERROR_INVALID_PARAMETER with one of the newer console mode flags in the bit field should gracefully degrade behavior and try again.
+            // An application receiving ERROR_INVALID_PARAMETER with one of the newer console mode
+            // flags in the bit field should gracefully degrade behavior and try again.
             requested_console_modes = windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING;
             console_mode = original_console_mode | requested_console_modes;
             if (windows.kernel32.SetConsoleMode(self.handle, console_mode) != 0) return true;
         }
 
         return self.isCygwinPty();
     }
+    return self.supportsAnsiEscapeCodes();
+}
+
+/// Test whether ANSI escape codes will be treated as such without
+/// attempting to enable support for ANSI escape codes.
+///
+/// See also `getOrEnableAnsiEscapeSupport`.
+pub fn supportsAnsiEscapeCodes(self: File) bool {
+    if (builtin.os.tag == .windows) {
+        var console_mode: windows.DWORD = 0;
+        if (windows.kernel32.GetConsoleMode(self.handle, &console_mode) != 0) {
+            if (console_mode & windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0) return true;
+        }
+
+        return self.isCygwinPty();
+    }
     if (builtin.os.tag == .wasi) {
         // WASI sanitizes stdout when fd is a tty so ANSI escape codes
         // will not be interpreted as actual cursor commands, and

lib/std/io/tty.zig

@@ -8,6 +8,7 @@ const native_os = builtin.os.tag;
 /// Detect suitable TTY configuration options for the given file (commonly stdout/stderr).
 /// This includes feature checks for ANSI escape codes and the Windows console API, as well as
 /// respecting the `NO_COLOR` and `CLICOLOR_FORCE` environment variables to override the default.
+/// Will attempt to enable ANSI escape code support if necessary/possible.
 pub fn detectConfig(file: File) Config {
     const force_color: ?bool = if (builtin.os.tag == .wasi)
         null // wasi does not support environment variables
@@ -20,7 +21,7 @@ pub fn detectConfig(file: File) Config {
 
     if (force_color == false) return .no_color;
 
-    if (file.supportsAnsiEscapeCodes()) return .escape_codes;
+    if (file.getOrEnableAnsiEscapeSupport()) return .escape_codes;
 
     if (native_os == .windows and file.isTty()) {
         var info: windows.CONSOLE_SCREEN_BUFFER_INFO = undefined;