Reline: Use a more robust detection of MinTTY

The previous detection per get_screen_size fails when stdout is passed
to a pipe. That is the case when running ruby tests in parallel ("-j" switch).
In this case Reline believes that it's running on MinTTY and the tests
are running with ANSI IOGate instead of the Windows adapter on MINGW.
So parallel test results were different to that of a single process.
This commit fixes these differencies.

The code is taken from git sources and translated to ruby.
NtQueryObject() is replaced by GetFileInformationByHandleEx(), because
NtQueryObject() is undocumented and is more difficult to use:
  c5a03b1e29/compat/winansi.c (L558)

lib/reline.rb

@@ -415,8 +415,7 @@ end
 
 if RbConfig::CONFIG['host_os'] =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/
   require 'reline/windows'
-  if Reline::Windows.get_screen_size == [0, 0]
+  if Reline::Windows.msys_tty?
-    # Maybe Mintty on Cygwin
     require 'reline/ansi'
     Reline::IOGate = Reline::ANSI
   else

lib/reline/windows.rb

@@ -72,6 +72,8 @@ class Reline::Windows
   STD_INPUT_HANDLE = -10
   STD_OUTPUT_HANDLE = -11
   WINDOW_BUFFER_SIZE_EVENT = 0x04
+  FILE_TYPE_PIPE = 0x0003
+  FILE_NAME_INFO = 2
   @@getwch = Win32API.new('msvcrt', '_getwch', [], 'I')
   @@kbhit = Win32API.new('msvcrt', '_kbhit', [], 'I')
   @@GetKeyState = Win32API.new('user32', 'GetKeyState', ['L'], 'L')
@@ -84,9 +86,36 @@ class Reline::Windows
   @@hConsoleInputHandle = @@GetStdHandle.call(STD_INPUT_HANDLE)
   @@GetNumberOfConsoleInputEvents = Win32API.new('kernel32', 'GetNumberOfConsoleInputEvents', ['L', 'P'], 'L')
   @@ReadConsoleInput = Win32API.new('kernel32', 'ReadConsoleInput', ['L', 'P', 'L', 'P'], 'L')
+  @@GetFileType = Win32API.new('kernel32', 'GetFileType', ['L'], 'L')
+  @@GetFileInformationByHandleEx = Win32API.new('kernel32', 'GetFileInformationByHandleEx', ['L', 'I', 'P', 'L'], 'I')
+
   @@input_buf = []
   @@output_buf = []
 
+  def self.msys_tty?(io=@@hConsoleInputHandle)
+    # check if fd is a pipe
+    if @@GetFileType.call(io) != FILE_TYPE_PIPE
+      return false
+    end
+
+    bufsize = 1024
+    p_buffer = "\0" * bufsize
+    res = @@GetFileInformationByHandleEx.call(io, FILE_NAME_INFO, p_buffer, bufsize - 2)
+    return false if res == 0
+
+    # get pipe name: p_buffer layout is:
+    #   struct _FILE_NAME_INFO {
+    #     DWORD FileNameLength;
+    #     WCHAR FileName[1];
+    #   } FILE_NAME_INFO
+    len = p_buffer[0, 4].unpack("L")[0]
+    name = p_buffer[4, len].encode(Encoding::UTF_8, Encoding::UTF_16LE, invalid: :replace)
+
+    # Check if this could be a MSYS2 pty pipe ('\msys-XXXX-ptyN-XX')
+    # or a cygwin pty pipe ('\cygwin-XXXX-ptyN-XX')
+    name =~ /(msys-|cygwin-).*-pty/ ? true : false
+  end
+
   def self.getwch
     unless @@input_buf.empty?
       return @@input_buf.shift