Issue #28164: Correctly handle special console filenames (patch by Eryk Sun)

Lib/test/test_winconsoleio.py

 '''Tests for WindowsConsoleIO
 '''
 
+import os
 import io
-import unittest
 import sys
+import unittest
+import tempfile
 
 if sys.platform != 'win32':
     raise unittest.SkipTest("test only relevant on win32")
@@ -19,6 +21,16 @@ class WindowsConsoleIOTests(unittest.TestCase):
         self.assertFalse(issubclass(ConIO, io.TextIOBase))
 
     def test_open_fd(self):
+        self.assertRaisesRegex(ValueError,
+            "negative file descriptor", ConIO, -1)
+
+        fd, _ = tempfile.mkstemp()
+        try:
+            self.assertRaisesRegex(ValueError,
+                "Cannot open non-console file", ConIO, fd)
+        finally:
+            os.close(fd)
+
         try:
             f = ConIO(0)
         except ValueError:
@@ -56,6 +68,20 @@ class WindowsConsoleIOTests(unittest.TestCase):
             f.close()
 
     def test_open_name(self):
+        self.assertRaises(ValueError, ConIO, sys.executable)
+
+        f = open('C:/con', 'rb', buffering=0)
+        self.assertIsInstance(f, ConIO)
+        f.close()
+
+        f = open(r'\\.\conin$', 'rb', buffering=0)
+        self.assertIsInstance(f, ConIO)
+        f.close()
+
+        f = open('//?/conout$', 'wb', buffering=0)
+        self.assertIsInstance(f, ConIO)
+        f.close()
+
         f = ConIO("CON")
         self.assertTrue(f.readable())
         self.assertFalse(f.writable())

Misc/NEWS

@@ -146,6 +146,8 @@ Library
 Windows
 -------
 
+- Issue #28164: Correctly handle special console filenames (patch by Eryk Sun)
+
 - Issue #29409: Implement PEP 529 for io.FileIO (Patch by Eryk Sun)
 
 - Issue #29392: Prevent crash when passing invalid arguments into msvcrt module.

Modules/_io/winconsoleio.c

@@ -60,51 +60,68 @@ char _get_console_type(HANDLE handle) {
 }
 
 char _PyIO_get_console_type(PyObject *path_or_fd) {
-    int fd;
-
-    fd = PyLong_AsLong(path_or_fd);
+    int fd = PyLong_AsLong(path_or_fd);
     PyErr_Clear();
     if (fd >= 0) {
         HANDLE handle;
         _Py_BEGIN_SUPPRESS_IPH
         handle = (HANDLE)_get_osfhandle(fd);
         _Py_END_SUPPRESS_IPH
-        if (!handle)
+        if (handle == INVALID_HANDLE_VALUE)
             return '\0';
         return _get_console_type(handle);
     }
 
-    PyObject *decoded, *decoded_upper;
+    PyObject *decoded;
+    wchar_t *decoded_wstr;
 
-    int d = PyUnicode_FSDecoder(path_or_fd, &decoded);
-    if (!d) {
+    if (!PyUnicode_FSDecoder(path_or_fd, &decoded)) {
         PyErr_Clear();
         return '\0';
     }
-    if (!PyUnicode_Check(decoded)) {
-        Py_CLEAR(decoded);
-        return '\0';
-    }
-    decoded_upper = PyObject_CallMethod(decoded, "upper", "");
+    decoded_wstr = PyUnicode_AsWideCharString(decoded, NULL);
     Py_CLEAR(decoded);
-    if (!decoded_upper) {
+    if (!decoded_wstr) {
         PyErr_Clear();
         return '\0';
     }
 
+    DWORD length;
+    wchar_t name_buf[MAX_PATH], *pname_buf = name_buf;
+    
+    length = GetFullPathNameW(decoded_wstr, MAX_PATH, pname_buf, NULL);
+    if (length > MAX_PATH) {
+        pname_buf = PyMem_New(wchar_t, length);
+        if (pname_buf)
+            length = GetFullPathNameW(decoded_wstr, length, pname_buf, NULL);
+        else
+            length = 0;
+    }
+    PyMem_Free(decoded_wstr);
+
     char m = '\0';
-    if (_PyUnicode_EqualToASCIIString(decoded_upper, "CONIN$")) {
-        m = 'r';
-    } else if (_PyUnicode_EqualToASCIIString(decoded_upper, "CONOUT$")) {
-        m = 'w';
-    } else if (_PyUnicode_EqualToASCIIString(decoded_upper, "CON")) {
-        m = 'x';
+    if (length) {
+        wchar_t *name = pname_buf;
+        if (length >= 4 && name[3] == L'\\' &&
+            (name[2] == L'.' || name[2] == L'?') &&
+            name[1] == L'\\' && name[0] == L'\\') {
+            name += 4;
+        }
+        if (!_wcsicmp(name, L"CONIN$")) {
+            m = 'r';
+        } else if (!_wcsicmp(name, L"CONOUT$")) {
+            m = 'w';
+        } else if (!_wcsicmp(name, L"CON")) {
+            m = 'x';
+        }
     }
 
-    Py_CLEAR(decoded_upper);
+    if (pname_buf != name_buf)
+        PyMem_Free(pname_buf);
     return m;
 }
 
+
 /*[clinic input]
 module _io
 class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type"
@@ -289,6 +306,11 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
         Py_CLEAR(decodedname);
         if (name == NULL)
             return -1;
+        if (console_type == '\0') {
+            PyErr_SetString(PyExc_ValueError,
+                "Cannot open non-console file");
+            return -1;
+        }
 
         if (wcslen(name) != length) {
             PyMem_Free(name);
@@ -370,6 +392,11 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
     if (console_type == '\0')
         console_type = _get_console_type(self->handle);
 
+    if (console_type == '\0') {
+        PyErr_SetString(PyExc_ValueError,
+            "Cannot open non-console file");
+        goto error;
+    }    
     if (self->writable && console_type != 'w') {
         PyErr_SetString(PyExc_ValueError,
             "Cannot open console input buffer for writing");