Add support for Windows using "mbcs" as the default Unicode encoding when...

Add support for Windows using "mbcs" as the default Unicode encoding when dealing with the file system.  As discussed on python-dev and in patch 410465.

Lib/ntpath.py

@@ -404,21 +404,12 @@ def normpath(path):
 # Return an absolute path.
 def abspath(path):
     """Return the absolute version of a path"""
-    try:
-        import win32api
-    except ImportError:
-        global abspath
-        def _abspath(path):
-            if not isabs(path):
-                path = join(os.getcwd(), path)
-            return normpath(path)
-        abspath = _abspath
-        return _abspath(path)
     if path: # Empty path must return current working directory.
+        from nt import _getfullpathname
         try:
-            path = win32api.GetFullPathName(path)
-        except win32api.error:
-            pass # Bad path - return unchanged.
+            path = _getfullpathname(path)
+        except WindowsError:
+                pass # Bad path - return unchanged.
     else:
         path = os.getcwd()
     return normpath(path)

Lib/test/output/test_unicode_file

+test_unicode_file
+All the Unicode tests appeared to work

Lib/test/test_support.py

@@ -63,6 +63,10 @@ if os.name == 'java':
     TESTFN = '$test'
 elif os.name != 'riscos':
     TESTFN = '@test'
+    # Unicode name only used if TEST_FN_ENCODING exists for the platform.
+    TESTFN_UNICODE=u"@test-\xe0\xf2" # 2 latin characters.
+    if os.name=="nt":
+        TESTFN_ENCODING="mbcs"
 else:
     TESTFN = 'test'
 del os

Lib/test/test_unicode_file.py

+# Test some Unicode file name semantics
+# We dont test many operations on files other than
+# that their names can be used with Unicode characters.
+import os
+
+from test_support import verify, TestSkipped, TESTFN_UNICODE
+try:
+    from test_support import TESTFN_ENCODING
+except ImportError:
+    raise TestSkipped("No Unicode filesystem semantics on this platform.")
+
+TESTFN_ENCODED = TESTFN_UNICODE.encode(TESTFN_ENCODING)
+
+# Check with creation as Unicode string.
+f = open(TESTFN_UNICODE, 'wb')
+if not os.path.isfile(TESTFN_UNICODE):
+    print "File doesn't exist after creating it"
+
+if not os.path.isfile(TESTFN_ENCODED):
+    print "File doesn't exist (encoded string) after creating it"
+
+f.close()
+
+# Test stat and chmod
+if os.stat(TESTFN_ENCODED) != os.stat(TESTFN_UNICODE):
+    print "os.stat() did not agree on the 2 filenames"
+os.chmod(TESTFN_ENCODED, 0777)
+os.chmod(TESTFN_UNICODE, 0777)
+
+# Test rename
+os.rename(TESTFN_ENCODED, TESTFN_ENCODED + ".new")
+os.rename(TESTFN_UNICODE+".new", TESTFN_ENCODED)
+
+os.unlink(TESTFN_ENCODED)
+if os.path.isfile(TESTFN_ENCODED) or \
+   os.path.isfile(TESTFN_UNICODE):
+    print "File exists after deleting it"
+
+# Check with creation as encoded string.
+f = open(TESTFN_ENCODED, 'wb')
+if not os.path.isfile(TESTFN_UNICODE) or \
+   not os.path.isfile(TESTFN_ENCODED):
+    print "File doesn't exist after creating it"
+
+path, base = os.path.split(os.path.abspath(TESTFN_ENCODED))
+if base not in os.listdir(path):
+    print "Filename did not appear in os.listdir()"
+
+f.close()
+os.unlink(TESTFN_UNICODE)
+if os.path.isfile(TESTFN_ENCODED) or \
+   os.path.isfile(TESTFN_UNICODE):
+    print "File exists after deleting it"
+
+# test os.open
+f = os.open(TESTFN_ENCODED, os.O_CREAT)
+if not os.path.isfile(TESTFN_UNICODE) or \
+   not os.path.isfile(TESTFN_ENCODED):
+    print "File doesn't exist after creating it"
+os.close(f)
+os.unlink(TESTFN_UNICODE)
+
+# Test directories etc
+cwd = os.getcwd()
+abs_encoded = os.path.abspath(TESTFN_ENCODED) + ".dir"
+abs_unicode = os.path.abspath(TESTFN_UNICODE) + ".dir"
+os.mkdir(abs_encoded)
+try:
+    os.chdir(abs_encoded)
+    os.chdir(abs_unicode)
+finally:
+    os.chdir(cwd)
+    os.rmdir(abs_unicode)
+os.mkdir(abs_unicode)
+try:
+    os.chdir(abs_encoded)
+    os.chdir(abs_unicode)
+finally:
+    os.chdir(cwd)
+    os.rmdir(abs_encoded)
+print "All the Unicode tests appeared to work"

Python/bltinmodule.c

@@ -13,6 +13,8 @@
 #include <unistd.h>
 #endif
 
+extern const char *Py_FileSystemDefaultEncoding;
+
 /* Forward */
 static PyObject *filterstring(PyObject *, PyObject *);
 static PyObject *filtertuple (PyObject *, PyObject *);
@@ -1530,14 +1532,16 @@ Return the octal representation of an integer or long integer.";
 static PyObject *
 builtin_open(PyObject *self, PyObject *args)
 {
-	char *name;
+	char *name = NULL;
 	char *mode = "r";
 	int bufsize = -1;
 	PyObject *f;
 
-	if (!PyArg_ParseTuple(args, "s|si:open", &name, &mode, &bufsize))
+	if (!PyArg_ParseTuple(args, "et|si:open", Py_FileSystemDefaultEncoding, 
+	                      &name, &mode, &bufsize))
 		return NULL;
 	f = PyFile_FromString(name, mode);
+	PyMem_Free(name); /* free the encoded string */
 	if (f != NULL)
 		PyFile_SetBufSize(f, bufsize);
 	return f;

Python/getargs.c

@@ -698,7 +698,7 @@ convertsimple1(PyObject *arg, char **p_format, va_list *p_va)
 			     's' (recode all objects via Unicode) or
 			     't' (only recode non-string objects) 
 			*/
-			if (*format != 's')
+			if (*format == 's')
 			    	recode_strings = 1;
 			else if (*format == 't')
 			    	recode_strings = 0;