Change command line processing API to use wchar_t.

Fixes #2128.

Doc/c-api/init.rst

@@ -161,7 +161,7 @@ Initialization, Finalization, and Threads
    haven't been explicitly destroyed at that point.
 
 
-.. cfunction:: void Py_SetProgramName(char *name)
+.. cfunction:: void Py_SetProgramName(wchar_t *name)
 
    .. index::
       single: Py_Initialize()
@@ -170,11 +170,12 @@ Initialization, Finalization, and Threads
 
    This function should be called before :cfunc:`Py_Initialize` is called for
    the first time, if it is called at all.  It tells the interpreter the value
-   of the ``argv[0]`` argument to the :cfunc:`main` function of the program.
+   of the ``argv[0]`` argument to the :cfunc:`main` function of the program
+   (converted to wide characters).
    This is used by :cfunc:`Py_GetPath` and some other functions below to find
    the Python run-time libraries relative to the interpreter executable.  The
    default value is ``'python'``.  The argument should point to a
-   zero-terminated character string in static storage whose contents will not
+   zero-terminated wide character string in static storage whose contents will not
    change for the duration of the program's execution.  No code in the Python
    interpreter will change the contents of this storage.
 
@@ -188,7 +189,7 @@ Initialization, Finalization, and Threads
    value.
 
 
-.. cfunction:: char* Py_GetPrefix()
+.. cfunction:: wchar_t* Py_GetPrefix()
 
    Return the *prefix* for installed platform-independent files. This is derived
    through a number of complicated rules from the program name set with
@@ -201,7 +202,7 @@ Initialization, Finalization, and Threads
    It is only useful on Unix.  See also the next function.
 
 
-.. cfunction:: char* Py_GetExecPrefix()
+.. cfunction:: wchar_t* Py_GetExecPrefix()
 
    Return the *exec-prefix* for installed platform-*dependent* files.  This is
    derived through a number of complicated rules from the program name set with
@@ -236,7 +237,7 @@ Initialization, Finalization, and Threads
    platform.
 
 
-.. cfunction:: char* Py_GetProgramFullPath()
+.. cfunction:: wchar_t* Py_GetProgramFullPath()
 
    .. index::
       single: Py_SetProgramName()
@@ -249,7 +250,7 @@ Initialization, Finalization, and Threads
    to Python code as ``sys.executable``.
 
 
-.. cfunction:: char* Py_GetPath()
+.. cfunction:: wchar_t* Py_GetPath()
 
    .. index::
       triple: module; search; path
@@ -342,7 +343,7 @@ Initialization, Finalization, and Threads
    ``sys.version``.
 
 
-.. cfunction:: void PySys_SetArgv(int argc, char **argv)
+.. cfunction:: void PySys_SetArgv(int argc, wchar_t **argv)
 
    .. index::
       single: main()

Doc/c-api/sys.rst

@@ -84,11 +84,11 @@ accessible to C code.  They all work with the current interpreter thread's
 
    Reset :data:`sys.warnoptions` to an empty list.
 
-.. cfunction:: void PySys_AddWarnOption(char *s)
+.. cfunction:: void PySys_AddWarnOption(wchar_t *s)
 
    Append *s* to :data:`sys.warnoptions`.
 
-.. cfunction:: void PySys_SetPath(char *path)
+.. cfunction:: void PySys_SetPath(wchar_t *path)
 
    Set :data:`sys.path` to a list object of paths found in *path* which should
    be a list of paths separated with the platform's search path delimiter

Doc/c-api/unicode.rst

@@ -336,6 +336,8 @@ the system's :ctype:`wchar_t`.
 .. cfunction:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size)
 
    Create a Unicode object from the :ctype:`wchar_t` buffer *w* of the given size.
+   Passing -1 as the size indicates that the function must itself compute the length,
+   using wcslen.
    Return *NULL* on failure.
 
 

Doc/c-api/veryhigh.rst

@@ -25,16 +25,18 @@ are only passed to these functions if it is certain that they were created by
 the same library that the Python runtime is using.
 
 
-.. cfunction:: int Py_Main(int argc, char **argv)
-
-   The main program for the standard interpreter.  This is made available for
-   programs which embed Python.  The *argc* and *argv* parameters should be
-   prepared exactly as those which are passed to a C program's :cfunc:`main`
-   function.  It is important to note that the argument list may be modified (but
-   the contents of the strings pointed to by the argument list are not). The return
-   value will be the integer passed to the :func:`sys.exit` function, ``1`` if the
-   interpreter exits due to an exception, or ``2`` if the parameter list does not
-   represent a valid Python command line.
+.. cfunction:: int Py_Main(int argc, wchar_t **argv)
+
+   The main program for the standard interpreter.  This is made
+   available for programs which embed Python.  The *argc* and *argv*
+   parameters should be prepared exactly as those which are passed to
+   a C program's :cfunc:`main` function (converted to wchar_t
+   according to the user's locale).  It is important to note that the
+   argument list may be modified (but the contents of the strings
+   pointed to by the argument list are not). The return value will be
+   the integer passed to the :func:`sys.exit` function, ``1`` if the
+   interpreter exits due to an exception, or ``2`` if the parameter
+   list does not represent a valid Python command line.
 
 
 .. cfunction:: int PyRun_AnyFile(FILE *fp, const char *filename)

Include/osdefs.h

@@ -12,20 +12,20 @@ extern "C" {
 #if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__) || defined(PYOS_OS2)
 #if defined(PYOS_OS2) && defined(PYCC_GCC)
 #define MAXPATHLEN 260
-#define SEP '/'
-#define ALTSEP '\\'
+#define SEP L'/'
+#define ALTSEP L'\\'
 #else
-#define SEP '\\'
-#define ALTSEP '/'
+#define SEP L'\\'
+#define ALTSEP L'/'
 #define MAXPATHLEN 256
 #endif
-#define DELIM ';'
+#define DELIM L';'
 #endif
 #endif
 
 /* Filename separator */
 #ifndef SEP
-#define SEP '/'
+#define SEP L'/'
 #endif
 
 /* Max pathname length */
@@ -39,7 +39,7 @@ extern "C" {
 
 /* Search path entry delimiter */
 #ifndef DELIM
-#define DELIM ':'
+#define DELIM L':'
 #endif
 
 #ifdef __cplusplus

Include/pygetopt.h

@@ -7,9 +7,9 @@ extern "C" {
 
 PyAPI_DATA(int) _PyOS_opterr;
 PyAPI_DATA(int) _PyOS_optind;
-PyAPI_DATA(char *) _PyOS_optarg;
+PyAPI_DATA(wchar_t *) _PyOS_optarg;
 
-PyAPI_FUNC(int) _PyOS_GetOpt(int argc, char **argv, char *optstring);
+PyAPI_FUNC(int) _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring);
 
 #ifdef __cplusplus
 }

Include/pythonrun.h

@@ -17,11 +17,11 @@ typedef struct {
 	int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
 } PyCompilerFlags;
 
-PyAPI_FUNC(void) Py_SetProgramName(char *);
-PyAPI_FUNC(char *) Py_GetProgramName(void);
+PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
+PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
 
-PyAPI_FUNC(void) Py_SetPythonHome(char *);
-PyAPI_FUNC(char *) Py_GetPythonHome(void);
+PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
+PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
 
 PyAPI_FUNC(void) Py_Initialize(void);
 PyAPI_FUNC(void) Py_InitializeEx(int);
@@ -81,7 +81,7 @@ PyAPI_FUNC(void) Py_Exit(int);
 PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
 
 /* Bootstrap */
-PyAPI_FUNC(int) Py_Main(int argc, char **argv);
+PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
 
 /* Use macros for a bunch of old variants */
 #define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
@@ -103,10 +103,10 @@ PyAPI_FUNC(int) Py_Main(int argc, char **argv);
         PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
 
 /* In getpath.c */
-PyAPI_FUNC(char *) Py_GetProgramFullPath(void);
-PyAPI_FUNC(char *) Py_GetPrefix(void);
-PyAPI_FUNC(char *) Py_GetExecPrefix(void);
-PyAPI_FUNC(char *) Py_GetPath(void);
+PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void);
+PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);
+PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
+PyAPI_FUNC(wchar_t *) Py_GetPath(void);
 
 /* In their own files */
 PyAPI_FUNC(const char *) Py_GetVersion(void);

Include/sysmodule.h

@@ -9,8 +9,8 @@ extern "C" {
 
 PyAPI_FUNC(PyObject *) PySys_GetObject(const char *);
 PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *);
-PyAPI_FUNC(void) PySys_SetArgv(int, char **);
-PyAPI_FUNC(void) PySys_SetPath(const char *);
+PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **);
+PyAPI_FUNC(void) PySys_SetPath(const wchar_t *);
 
 PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...)
 			Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
@@ -21,7 +21,7 @@ PyAPI_DATA(PyObject *) _PySys_TraceFunc, *_PySys_ProfileFunc;
 PyAPI_DATA(int) _PySys_CheckInterval;
 
 PyAPI_FUNC(void) PySys_ResetWarnOptions(void);
-PyAPI_FUNC(void) PySys_AddWarnOption(const char *);
+PyAPI_FUNC(void) PySys_AddWarnOption(const wchar_t *);
 
 #ifdef __cplusplus
 }

Misc/NEWS

@@ -12,6 +12,10 @@ What's New in Python 3.0a5?
 Core and Builtins
 -----------------
 
+- The command line processing was converted to pass Unicode strings
+  through as unmodified as possible; as a consequence, the C API
+  related to command line arguments was changed to use wchar_t.
+
 Extension Modules
 -----------------
 

Modules/_tkinter.c

@@ -3012,7 +3012,7 @@ ins_string(PyObject *d, char *name, char *val)
 PyMODINIT_FUNC
 init_tkinter(void)
 {
-	PyObject *m, *d;
+	PyObject *m, *d, *uexe, *cexe;
 
 	Py_TYPE(&Tkapp_Type) = &PyType_Type;
 
@@ -3065,7 +3065,16 @@ init_tkinter(void)
 
 	/* This helps the dynamic loader; in Unicode aware Tcl versions
 	   it also helps Tcl find its encodings. */
-	Tcl_FindExecutable(Py_GetProgramName());
+	uexe = PyUnicode_FromWideChar(Py_GetProgramName(), -1);
+	if (uexe) {
+		cexe = PyUnicode_AsEncodedString(uexe, 
+						 Py_FileSystemDefaultEncoding, 
+						 NULL);
+		if (cexe)
+			Tcl_FindExecutable(PyString_AsString(cexe));
+		Py_XDECREF(cexe);
+		Py_DECREF(uexe);
+	}
 
 	if (PyErr_Occurred())
 		return;

Modules/main.c

@@ -12,6 +12,7 @@
 #include <windows.h>
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
+#define PATH_MAX MAXPATHLEN
 #endif
 #endif
 
@@ -40,17 +41,17 @@ extern "C" {
 #endif
 
 /* For Py_GetArgcArgv(); set by main() */
-static char **orig_argv;
+static wchar_t **orig_argv;
 static int  orig_argc;
 
 /* command line options */
-#define BASE_OPTS "bBc:dEhim:OStuvVW:xX?"
+#define BASE_OPTS L"bBc:dEhim:OStuvVW:xX?"
 
 #define PROGRAM_OPTS BASE_OPTS
 
 /* Short usage message (with %s for argv0) */
 static char *usage_line =
-"usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...\n";
+"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n";
 
 /* Long usage message, split into parts < 512 bytes */
 static char *usage_1 = "\
@@ -96,9 +97,30 @@ PYTHONHOME   : alternate <prefix> directory (or <prefix>%c<exec_prefix>).\n\
 PYTHONCASEOK : ignore case in 'import' statements (Windows).\n\
 ";
 
+#ifndef MS_WINDOWS
+static FILE*
+_wfopen(const wchar_t *path, const wchar_t *mode)
+{
+	char cpath[PATH_MAX];
+	char cmode[10];
+	size_t r;
+	r = wcstombs(cpath, path, PATH_MAX);
+	if (r == (size_t)-1 || r >= PATH_MAX) {
+		errno = EINVAL;
+		return NULL;
+	}
+	r = wcstombs(cmode, mode, 10);
+	if (r == (size_t)-1 || r >= 10) {
+		errno = EINVAL;
+		return NULL;
+	}
+	return fopen(cpath, cmode);
+}
+#endif
+
 
 static int
-usage(int exitcode, char* program)
+usage(int exitcode, wchar_t* program)
 {
 	FILE *f = exitcode ? stderr : stdout;
 
@@ -187,11 +209,11 @@ static int RunModule(char *module, int set_argv0)
 	return 0;
 }
 
-static int RunMainFromImporter(char *filename)
+static int RunMainFromImporter(wchar_t *filename)
 {
 	PyObject *argv0 = NULL, *importer = NULL;
 
-	if ((argv0 = PyUnicode_DecodeFSDefault(filename)) &&
+	if ((argv0 = PyUnicode_FromWideChar(filename,wcslen(filename))) &&
 	    (importer = PyImport_GetImporter(argv0)) &&
 	    (importer->ob_type != &PyNullImporter_Type))
 	{
@@ -249,12 +271,12 @@ WaitForThreadShutdown(void)
 /* Main program */
 
 int
-Py_Main(int argc, char **argv)
+Py_Main(int argc, wchar_t **argv)
 {
 	int c;
 	int sts;
 	char *command = NULL;
-	char *filename = NULL;
+	wchar_t *filename = NULL;
 	char *module = NULL;
 	FILE *fp = stdin;
 	char *p;
@@ -275,14 +297,19 @@ Py_Main(int argc, char **argv)
 
 	while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) {
 		if (c == 'c') {
+			size_t r1 = wcslen(_PyOS_optarg) + 2;
+			size_t r2;
 			/* -c is the last option; following arguments
 			   that look like options are left for the
 			   command to interpret. */
-			command = (char *)malloc(strlen(_PyOS_optarg) + 2);
+			command = (char *)malloc(r1);
 			if (command == NULL)
 				Py_FatalError(
 				   "not enough memory to copy -c argument");
-			strcpy(command, _PyOS_optarg);
+			r2 = wcstombs(command, _PyOS_optarg, r1);
+			if (r2 > r1-2)
+				Py_FatalError(
+				    "not enough memory to copy -c argument");
 			strcat(command, "\n");
 			break;
 		}
@@ -291,11 +318,16 @@ Py_Main(int argc, char **argv)
 			/* -m is the last option; following arguments
 			   that look like options are left for the
 			   module to interpret. */
-			module = (char *)malloc(strlen(_PyOS_optarg) + 2);
+			size_t r1 = wcslen(_PyOS_optarg) + 1;
+			size_t r2;
+			module = (char *)malloc(r1);
 			if (module == NULL)
 				Py_FatalError(
 				   "not enough memory to copy -m argument");
-			strcpy(module, _PyOS_optarg);
+			r2 = wcstombs(module, _PyOS_optarg, r1);
+			if (r2 >= r1)
+				Py_FatalError(
+				   "not enough memory to copy -m argument");
 			break;
 		}
 
@@ -355,7 +387,7 @@ Py_Main(int argc, char **argv)
 			version++;
 			break;
 
-		case 'W':
+		case 'W': 
 			PySys_AddWarnOption(_PyOS_optarg);
 			break;
 
@@ -384,7 +416,7 @@ Py_Main(int argc, char **argv)
 		unbuffered = 1;
 
 	if (command == NULL && module == NULL && _PyOS_optind < argc &&
-	    strcmp(argv[_PyOS_optind], "-") != 0)
+	    wcscmp(argv[_PyOS_optind], L"-") != 0)
 	{
 #ifdef __VMS
 		filename = decc$translate_vms(argv[_PyOS_optind]);
@@ -462,14 +494,14 @@ Py_Main(int argc, char **argv)
 	if (command != NULL) {
 		/* Backup _PyOS_optind and force sys.argv[0] = '-c' */
 		_PyOS_optind--;
-		argv[_PyOS_optind] = "-c";
+		argv[_PyOS_optind] = L"-c";
 	}
 
 	if (module != NULL) {
 		/* Backup _PyOS_optind and force sys.argv[0] = '-c'
 		   so that PySys_SetArgv correctly sets sys.path[0] to ''*/
 		_PyOS_optind--;
-		argv[_PyOS_optind] = "-c";
+		argv[_PyOS_optind] = L"-c";
 	}
 
 	PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind);
@@ -506,8 +538,8 @@ Py_Main(int argc, char **argv)
 		}
 
 		if (sts==-1 && filename!=NULL) {
-			if ((fp = fopen(filename, "r")) == NULL) {
-				fprintf(stderr, "%s: can't open file '%s': [Errno %d] %s\n",
+			if ((fp = _wfopen(filename, L"r")) == NULL) {
+				fprintf(stderr, "%s: can't open file '%ls': [Errno %d] %s\n",
 					argv[0], filename, errno, strerror(errno));
 
 				return 2;
@@ -528,7 +560,7 @@ Py_Main(int argc, char **argv)
 				struct stat sb;
 				if (fstat(fileno(fp), &sb) == 0 &&
 				    S_ISDIR(sb.st_mode)) {
-					fprintf(stderr, "%s: '%s' is a directory, cannot continue\n", argv[0], filename);
+					fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", argv[0], filename);
 					fclose(fp);
 					return 1;
 				}
@@ -536,9 +568,17 @@ Py_Main(int argc, char **argv)
 		}
 
 		if (sts==-1) {
+			char cfilename[PATH_MAX];
+			char *p_cfilename = "<stdin>";
+			if (filename) {
+				size_t r = wcstombs(cfilename, filename, PATH_MAX);
+				p_cfilename = cfilename;
+				if (r == (size_t)-1 || r >= PATH_MAX)
+					p_cfilename = "<decoding error>";
+			}
 			sts = PyRun_AnyFileExFlags(
 				fp,
-				filename == NULL ? "<stdin>" : filename,
+				p_cfilename,
 				filename != NULL, &cf) != 0;
 		}
 		
@@ -589,7 +629,7 @@ Py_Main(int argc, char **argv)
    This is rare, but it is needed by the secureware extension. */
 
 void
-Py_GetArgcArgv(int *argc, char ***argv)
+Py_GetArgcArgv(int *argc, wchar_t ***argv)
 {
 	*argc = orig_argc;
 	*argv = orig_argv;

Modules/python.c

 /* Minimal main program -- everything is loaded from the library */
 
 #include "Python.h"
+#include <locale.h>
 
 #ifdef __FreeBSD__
 #include <floatingpoint.h>
 #endif
 
+#ifdef MS_WINDOWS
+int
+wmain(int argc, wchar_t **argv)
+{
+	return Py_Main(argc, argv);
+}
+#else
 int
 main(int argc, char **argv)
 {
+	wchar_t **argv_copy = PyMem_Malloc(sizeof(wchar_t*)*argc);
+	/* We need a second copies, as Python might modify the first one. */
+	wchar_t **argv_copy2 = PyMem_Malloc(sizeof(wchar_t*)*argc);
+	int i, res;
+	char *oldloc;
 	/* 754 requires that FP exceptions run in "no stop" mode by default,
 	 * and until C vendors implement C99's ways to control FP exceptions,
 	 * Python requires non-stop mode.  Alas, some platforms enable FP
@@ -20,5 +33,33 @@ main(int argc, char **argv)
 	m = fpgetmask();
 	fpsetmask(m & ~FP_X_OFL);
 #endif
-	return Py_Main(argc, argv);
+	if (!argv_copy || !argv_copy2) {
+		fprintf(stderr, "out of memory");
+		return 1;
+	}
+	oldloc = setlocale(LC_ALL, NULL);
+	setlocale(LC_ALL, "");
+	for (i = 0; i < argc; i++) {
+		size_t argsize = mbstowcs(NULL, argv[i], 0);
+		if (argsize == (size_t)-1) {
+			fprintf(stderr, "Could not convert argument %d to string", i);
+			return 1;
+		}
+		argv_copy[i] = PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+		argv_copy2[i] = argv_copy[i];
+		if (!argv_copy[i]) {
+			fprintf(stderr, "out of memory");
+			return 1;
+		}
+		mbstowcs(argv_copy[i], argv[i], argsize+1);
+	}
+	setlocale(LC_ALL, oldloc);
+	res = Py_Main(argc, argv_copy);
+	for (i = 0; i < argc; i++) {
+		PyMem_Free(argv_copy2[i]);
+	}
+	PyMem_Free(argv_copy);
+	PyMem_Free(argv_copy2);
+	return res;
 }
+#endif

Objects/unicodeobject.c

@@ -553,10 +553,16 @@ PyObject *PyUnicode_FromWideChar(register const wchar_t *w,
     PyUnicodeObject *unicode;
 
     if (w == NULL) {
+        if (size == 0)
+            return PyUnicode_FromStringAndSize(NULL, 0);
 	PyErr_BadInternalCall();
 	return NULL;
     }
 
+    if (size == -1) {
+        size = wcslen(w);
+    }
+
     unicode = _PyUnicode_New(size);
     if (!unicode)
         return NULL;

PC/WinMain.c

@@ -12,5 +12,5 @@ int WINAPI WinMain(
     int nCmdShow              /* show state of window */
 )
 {
-    return Py_Main(__argc, __argv);
+    return Py_Main(__argc, __wargv);
 }

PC/pyconfig.h

@@ -80,7 +80,7 @@ WIN32 is still required for the locale module.
 #define MS_WIN32 /* only support win32 and greater. */
 #define MS_WINDOWS
 #ifndef PYTHONPATH
-#	define PYTHONPATH ".\\DLLs;.\\lib;.\\lib\\plat-win;.\\lib\\lib-tk"
+#	define PYTHONPATH L".\\DLLs;.\\lib;.\\lib\\plat-win;.\\lib\\lib-tk"
 #endif
 #define NT_THREADS
 #define WITH_THREAD

Python/frozenmain.c

@@ -2,6 +2,7 @@
 /* Python interpreter main program for frozen scripts */
 
 #include "Python.h"
+#include <locale.h>
 
 #ifdef MS_WINDOWS
 extern void PyWinFreeze_ExeInit(void);
@@ -15,9 +16,13 @@ int
 Py_FrozenMain(int argc, char **argv)
 {
 	char *p;
-	int n, sts;
+	int i, n, sts;
 	int inspect = 0;
 	int unbuffered = 0;
+	char *oldloc;
+	wchar_t **argv_copy = PyMem_Malloc(sizeof(wchar_t*)*argc);
+	/* We need a second copies, as Python might modify the first one. */
+	wchar_t **argv_copy2 = PyMem_Malloc(sizeof(wchar_t*)*argc);
 
 	Py_FrozenFlag = 1; /* Suppress errors from getpath.c */
 
@@ -32,10 +37,33 @@ Py_FrozenMain(int argc, char **argv)
 		setbuf(stderr, (char *)NULL);
 	}
 
+	if (!argv_copy) {
+		fprintf(stderr, "out of memory");
+		return 1;
+	}
+
+	oldloc = setlocale(LC_ALL, NULL);
+	setlocale(LC_ALL, "");
+	for (i = 0; i < argc; i++) {
+		size_t argsize = mbstowcs(NULL, argv[i], 0);
+		if (argsize == (size_t)-1) {
+			fprintf(stderr, "Could not convert argument %d to string", i);
+			return 1;
+		}
+		argv_copy[i] = PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+		argv_copy2[i] = argv_copy[i];
+		if (!argv_copy[i]) {
+			fprintf(stderr, "out of memory");
+			return 1;
+		}
+		mbstowcs(argv_copy[i], argv[i], argsize+1);
+	}
+	setlocale(LC_ALL, oldloc);
+
 #ifdef MS_WINDOWS
 	PyInitFrozenExtensions();
 #endif /* MS_WINDOWS */
-	Py_SetProgramName(argv[0]);
+	Py_SetProgramName(argv_copy[0]);
 	Py_Initialize();
 #ifdef MS_WINDOWS
 	PyWinFreeze_ExeInit();
@@ -45,7 +73,7 @@ Py_FrozenMain(int argc, char **argv)
 		fprintf(stderr, "Python %s\n%s\n",
 			Py_GetVersion(), Py_GetCopyright());
 
-	PySys_SetArgv(argc, argv);
+	PySys_SetArgv(argc, argv_copy);
 
 	n = PyImport_ImportFrozenModule("__main__");
 	if (n == 0)
@@ -64,5 +92,10 @@ Py_FrozenMain(int argc, char **argv)
 	PyWinFreeze_ExeTerm();
 #endif
 	Py_Finalize();
+	for (i = 0; i < argc; i++) {
+		PyMem_Free(argv_copy2[i]);
+	}
+	PyMem_Free(argv_copy);
+	PyMem_Free(argv_copy2);
 	return sts;
 }

Python/getopt.c

@@ -27,8 +27,11 @@
 /* Modified to support --help and --version, as well as /? on Windows
  * by Georg Brandl. */
 
+#include <Python.h>
 #include <stdio.h>
 #include <string.h>
+#include <wchar.h>
+#include <pygetopt.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -36,40 +39,40 @@ extern "C" {
 
 int _PyOS_opterr = 1;          /* generate error messages */
 int _PyOS_optind = 1;          /* index into argv array   */
-char *_PyOS_optarg = NULL;     /* optional argument       */
+wchar_t *_PyOS_optarg = NULL;     /* optional argument       */
 
-int _PyOS_GetOpt(int argc, char **argv, char *optstring)
+int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring)
 {
-	static char *opt_ptr = "";
-	char *ptr;
-	int option;
+	static wchar_t *opt_ptr = L"";
+	wchar_t *ptr;
+	wchar_t option;
 
 	if (*opt_ptr == '\0') {
 
 		if (_PyOS_optind >= argc)
 			return -1;
 #ifdef MS_WINDOWS
-		else if (strcmp(argv[_PyOS_optind], "/?") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"/?") == 0) {
 			++_PyOS_optind;
 			return 'h';
 		}
 #endif
 
-		else if (argv[_PyOS_optind][0] != '-' ||
-		         argv[_PyOS_optind][1] == '\0' /* lone dash */ )
+		else if (argv[_PyOS_optind][0] != L'-' ||
+		         argv[_PyOS_optind][1] == L'\0' /* lone dash */ )
 			return -1;
 
-		else if (strcmp(argv[_PyOS_optind], "--") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"--") == 0) {
 			++_PyOS_optind;
 			return -1;
 		}
 
-		else if (strcmp(argv[_PyOS_optind], "--help") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"--help") == 0) {
 			++_PyOS_optind;
 			return 'h';
 		}
 
-		else if (strcmp(argv[_PyOS_optind], "--version") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"--version") == 0) {
 			++_PyOS_optind;
 			return 'V';
 		}
@@ -78,27 +81,27 @@ int _PyOS_GetOpt(int argc, char **argv, char *optstring)
 		opt_ptr = &argv[_PyOS_optind++][1]; 
 	}
 
-	if ( (option = *opt_ptr++) == '\0')
+	if ( (option = *opt_ptr++) == L'\0')
 		return -1;
 	
-	if ((ptr = strchr(optstring, option)) == NULL) {
+	if ((ptr = wcschr(optstring, option)) == NULL) {
 		if (_PyOS_opterr)
-			fprintf(stderr, "Unknown option: -%c\n", option);
+		  fprintf(stderr, "Unknown option: -%c\n", (char)option);
 
 		return '_';
 	}
 
-	if (*(ptr + 1) == ':') {
-		if (*opt_ptr != '\0') {
+	if (*(ptr + 1) == L':') {
+		if (*opt_ptr != L'\0') {
 			_PyOS_optarg  = opt_ptr;
-			opt_ptr = "";
+			opt_ptr = L"";
 		}
 
 		else {
 			if (_PyOS_optind >= argc) {
 				if (_PyOS_opterr)
 					fprintf(stderr,
-			    "Argument expected for the -%c option\n", option);
+						"Argument expected for the -%c option\n", (char)option);
 				return '_';
 			}
 

Python/pythonrun.c

@@ -17,6 +17,7 @@
 #include "ast.h"
 #include "eval.h"
 #include "marshal.h"
+#include "osdefs.h"
 
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
@@ -30,6 +31,7 @@
 #ifdef MS_WINDOWS
 #undef BYTE
 #include "windows.h"
+#define PATH_MAX MAXPATHLEN
 #endif
 
 #ifndef Py_REF_DEBUG
@@ -44,7 +46,7 @@
 extern "C" {
 #endif
 
-extern char *Py_GetPath(void);
+extern wchar_t *Py_GetPath(void);
 
 extern grammar _PyParser_Grammar; /* From graminit.c */
 
@@ -646,35 +648,43 @@ Py_EndInterpreter(PyThreadState *tstate)
 	PyInterpreterState_Delete(interp);
 }
 
-static char *progname = "python";
+static wchar_t *progname = L"python";
 
 void
-Py_SetProgramName(char *pn)
+Py_SetProgramName(wchar_t *pn)
 {
 	if (pn && *pn)
 		progname = pn;
 }
 
-char *
+wchar_t *
 Py_GetProgramName(void)
 {
 	return progname;
 }
 
-static char *default_home = NULL;
+static wchar_t *default_home = NULL;
+static wchar_t env_home[PATH_MAX+1];
 
 void
-Py_SetPythonHome(char *home)
+Py_SetPythonHome(wchar_t *home)
 {
 	default_home = home;
 }
 
-char *
+wchar_t *
 Py_GetPythonHome(void)
 {
-	char *home = default_home;
-	if (home == NULL && !Py_IgnoreEnvironmentFlag)
-		home = Py_GETENV("PYTHONHOME");
+	wchar_t *home = default_home;
+	if (home == NULL && !Py_IgnoreEnvironmentFlag) {
+		char* chome = Py_GETENV("PYTHONHOME");
+		if (chome) {
+			size_t r = mbstowcs(env_home, chome, PATH_MAX+1);
+			if (r != (size_t)-1 && r <= PATH_MAX)
+				home = env_home;
+		}
+
+	}
 	return home;
 }
 

Python/sysmodule.c

@@ -882,7 +882,7 @@ PySys_ResetWarnOptions(void)
 }
 
 void
-PySys_AddWarnOption(const char *s)
+PySys_AddWarnOption(const wchar_t *s)
 {
 	PyObject *str;
 
@@ -892,7 +892,7 @@ PySys_AddWarnOption(const char *s)
 		if (warnoptions == NULL)
 			return;
 	}
-	str = PyUnicode_FromString(s);
+	str = PyUnicode_FromWideChar(s, -1);
 	if (str != NULL) {
 		PyList_Append(warnoptions, str);
 		Py_DECREF(str);
@@ -1222,12 +1222,12 @@ _PySys_Init(void)
 	SET_SYS_FROM_STRING("platform",
 			    PyUnicode_FromString(Py_GetPlatform()));
 	SET_SYS_FROM_STRING("executable",
-			    PyUnicode_DecodeFSDefault(
-				Py_GetProgramFullPath()));
+			    PyUnicode_FromWideChar(
+				   Py_GetProgramFullPath(), -1));
 	SET_SYS_FROM_STRING("prefix",
-			    PyUnicode_DecodeFSDefault(Py_GetPrefix()));
+			    PyUnicode_FromWideChar(Py_GetPrefix(), -1));
 	SET_SYS_FROM_STRING("exec_prefix",
-		   	    PyUnicode_DecodeFSDefault(Py_GetExecPrefix()));
+		   	    PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
 	SET_SYS_FROM_STRING("maxsize",
 			    PyLong_FromSsize_t(PY_SSIZE_T_MAX));
 	SET_SYS_FROM_STRING("float_info",
@@ -1280,15 +1280,15 @@ _PySys_Init(void)
 }
 
 static PyObject *
-makepathobject(const char *path, int delim)
+makepathobject(const wchar_t *path, wchar_t delim)
 {
 	int i, n;
-	const char *p;
+	const wchar_t *p;
 	PyObject *v, *w;
 
 	n = 1;
 	p = path;
-	while ((p = strchr(p, delim)) != NULL) {
+	while ((p = wcschr(p, delim)) != NULL) {
 		n++;
 		p++;
 	}
@@ -1296,10 +1296,10 @@ makepathobject(const char *path, int delim)
 	if (v == NULL)
 		return NULL;
 	for (i = 0; ; i++) {
-		p = strchr(path, delim);
+		p = wcschr(path, delim);
 		if (p == NULL)
-			p = strchr(path, '\0'); /* End of string */
-		w = PyUnicode_DecodeFSDefaultAndSize(path, (Py_ssize_t) (p - path));
+			p = wcschr(path, L'\0'); /* End of string */
+		w = PyUnicode_FromWideChar(path, (Py_ssize_t)(p - path));
 		if (w == NULL) {
 			Py_DECREF(v);
 			return NULL;
@@ -1313,7 +1313,7 @@ makepathobject(const char *path, int delim)
 }
 
 void
-PySys_SetPath(const char *path)
+PySys_SetPath(const wchar_t *path)
 {
 	PyObject *v;
 	if ((v = makepathobject(path, DELIM)) == NULL)
@@ -1324,12 +1324,12 @@ PySys_SetPath(const char *path)
 }
 
 static PyObject *
-makeargvobject(int argc, char **argv)
+makeargvobject(int argc, wchar_t **argv)
 {
 	PyObject *av;
 	if (argc <= 0 || argv == NULL) {
 		/* Ensure at least one (empty) argument is seen */
-		static char *empty_argv[1] = {""};
+		static wchar_t *empty_argv[1] = {L""};
 		argv = empty_argv;
 		argc = 1;
 	}
@@ -1351,7 +1351,7 @@ makeargvobject(int argc, char **argv)
 			} else
 				v = PyUnicode_FromString(argv[i]);
 #else
-			PyObject *v = PyUnicode_FromString(argv[i]);
+			PyObject *v = PyUnicode_FromWideChar(argv[i], -1);
 #endif
 			if (v == NULL) {
 				Py_DECREF(av);
@@ -1364,13 +1364,38 @@ makeargvobject(int argc, char **argv)
 	return av;
 }
 
+#ifdef HAVE_REALPATH
+static wchar_t*
+_wrealpath(const wchar_t *path, wchar_t *resolved_path)
+{
+	char cpath[PATH_MAX];
+	char cresolved_path[PATH_MAX];
+	char *res;
+	size_t r;
+	r = wcstombs(cpath, path, PATH_MAX);
+	if (r == (size_t)-1 || r >= PATH_MAX) {
+		errno = EINVAL;
+		return NULL;
+	}
+	res = realpath(cpath, cresolved_path);
+	if (res == NULL)
+		return NULL;
+	r = mbstowcs(resolved_path, cresolved_path, PATH_MAX);
+	if (r == (size_t)-1 || r >= PATH_MAX) {
+		errno = EINVAL;
+		return NULL;
+	}
+	return resolved_path;
+}
+#endif
+
 void
-PySys_SetArgv(int argc, char **argv)
+PySys_SetArgv(int argc, wchar_t **argv)
 {
 #if defined(HAVE_REALPATH)
-	char fullpath[MAXPATHLEN];
+	wchar_t fullpath[MAXPATHLEN];
 #elif defined(MS_WINDOWS)
-	char fullpath[MAX_PATH];
+	wchar_t fullpath[MAX_PATH];
 #endif
 	PyObject *av = makeargvobject(argc, argv);
 	PyObject *path = PySys_GetObject("path");
@@ -1379,53 +1404,54 @@ PySys_SetArgv(int argc, char **argv)
 	if (PySys_SetObject("argv", av) != 0)
 		Py_FatalError("can't assign sys.argv");
 	if (path != NULL) {
-		char *argv0 = argv[0];
-		char *p = NULL;
+		wchar_t *argv0 = argv[0];
+		wchar_t *p = NULL;
 		Py_ssize_t n = 0;
 		PyObject *a;
+		extern int _Py_wreadlink(const wchar_t *, wchar_t *, size_t);
 #ifdef HAVE_READLINK
-		char link[MAXPATHLEN+1];
-		char argv0copy[2*MAXPATHLEN+1];
+		wchar_t link[MAXPATHLEN+1];
+		wchar_t argv0copy[2*MAXPATHLEN+1];
 		int nr = 0;
-		if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0)
-			nr = readlink(argv0, link, MAXPATHLEN);
+		if (argc > 0 && argv0 != NULL && wcscmp(argv0, L"-c") != 0)
+			nr = _Py_wreadlink(argv0, link, MAXPATHLEN);
 		if (nr > 0) {
 			/* It's a symlink */
 			link[nr] = '\0';
 			if (link[0] == SEP)
 				argv0 = link; /* Link to absolute path */
-			else if (strchr(link, SEP) == NULL)
+			else if (wcschr(link, SEP) == NULL)
 				; /* Link without path */
 			else {
 				/* Must join(dirname(argv0), link) */
-				char *q = strrchr(argv0, SEP);
+				wchar_t *q = wcsrchr(argv0, SEP);
 				if (q == NULL)
 					argv0 = link; /* argv0 without path */
 				else {
 					/* Must make a copy */
-					strcpy(argv0copy, argv0);
-					q = strrchr(argv0copy, SEP);
-					strcpy(q+1, link);
+					wcscpy(argv0copy, argv0);
+					q = wcsrchr(argv0copy, SEP);
+					wcscpy(q+1, link);
 					argv0 = argv0copy;
 				}
 			}
 		}
 #endif /* HAVE_READLINK */
 #if SEP == '\\' /* Special case for MS filename syntax */
-		if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) {
-			char *q;
+		if (argc > 0 && argv0 != NULL && wcscmp(argv0, L"-c") != 0) {
+			wchar_t *q;
 #ifdef MS_WINDOWS
-			char *ptemp;
-			if (GetFullPathName(argv0,
-					   sizeof(fullpath),
+			wchar_t *ptemp;
+			if (GetFullPathNameW(argv0,
+					   sizeof(fullpath)/sizeof(fullpath[0]),
 					   fullpath,
 					   &ptemp)) {
 				argv0 = fullpath;
 			}
 #endif
-			p = strrchr(argv0, SEP);
+			p = wcsrchr(argv0, SEP);
 			/* Test for alternate separator */
-			q = strrchr(p ? p : argv0, '/');
+			q = wcsrchr(p ? p : argv0, '/');
 			if (q != NULL)
 				p = q;
 			if (p != NULL) {
@@ -1435,13 +1461,13 @@ PySys_SetArgv(int argc, char **argv)
 			}
 		}
 #else /* All other filename syntaxes */
-		if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) {
+		if (argc > 0 && argv0 != NULL && wcscmp(argv0, L"-c") != 0) {
 #if defined(HAVE_REALPATH)
-			if (realpath(argv0, fullpath)) {
+			if (_wrealpath(argv0, fullpath)) {
 				argv0 = fullpath;
 			}
 #endif
-			p = strrchr(argv0, SEP);
+			p = wcsrchr(argv0, SEP);
 		}
 		if (p != NULL) {
 			n = p + 1 - argv0;
@@ -1451,7 +1477,7 @@ PySys_SetArgv(int argc, char **argv)
 #endif /* Unix */
 		}
 #endif /* All others */
-		a = PyUnicode_FromStringAndSize(argv0, n);
+		a = PyUnicode_FromWideChar(argv0, n);
 		if (a == NULL)
 			Py_FatalError("no mem for sys.path insertion");
 		if (PyList_Insert(path, 0, a) < 0)