merge

Cython/Compiler/ModuleNode.py

@@ -2510,13 +2510,30 @@ refnanny_utility_code = UtilityCode(proto="""
 
 main_method = UtilityCode(
 impl = """
-#if PY_MAJOR_VERSION < 3 || (!defined(WIN32) && !defined(MS_WINDOWS))
+#ifdef __FreeBSD__
+#include <floatingpoint.h>
+#endif
+
+#if PY_MAJOR_VERSION < 3
 int main(int argc, char** argv) {
-#else
+#elif defined(WIN32) || defined(MS_WINDOWS)
 int wmain(int argc, wchar_t **argv) {
+#else
+static int __Pyx_main(int argc, wchar_t **argv) {
 #endif
     int r = 0;
     PyObject* m = NULL;
+    /* 754 requires that FP exceptions run in "no stop" mode by default,
+     * and until C vendors implement C99's ways to control FP exceptions,
+     * Python requires non-stop mode.  Alas, some platforms enable FP
+     * exceptions by default.  Here we disable them.
+     */
+#ifdef __FreeBSD__
+    fp_except_t m;
+
+    m = fpgetmask();
+    fpsetmask(m & ~FP_X_OFL);
+#endif
     Py_SetProgramName(argv[0]);
     Py_Initialize();
     PySys_SetArgv(argc, argv);
@@ -2524,17 +2541,157 @@ int wmain(int argc, wchar_t **argv) {
 #if PY_MAJOR_VERSION < 3
         init%(module_name)s();
 #else
-        m = PyInit_%(module_name)s(name);
+        m = PyInit_%(module_name)s();
 #endif
     if (PyErr_Occurred() != NULL) {
         r = 1;
         PyErr_Print(); /* This exits with the right code if SystemExit. */
+#if PY_MAJOR_VERSION < 3
         if (Py_FlushLine()) PyErr_Clear();
+#endif
     }
     Py_XDECREF(m);
     Py_Finalize();
     return r;
 }
+
+
+#if PY_MAJOR_VERSION >= 3 && !defined(WIN32) && !defined(MS_WINDOWS)
+#include <locale.h>
+
+static wchar_t*
+__Pyx_char2wchar(char* arg)
+{
+	wchar_t *res;
+#ifdef HAVE_BROKEN_MBSTOWCS
+	/* Some platforms have a broken implementation of
+	 * mbstowcs which does not count the characters that
+	 * would result from conversion.  Use an upper bound.
+	 */
+	size_t argsize = strlen(arg);
+#else
+	size_t argsize = mbstowcs(NULL, arg, 0);
+#endif
+	size_t count;
+	unsigned char *in;
+	wchar_t *out;
+#ifdef HAVE_MBRTOWC
+	mbstate_t mbs;
+#endif
+	if (argsize != (size_t)-1) {
+		res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+		if (!res)
+			goto oom;
+		count = mbstowcs(res, arg, argsize+1);
+		if (count != (size_t)-1) {
+			wchar_t *tmp;
+			/* Only use the result if it contains no
+			   surrogate characters. */
+			for (tmp = res; *tmp != 0 &&
+				     (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+				;
+			if (*tmp == 0)
+				return res;
+		}
+		PyMem_Free(res);
+	}
+	/* Conversion failed. Fall back to escaping with surrogateescape. */
+#ifdef HAVE_MBRTOWC
+	/* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+	/* Overallocate; as multi-byte characters are in the argument, the
+	   actual output could use less memory. */
+	argsize = strlen(arg) + 1;
+	res = PyMem_Malloc(argsize*sizeof(wchar_t));
+	if (!res) goto oom;
+	in = (unsigned char*)arg;
+	out = res;
+	memset(&mbs, 0, sizeof mbs);
+	while (argsize) {
+		size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+		if (converted == 0)
+			/* Reached end of string; null char stored. */
+			break;
+		if (converted == (size_t)-2) {
+			/* Incomplete character. This should never happen,
+			   since we provide everything that we have -
+			   unless there is a bug in the C library, or I
+			   misunderstood how mbrtowc works. */
+			fprintf(stderr, "unexpected mbrtowc result -2\\n");
+			return NULL;
+		}
+		if (converted == (size_t)-1) {
+			/* Conversion error. Escape as UTF-8b, and start over
+			   in the initial shift state. */
+			*out++ = 0xdc00 + *in++;
+			argsize--;
+			memset(&mbs, 0, sizeof mbs);
+			continue;
+		}
+		if (*out >= 0xd800 && *out <= 0xdfff) {
+			/* Surrogate character.  Escape the original
+			   byte sequence with surrogateescape. */
+			argsize -= converted;
+			while (converted--)
+				*out++ = 0xdc00 + *in++;
+			continue;
+		}
+		/* successfully converted some bytes */
+		in += converted;
+		argsize -= converted;
+		out++;
+	}
+#else
+	/* Cannot use C locale for escaping; manually escape as if charset
+	   is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+	   correctly in the locale's charset, which must be an ASCII superset. */
+	res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+	if (!res) goto oom;
+	in = (unsigned char*)arg;
+	out = res;
+	while(*in)
+		if(*in < 128)
+			*out++ = *in++;
+		else
+			*out++ = 0xdc00 + *in++;
+	*out = 0;
+#endif
+	return res;
+oom:
+	fprintf(stderr, "out of memory\\n");
+	return NULL;
+}
+
+int
+main(int argc, char **argv)
+{
+	wchar_t **argv_copy = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*argc);
+	/* We need a second copies, as Python might modify the first one. */
+	wchar_t **argv_copy2 = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*argc);
+	int i, res;
+	char *oldloc;
+	if (!argv_copy || !argv_copy2) {
+		fprintf(stderr, "out of memory\\n");
+		return 1;
+	}
+	oldloc = strdup(setlocale(LC_ALL, NULL));
+	setlocale(LC_ALL, "");
+	for (i = 0; i < argc; i++) {
+		argv_copy2[i] = argv_copy[i] = __Pyx_char2wchar(argv[i]);
+		if (!argv_copy[i])
+			return 1;
+	}
+	setlocale(LC_ALL, oldloc);
+	free(oldloc);
+	res = __Pyx_main(argc, argv_copy);
+	for (i = 0; i < argc; i++) {
+		PyMem_Free(argv_copy2[i]);
+	}
+	PyMem_Free(argv_copy);
+	PyMem_Free(argv_copy2);
+	return res;
+}
+#endif
 """)
 
 packed_struct_utility_code = UtilityCode(proto="""

Demos/freeze/Makefile

@@ -4,11 +4,11 @@ CYTHON_FREEZE = ../../bin/cython_freeze
 PYTHON = python
 RST2HTML = rst2html
 
-PY_LDFLAGS = $(shell $(PYTHON) -c 'from distutils.sysconfig import get_config_var as g; print " ".join([g("LINKFORSHARED"), "-L"+g("LIBPL")])')
-PY_CPPFLAGS = $(shell $(PYTHON) -c 'from distutils.sysconfig import *; print "-I"+get_python_inc()')
-PY_LDLIBS = $(shell $(PYTHON) -c 'from distutils.sysconfig import get_config_var as g; print " ".join(["-lpython"+g("VERSION"), g("SYSLIBS"), g("LIBS"), g("LOCALMODLIBS")])')
+PY_LDFLAGS = $(shell $(PYTHON) -c 'from distutils.sysconfig import get_config_var as g; import sys; sys.stdout.write(" ".join([g("LINKFORSHARED"), "-L"+g("LIBPL")]) + "\n")')
+PY_CPPFLAGS = $(shell $(PYTHON) -c 'from distutils.sysconfig import *; import sys; sys.stdout.write("-I"+get_python_inc() + "\n")')
+PY_LDLIBS = $(shell $(PYTHON) -c 'from distutils.sysconfig import get_config_var as g; import sys; sys.stdout.write(" ".join(["-lpython"+g("VERSION"), g("SYSLIBS"), g("LIBS"), g("LOCALMODLIBS")]) + "\n")')
 
-CFLAGS = -fPIC -g -O2 -Wall -Wextra
+CFLAGS = -fPIC -fno-strict-aliasing -g -O2 -Wall -Wextra
 CPPFLAGS = $(PY_CPPFLAGS)
 LDFLAGS = $(PY_LDFLAGS)
 LDLIBS = $(PY_LDLIBS)

Demos/freeze/cmath.pyx

@@ -19,6 +19,6 @@ if __name__ == "__main__":
     import sys
     if len(sys.argv) != 2:
         sys.stderr.write("USAGE: %s n\nPrints n!.\n" % sys.argv[0])
-        sys.exit(1)
+        sys.exit(2)
     n, = map(float, sys.argv[1:])
     print factorial(n)

Demos/freeze/combinatorics.pyx

@@ -9,6 +9,6 @@ if __name__ == "__main__":
     import sys
     if len(sys.argv) != 3:
         sys.stderr.write("USAGE: %s n r\nPrints n-choose-r.\n" % sys.argv[0])
-        sys.exit(1)
+        sys.exit(2)
     n, r = map(float, sys.argv[1:])
     print nCr(n, r)

bin/cython_freeze

@@ -38,9 +38,14 @@ modules = [format_modname(x) for x in args]
 
 print """\
 #include <Python.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 
+#ifdef __FreeBSD__
+#include <floatingpoint.h>
+#endif
+
 #if PY_MAJOR_VERSION < 3
 # define MODINIT(name)  init ## name
 #else
@@ -65,10 +70,12 @@ if not options.pymain:
     print "\nextern int __pyx_module_is_main_%s;" % modules[0]
 
 print """
-#if PY_MAJOR_VERSION < 3 || (!defined(WIN32) && !defined(MS_WINDOWS))
+#if PY_MAJOR_VERSION < 3
 int main(int argc, char** argv) {
-#else
+#elif defined(WIN32) || defined(MS_WINDOWS)
 int wmain(int argc, wchar_t **argv) {
+#else
+static int python_main(int argc, wchar_t **argv) {
 #endif
 """,
 if not options.pymain:
@@ -77,6 +84,17 @@ if not options.pymain:
     int r = 0;
 """,
 print """\
+    /* 754 requires that FP exceptions run in "no stop" mode by default,
+     * and until C vendors implement C99's ways to control FP exceptions,
+     * Python requires non-stop mode.  Alas, some platforms enable FP
+     * exceptions by default.  Here we disable them.
+     */
+#ifdef __FreeBSD__
+    fp_except_t m;
+
+    m = fpgetmask();
+    fpsetmask(m & ~FP_X_OFL);
+#endif
     if (PyImport_ExtendInittab(inittab)) {
         fprintf(stderr, "No memory\\n");
         exit(1);
@@ -97,11 +115,149 @@ else:
     if (!m) {
         r = 1;
         PyErr_Print(); /* This exits with the right code if SystemExit. */
+#if PY_MAJOR_VERSION < 3
         if (Py_FlushLine())
             PyErr_Clear();
+#endif
     }
     Py_XDECREF(m);
     Py_Finalize();
     return r;
 }
 """ % {'main' : modules[0]},
+
+print r"""
+#if PY_MAJOR_VERSION >= 3 && !defined(WIN32) && !defined(MS_WINDOWS)
+static wchar_t*
+char2wchar(char* arg)
+{
+	wchar_t *res;
+#ifdef HAVE_BROKEN_MBSTOWCS
+	/* Some platforms have a broken implementation of
+	 * mbstowcs which does not count the characters that
+	 * would result from conversion.  Use an upper bound.
+	 */
+	size_t argsize = strlen(arg);
+#else
+	size_t argsize = mbstowcs(NULL, arg, 0);
+#endif
+	size_t count;
+	unsigned char *in;
+	wchar_t *out;
+#ifdef HAVE_MBRTOWC
+	mbstate_t mbs;
+#endif
+	if (argsize != (size_t)-1) {
+		res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+		if (!res)
+			goto oom;
+		count = mbstowcs(res, arg, argsize+1);
+		if (count != (size_t)-1) {
+			wchar_t *tmp;
+			/* Only use the result if it contains no
+			   surrogate characters. */
+			for (tmp = res; *tmp != 0 &&
+				     (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+				;
+			if (*tmp == 0)
+				return res;
+		}
+		PyMem_Free(res);
+	}
+	/* Conversion failed. Fall back to escaping with surrogateescape. */
+#ifdef HAVE_MBRTOWC
+	/* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+	/* Overallocate; as multi-byte characters are in the argument, the
+	   actual output could use less memory. */
+	argsize = strlen(arg) + 1;
+	res = PyMem_Malloc(argsize*sizeof(wchar_t));
+	if (!res) goto oom;
+	in = (unsigned char*)arg;
+	out = res;
+	memset(&mbs, 0, sizeof mbs);
+	while (argsize) {
+		size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+		if (converted == 0)
+			/* Reached end of string; null char stored. */
+			break;
+		if (converted == (size_t)-2) {
+			/* Incomplete character. This should never happen,
+			   since we provide everything that we have -
+			   unless there is a bug in the C library, or I
+			   misunderstood how mbrtowc works. */
+			fprintf(stderr, "unexpected mbrtowc result -2\n");
+			return NULL;
+		}
+		if (converted == (size_t)-1) {
+			/* Conversion error. Escape as UTF-8b, and start over
+			   in the initial shift state. */
+			*out++ = 0xdc00 + *in++;
+			argsize--;
+			memset(&mbs, 0, sizeof mbs);
+			continue;
+		}
+		if (*out >= 0xd800 && *out <= 0xdfff) {
+			/* Surrogate character.  Escape the original
+			   byte sequence with surrogateescape. */
+			argsize -= converted;
+			while (converted--)
+				*out++ = 0xdc00 + *in++;
+			continue;
+		}
+		/* successfully converted some bytes */
+		in += converted;
+		argsize -= converted;
+		out++;
+	}
+#else
+	/* Cannot use C locale for escaping; manually escape as if charset
+	   is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+	   correctly in the locale's charset, which must be an ASCII superset. */
+	res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+	if (!res) goto oom;
+	in = (unsigned char*)arg;
+	out = res;
+	while(*in)
+		if(*in < 128)
+			*out++ = *in++;
+		else
+			*out++ = 0xdc00 + *in++;
+	*out = 0;
+#endif
+	return res;
+oom:
+	fprintf(stderr, "out of memory\n");
+	return NULL;
+}
+
+int
+main(int argc, char **argv)
+{
+	wchar_t **argv_copy = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*argc);
+	/* We need a second copies, as Python might modify the first one. */
+	wchar_t **argv_copy2 = (wchar_t **)PyMem_Malloc(sizeof(wchar_t*)*argc);
+	int i, res;
+	char *oldloc;
+	if (!argv_copy || !argv_copy2) {
+		fprintf(stderr, "out of memory\n");
+		return 1;
+	}
+	oldloc = strdup(setlocale(LC_ALL, NULL));
+	setlocale(LC_ALL, "");
+	for (i = 0; i < argc; i++) {
+		argv_copy2[i] = argv_copy[i] = char2wchar(argv[i]);
+		if (!argv_copy[i])
+			return 1;
+	}
+	setlocale(LC_ALL, oldloc);
+	free(oldloc);
+	res = python_main(argc, argv_copy);
+	for (i = 0; i < argc; i++) {
+		PyMem_Free(argv_copy2[i]);
+	}
+	PyMem_Free(argv_copy);
+	PyMem_Free(argv_copy2);
+	return res;
+}
+#endif"""