Issue #3696: Error parsing arguments on OpenBSD <= 4.4 and Cygwin.

Patch by Amaury Forgeot d'Arc, reviewed by me.

Misc/NEWS

@@ -12,6 +12,11 @@ What's New in Python 3.0 release candidate 1
 Core and Builtins
 -----------------
 
+- Issue #3696: Error parsing arguments on OpenBSD <= 4.4 and Cygwin. On
+  these systems, the mbstowcs() function is slightly buggy and must be
+  replaced with strlen() for the purpose of counting of number of wide
+  characters needed to represent the multi-byte character string.
+
 - Issue #3697: "Fatal Python error: Cannot recover from stack overflow"
   could be easily encountered under Windows in debug mode when exercising
   the recursion limit checking code, due to bogus handling of recursion

Modules/_localemodule.c

@@ -49,7 +49,11 @@ static PyObject *Error;
 static PyObject*
 str2uni(const char* s)
 {
+#ifdef HAVE_BROKEN_MBSTOWCS
+    size_t needed = strlen(s);
+#else
     size_t needed = mbstowcs(NULL, s, 0);
+#endif
     size_t res1;
     wchar_t smallbuf[30];
     wchar_t *dest;
@@ -67,7 +71,11 @@ str2uni(const char* s)
     }
     /* This shouldn't fail now */
     res1 = mbstowcs(dest, s, needed+1);
+#ifdef HAVE_BROKEN_MBSTOWCS
+    assert(res1 != (size_t)-1);
+#else
     assert(res1 == needed);
+#endif
     res2 = PyUnicode_FromWideChar(dest, res1);
     if (dest != smallbuf)
         PyMem_Free(dest);

Modules/python.c

@@ -40,7 +40,16 @@ main(int argc, char **argv)
 	oldloc = setlocale(LC_ALL, NULL);
 	setlocale(LC_ALL, "");
 	for (i = 0; i < argc; i++) {
+#ifdef HAVE_BROKEN_MBSTOWCS
+		/* Some platforms have a broken implementation of
+		 * mbstowcs which does not count the characters that
+		 * would result from conversion.  Use an upper bound.
+		 */
+		size_t argsize = strlen(argv[i]);
+#else
 		size_t argsize = mbstowcs(NULL, argv[i], 0);
+#endif
+		size_t count;
 		if (argsize == (size_t)-1) {
 			fprintf(stderr, "Could not convert argument %d to string", i);
 			return 1;
@@ -51,7 +60,11 @@ main(int argc, char **argv)
 			fprintf(stderr, "out of memory");
 			return 1;
 		}
-		mbstowcs(argv_copy[i], argv[i], argsize+1);
+		count = mbstowcs(argv_copy[i], argv[i], argsize+1);
+		if (count == (size_t)-1) {
+			fprintf(stderr, "Could not convert argument %d to string", i);
+			return 1;
+		}
 	}
 	setlocale(LC_ALL, oldloc);
 	res = Py_Main(argc, argv_copy);

Python/frozenmain.c

@@ -45,7 +45,12 @@ Py_FrozenMain(int argc, char **argv)
 	oldloc = setlocale(LC_ALL, NULL);
 	setlocale(LC_ALL, "");
 	for (i = 0; i < argc; i++) {
+#ifdef HAVE_BROKEN_MBSTOWCS
+		size_t argsize = strlen(argv[i]);
+#else
 		size_t argsize = mbstowcs(NULL, argv[i], 0);
+#endif
+		size_t count;
 		if (argsize == (size_t)-1) {
 			fprintf(stderr, "Could not convert argument %d to string", i);
 			return 1;
@@ -56,7 +61,11 @@ Py_FrozenMain(int argc, char **argv)
 			fprintf(stderr, "out of memory");
 			return 1;
 		}
-		mbstowcs(argv_copy[i], argv[i], argsize+1);
+		count = mbstowcs(argv_copy[i], argv[i], argsize+1);
+		if (count == (size_t)-1) {
+			fprintf(stderr, "Could not convert argument %d to string", i);
+			return 1;
+		}
 	}
 	setlocale(LC_ALL, oldloc);
 

configure

 #! /bin/sh
-# From configure.in Revision: 65206 .
+# From configure.in Revision: 65857 .
 # Guess values for system-dependent variables and create Makefiles.
 # Generated by GNU Autoconf 2.61 for python 3.0.
 #
@@ -24315,6 +24315,71 @@ _ACEOF
 fi
 
 
+{ echo "$as_me:$LINENO: checking for broken mbstowcs" >&5
+echo $ECHO_N "checking for broken mbstowcs... $ECHO_C" >&6; }
+if test "$cross_compiling" = yes; then
+  ac_cv_broken_mbstowcs=no
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+#include<stdlib.h>
+int main() {
+    size_t len = -1;
+    const char *str = "text";
+    len = mbstowcs(NULL, str, 0);
+    return (len != 4);
+}
+
+_ACEOF
+rm -f conftest$ac_exeext
+if { (ac_try="$ac_link"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_link") 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_try") 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_broken_mbstowcs=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_broken_mbstowcs=yes
+fi
+rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+{ echo "$as_me:$LINENO: result: $ac_cv_broken_mbstowcs" >&5
+echo "${ECHO_T}$ac_cv_broken_mbstowcs" >&6; }
+if test "$ac_cv_broken_mbstowcs" = yes
+then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_BROKEN_MBSTOWCS 1
+_ACEOF
+
+fi
+
 
 
 for h in `(cd $srcdir;echo Python/thread_*.h)`

configure.in

@@ -3571,6 +3571,27 @@ AC_CHECK_TYPE(socklen_t,,
 #endif
 ])
 
+AC_MSG_CHECKING(for broken mbstowcs)
+AC_TRY_RUN([
+#include<stdlib.h>
+int main() {
+    size_t len = -1;
+    const char *str = "text";
+    len = mbstowcs(NULL, str, 0);
+    return (len != 4);
+}
+],
+ac_cv_broken_mbstowcs=no,
+ac_cv_broken_mbstowcs=yes,
+ac_cv_broken_mbstowcs=no)
+AC_MSG_RESULT($ac_cv_broken_mbstowcs)
+if test "$ac_cv_broken_mbstowcs" = yes
+then
+  AC_DEFINE(HAVE_BROKEN_MBSTOWCS, 1,
+  [Define if mbstowcs(NULL, "text", 0) does not return the number of 
+   wide chars that would be converted.])
+fi
+
 AC_SUBST(THREADHEADERS)
 
 for h in `(cd $srcdir;echo Python/thread_*.h)`

pyconfig.h.in

@@ -803,6 +803,10 @@
 /* Define to 1 if you have the `wcsxfrm' function. */
 #undef HAVE_WCSXFRM
 
+/* Define if mbstowcs(NULL, "text", 0) does not return the number of 
+   wide chars that would be converted */
+#undef HAVE_BROKEN_MBSTOWCS
+
 /* Define if tzset() actually switches the local timezone in a meaningful way.
    */
 #undef HAVE_WORKING_TZSET