Branch merge

Doc/distutils/apiref.rst

@@ -2006,6 +2006,7 @@ This is described in more detail in :pep:`301`.
 
 .. % todo
 
+
 :mod:`distutils.command.check` --- Check the meta-data of a package
 ===================================================================
 

Lib/plistlib.py

@@ -68,13 +68,15 @@ def readPlist(pathOrFile):
     usually is a dictionary).
     """
     didOpen = False
-    if isinstance(pathOrFile, str):
-        pathOrFile = open(pathOrFile, 'rb')
-        didOpen = True
-    p = PlistParser()
-    rootObject = p.parse(pathOrFile)
-    if didOpen:
-        pathOrFile.close()
+    try:
+        if isinstance(pathOrFile, str):
+            pathOrFile = open(pathOrFile, 'rb')
+            didOpen = True
+        p = PlistParser()
+        rootObject = p.parse(pathOrFile)
+    finally:
+        if didOpen:
+            pathOrFile.close()
     return rootObject
 
 
@@ -83,15 +85,17 @@ def writePlist(rootObject, pathOrFile):
     file name or a (writable) file object.
     """
     didOpen = False
-    if isinstance(pathOrFile, str):
-        pathOrFile = open(pathOrFile, 'wb')
-        didOpen = True
-    writer = PlistWriter(pathOrFile)
-    writer.writeln("<plist version=\"1.0\">")
-    writer.writeValue(rootObject)
-    writer.writeln("</plist>")
-    if didOpen:
-        pathOrFile.close()
+    try:
+        if isinstance(pathOrFile, str):
+            pathOrFile = open(pathOrFile, 'wb')
+            didOpen = True
+        writer = PlistWriter(pathOrFile)
+        writer.writeln("<plist version=\"1.0\">")
+        writer.writeValue(rootObject)
+        writer.writeln("</plist>")
+    finally:
+        if didOpen:
+            pathOrFile.close()
 
 
 def readPlistFromBytes(data):
@@ -352,7 +356,6 @@ class Data:
     def __repr__(self):
         return "%s(%s)" % (self.__class__.__name__, repr(self.data))
 
-
 class PlistParser:
 
     def __init__(self):
@@ -362,11 +365,11 @@ class PlistParser:
 
     def parse(self, fileobj):
         from xml.parsers.expat import ParserCreate
-        parser = ParserCreate()
-        parser.StartElementHandler = self.handleBeginElement
-        parser.EndElementHandler = self.handleEndElement
-        parser.CharacterDataHandler = self.handleData
-        parser.ParseFile(fileobj)
+        self.parser = ParserCreate()
+        self.parser.StartElementHandler = self.handleBeginElement
+        self.parser.EndElementHandler = self.handleEndElement
+        self.parser.CharacterDataHandler = self.handleData
+        self.parser.ParseFile(fileobj)
         return self.root
 
     def handleBeginElement(self, element, attrs):
@@ -385,12 +388,18 @@ class PlistParser:
 
     def addObject(self, value):
         if self.currentKey is not None:
+            if not isinstance(self.stack[-1], type({})):
+                raise ValueError("unexpected element at line %d" %
+                                 self.parser.CurrentLineNumber)
             self.stack[-1][self.currentKey] = value
             self.currentKey = None
         elif not self.stack:
             # this is the root object
             self.root = value
         else:
+            if not isinstance(self.stack[-1], type([])):
+                raise ValueError("unexpected element at line %d" %
+                                 self.parser.CurrentLineNumber)
             self.stack[-1].append(value)
 
     def getData(self):
@@ -405,9 +414,15 @@ class PlistParser:
         self.addObject(d)
         self.stack.append(d)
     def end_dict(self):
+        if self.currentKey:
+            raise ValueError("missing value for key '%s' at line %d" %
+                             (self.currentKey,self.parser.CurrentLineNumber))
         self.stack.pop()
 
     def end_key(self):
+        if self.currentKey or not isinstance(self.stack[-1], type({})):
+            raise ValueError("unexpected key at line %d" %
+                             self.parser.CurrentLineNumber)
         self.currentKey = self.getData()
 
     def begin_array(self, attrs):

Lib/test/test_plistlib.py

@@ -175,6 +175,32 @@ class TestPlistlib(unittest.TestCase):
         self.assertEqual(test1, result1)
         self.assertEqual(test2, result2)
 
+    def test_invalidarray(self):
+        for i in ["<key>key inside an array</key>",
+                  "<key>key inside an array2</key><real>3</real>",
+                  "<true/><key>key inside an array3</key>"]:
+            self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+                              ("<plist><array>%s</array></plist>"%i).encode())
+
+    def test_invaliddict(self):
+        for i in ["<key><true/>k</key><string>compound key</string>",
+                  "<key>single key</key>",
+                  "<string>missing key</string>",
+                  "<key>k1</key><string>v1</string><real>5.3</real>"
+                  "<key>k1</key><key>k2</key><string>double key</string>"]:
+            self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+                              ("<plist><dict>%s</dict></plist>"%i).encode())
+            self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+                              ("<plist><array><dict>%s</dict></array></plist>"%i).encode())
+
+    def test_invalidinteger(self):
+        self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+                          b"<plist><integer>not integer</integer></plist>")
+
+    def test_invalidreal(self):
+        self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+                          b"<plist><integer>not real</integer></plist>")
+
 
 def test_main():
     support.run_unittest(TestPlistlib)

Lib/test/test_subprocess.py

@@ -1399,15 +1399,23 @@ class ProcessTestCaseNoPoll(ProcessTestCase):
 @unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False),
                      "_posixsubprocess extension module not found.")
 class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase):
-    def setUp(self):
-        subprocess._posixsubprocess = None
-        ProcessTestCase.setUp(self)
-        POSIXProcessTestCase.setUp(self)
-
-    def tearDown(self):
-        subprocess._posixsubprocess = sys.modules['_posixsubprocess']
-        POSIXProcessTestCase.tearDown(self)
-        ProcessTestCase.tearDown(self)
+    @classmethod
+    def setUpClass(cls):
+        global subprocess
+        assert subprocess._posixsubprocess
+        # Reimport subprocess while forcing _posixsubprocess to not exist.
+        with support.check_warnings(('.*_posixsubprocess .* not being used.*',
+                                     RuntimeWarning)):
+            subprocess = support.import_fresh_module(
+                    'subprocess', blocked=['_posixsubprocess'])
+        assert not subprocess._posixsubprocess
+
+    @classmethod
+    def tearDownClass(cls):
+        global subprocess
+        # Reimport subprocess as it should be, restoring order to the universe.
+        subprocess = support.import_fresh_module('subprocess')
+        assert subprocess._posixsubprocess
 
 
 class HelperFunctionTests(unittest.TestCase):

Lib/test/test_threading.py

@@ -12,6 +12,7 @@ import unittest
 import weakref
 import os
 from test.script_helper import assert_python_ok, assert_python_failure
+import subprocess
 
 from test import lock_tests
 
@@ -691,6 +692,37 @@ class ThreadingExceptionTests(BaseTestCase):
         thread.start()
         self.assertRaises(RuntimeError, setattr, thread, "daemon", True)
 
+    @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem')
+    def test_recursion_limit(self):
+        # Issue 9670
+        # test that excessive recursion within a non-main thread causes
+        # an exception rather than crashing the interpreter on platforms
+        # like Mac OS X or FreeBSD which have small default stack sizes
+        # for threads
+        script = """if True:
+            import threading
+
+            def recurse():
+                return recurse()
+
+            def outer():
+                try:
+                    recurse()
+                except RuntimeError:
+                    pass
+
+            w = threading.Thread(target=outer)
+            w.start()
+            w.join()
+            print('end of main thread')
+            """
+        expected_output = "end of main thread\n"
+        p = subprocess.Popen([sys.executable, "-c", script],
+                             stdout=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        data = stdout.decode().replace('\r', '')
+        self.assertEqual(p.returncode, 0, "Unexpected error")
+        self.assertEqual(data, expected_output)
 
 class LockTests(lock_tests.LockTests):
     locktype = staticmethod(threading.Lock)

Mac/Makefile.in

@@ -76,6 +76,13 @@ installunixtools:
 	do \
 		ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
 	done
+ifneq ($(LIPO_32BIT_FLAGS),)
+	for fn in python3-32 pythonw3-32 \
+		  python$(VERSION)-32 pythonw$(VERSION)-32 ;\
+	do \
+		ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
+	done
+endif
 
 
 #
@@ -90,6 +97,12 @@ altinstallunixtools:
 	do \
 		ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
 	done
+ifneq ($(LIPO_32BIT_FLAGS),)
+	for fn in python$(VERSION)-32 pythonw$(VERSION)-32 ;\
+	do \
+		ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\
+	done
+endif
 
 pythonw: $(srcdir)/Tools/pythonw.c Makefile
 	$(CC) $(LDFLAGS) -DPYTHONFRAMEWORK='"$(PYTHONFRAMEWORK)"' -o $@ $(srcdir)/Tools/pythonw.c -I.. -I$(srcdir)/../Include ../$(PYTHONFRAMEWORK).framework/Versions/$(VERSION)/$(PYTHONFRAMEWORK)

Misc/ACKS

@@ -608,6 +608,7 @@ Paul Moore
 Derek Morr
 James A Morrison
 Pablo Mouzo
+Mher Movsisyan
 Sjoerd Mullender
 Sape Mullender
 Michael Muller

Misc/NEWS

@@ -10,9 +10,17 @@ What's New in Python 3.2.1 release candidate 2?
 Core and Builtins
 -----------------
 
+- Issue #9670: Increase the default stack size for secondary threads on
+  Mac OS X and FreeBSD to reduce the chances of a crash instead of a
+  "maximum recursion depth" RuntimeError exception.
+  (patch by Ronald Oussoren)
+
 Library
 -------
 
+- Issue #985064: Make plistlib more resilient to faulty input plists.
+  Patch by Mher Movsisyan.
+
 - Issue #12175: RawIOBase.readall() now returns None if read() returns None.
 
 - Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError
@@ -36,6 +44,12 @@ Library
 - Issue #12065: connect_ex() on an SSL socket now returns the original errno
   when the socket's timeout expires (it used to return None).
 
+Build
+-----
+
+- Issue #11217: For 64-bit/32-bit Mac OS X universal framework builds,
+  ensure "make install" creates symlinks in --prefix bin for the "-32"
+  files in the framework bin directory like the installer does.
 
 
 What's New in Python 3.2.1 release candidate 1?

Python/compile.c

@@ -1962,7 +1962,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
     compiler_use_next_block(c, except);
     for (i = 0; i < n; i++) {
         excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
-                                        s->v.TryExcept.handlers, i);
+            s->v.TryExcept.handlers, i);
         if (!handler->v.ExceptHandler.type && i < n-1)
             return compiler_error(c, "default 'except:' must be last");
         c->u->u_lineno_set = 0;
@@ -1979,70 +1979,70 @@ compiler_try_except(struct compiler *c, stmt_ty s)
         }
         ADDOP(c, POP_TOP);
         if (handler->v.ExceptHandler.name) {
-        basicblock *cleanup_end, *cleanup_body;
+            basicblock *cleanup_end, *cleanup_body;
 
-        cleanup_end = compiler_new_block(c);
-        cleanup_body = compiler_new_block(c);
-        if(!(cleanup_end || cleanup_body))
-        return 0;
-
-        compiler_nameop(c, handler->v.ExceptHandler.name, Store);
-        ADDOP(c, POP_TOP);
+            cleanup_end = compiler_new_block(c);
+            cleanup_body = compiler_new_block(c);
+            if (!(cleanup_end || cleanup_body))
+                return 0;
 
-        /*
-        try:
-            # body
-        except type as name:
-            try:
-            # body
-            finally:
-            name = None
-            del name
-        */
+            compiler_nameop(c, handler->v.ExceptHandler.name, Store);
+            ADDOP(c, POP_TOP);
 
-        /* second try: */
-        ADDOP_JREL(c, SETUP_FINALLY, cleanup_end);
-        compiler_use_next_block(c, cleanup_body);
-        if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
-            return 0;
+            /*
+              try:
+              # body
+              except type as name:
+              try:
+              # body
+              finally:
+              name = None
+              del name
+            */
+
+            /* second try: */
+            ADDOP_JREL(c, SETUP_FINALLY, cleanup_end);
+            compiler_use_next_block(c, cleanup_body);
+            if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
+                return 0;
 
-        /* second # body */
-        VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
-        ADDOP(c, POP_BLOCK);
-        ADDOP(c, POP_EXCEPT);
-        compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
+            /* second # body */
+            VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
+            ADDOP(c, POP_BLOCK);
+            ADDOP(c, POP_EXCEPT);
+            compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
 
-        /* finally: */
-        ADDOP_O(c, LOAD_CONST, Py_None, consts);
-        compiler_use_next_block(c, cleanup_end);
-        if (!compiler_push_fblock(c, FINALLY_END, cleanup_end))
-            return 0;
+            /* finally: */
+            ADDOP_O(c, LOAD_CONST, Py_None, consts);
+            compiler_use_next_block(c, cleanup_end);
+            if (!compiler_push_fblock(c, FINALLY_END, cleanup_end))
+                return 0;
 
-        /* name = None */
-        ADDOP_O(c, LOAD_CONST, Py_None, consts);
-        compiler_nameop(c, handler->v.ExceptHandler.name, Store);
+            /* name = None */
+            ADDOP_O(c, LOAD_CONST, Py_None, consts);
+            compiler_nameop(c, handler->v.ExceptHandler.name, Store);
 
-        /* del name */
-        compiler_nameop(c, handler->v.ExceptHandler.name, Del);
+            /* del name */
+            compiler_nameop(c, handler->v.ExceptHandler.name, Del);
 
-        ADDOP(c, END_FINALLY);
-        compiler_pop_fblock(c, FINALLY_END, cleanup_end);
+            ADDOP(c, END_FINALLY);
+            compiler_pop_fblock(c, FINALLY_END, cleanup_end);
         }
         else {
-        basicblock *cleanup_body;
+            basicblock *cleanup_body;
 
-        cleanup_body = compiler_new_block(c);
-        if(!cleanup_body)
-        return 0;
+            cleanup_body = compiler_new_block(c);
+            if (!cleanup_body)
+                return 0;
 
             ADDOP(c, POP_TOP);
-        ADDOP(c, POP_TOP);
-        compiler_use_next_block(c, cleanup_body);
-        if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
-            return 0;
+            ADDOP(c, POP_TOP);
+            compiler_use_next_block(c, cleanup_body);
+            if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
+                return 0;
             VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
-        ADDOP(c, POP_EXCEPT);
-        compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
+            ADDOP(c, POP_EXCEPT);
+            compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
         }
         ADDOP_JREL(c, JUMP_FORWARD, end);
         compiler_use_next_block(c, except);

Python/thread_pthread.h

@@ -18,6 +18,18 @@
 #ifndef THREAD_STACK_SIZE
 #define THREAD_STACK_SIZE       0       /* use default stack size */
 #endif
+
+#if (defined(__APPLE__) || defined(__FreeBSD__)) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
+   /* The default stack size for new threads on OSX is small enough that
+    * we'll get hard crashes instead of 'maximum recursion depth exceeded'
+    * exceptions.
+    *
+    * The default stack size below is the minimal stack size where a
+    * simple recursive function doesn't cause a hard crash.
+    */
+#undef  THREAD_STACK_SIZE
+#define THREAD_STACK_SIZE       0x400000
+#endif
 /* for safety, ensure a viable minimum stacksize */
 #define THREAD_STACK_MIN        0x8000  /* 32kB */
 #else  /* !_POSIX_THREAD_ATTR_STACKSIZE */