Merged revisions 72368 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

................
  r72368 | benjamin.peterson | 2009-05-05 18:13:58 -0500 (Tue, 05 May 2009) | 53 lines

  Merged revisions 68503,68507,68694,69054,69673,69679-69681,70991,70999,71003,71695 via svnmerge from
  svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

  ........
    r68503 | benjamin.peterson | 2009-01-10 14:14:49 -0600 (Sat, 10 Jan 2009) | 1 line

    use variable
  ........
    r68507 | benjamin.peterson | 2009-01-10 15:13:16 -0600 (Sat, 10 Jan 2009) | 1 line

    rewrap
  ........
    r68694 | benjamin.peterson | 2009-01-17 17:55:59 -0600 (Sat, 17 Jan 2009) | 1 line

    test for specific node type
  ........
    r69054 | guilherme.polo | 2009-01-28 10:01:54 -0600 (Wed, 28 Jan 2009) | 2 lines

    Added mapping for the ttk module.
  ........
    r69673 | benjamin.peterson | 2009-02-16 09:38:22 -0600 (Mon, 16 Feb 2009) | 1 line

    fix handling of as imports #5279
  ........
    r69679 | benjamin.peterson | 2009-02-16 11:36:06 -0600 (Mon, 16 Feb 2009) | 1 line

    make Base.get_next_sibling() and Base.get_prev_sibling() properties
  ........
    r69680 | benjamin.peterson | 2009-02-16 11:41:48 -0600 (Mon, 16 Feb 2009) | 1 line

    normalize docstrings in pytree according to PEP 11
  ........
    r69681 | benjamin.peterson | 2009-02-16 11:43:09 -0600 (Mon, 16 Feb 2009) | 1 line

    use a set
  ........
    r70991 | benjamin.peterson | 2009-04-01 15:54:50 -0500 (Wed, 01 Apr 2009) | 1 line

    map urllib.urlopen to urllib.request.open #5637
  ........
    r70999 | benjamin.peterson | 2009-04-01 17:36:47 -0500 (Wed, 01 Apr 2009) | 1 line

    add very alpha support to 2to3 for running concurrently with multiprocessing
  ........
    r71003 | benjamin.peterson | 2009-04-01 18:10:43 -0500 (Wed, 01 Apr 2009) | 1 line

    fix when multiprocessing is not available or used
  ........
    r71695 | benjamin.peterson | 2009-04-17 22:21:29 -0500 (Fri, 17 Apr 2009) | 1 line

    refactor multiprocessing support, so it's less hacky to employ and only loads mp when needed
  ........
................

Lib/lib2to3/fixer_util.py

@@ -226,7 +226,7 @@ def is_probably_builtin(node):
     """
     Check that something isn't an attribute or function name etc.
     """
-    prev = node.get_prev_sibling()
+    prev = node.prev_sibling
     if prev is not None and prev.type == token.DOT:
         # Attribute lookup.
         return False

Lib/lib2to3/fixes/fix_except.py

@@ -25,11 +25,11 @@ The following cases will be converted:
 from .. import pytree
 from ..pgen2 import token
 from .. import fixer_base
-from ..fixer_util import Assign, Attr, Name, is_tuple, is_list
+from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms
 
 def find_excepts(nodes):
     for i, n in enumerate(nodes):
-        if isinstance(n, pytree.Node):
+        if n.type == syms.except_clause:
             if n.children[0].value == 'except':
                 yield (n, nodes[i+2])
 

Lib/lib2to3/fixes/fix_imports.py

@@ -27,6 +27,7 @@ MAPPING = {'StringIO':  'io',
            'ScrolledText': 'tkinter.scrolledtext',
            'Tkconstants': 'tkinter.constants',
            'Tix': 'tkinter.tix',
+           'ttk': 'tkinter.ttk',
            'Tkinter': 'tkinter',
            'markupbase': '_markupbase',
            '_winreg': 'winreg',
@@ -121,17 +122,18 @@ class FixImports(fixer_base.BaseFix):
     def transform(self, node, results):
         import_mod = results.get("module_name")
         if import_mod:
-            new_name = self.mapping[import_mod.value]
+            mod_name = import_mod.value
+            new_name = self.mapping[mod_name]
             import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
             if "name_import" in results:
                 # If it's not a "from x import x, y" or "import x as y" import,
                 # marked its usage to be replaced.
-                self.replace[import_mod.value] = new_name
+                self.replace[mod_name] = new_name
             if "multiple_imports" in results:
-                # This is a nasty hack to fix multiple imports on a
-                # line (e.g., "import StringIO, urlparse"). The problem is that I
-                # can't figure out an easy way to make a pattern recognize the
-                # keys of MAPPING randomly sprinkled in an import statement.
+                # This is a nasty hack to fix multiple imports on a line (e.g.,
+                # "import StringIO, urlparse"). The problem is that I can't
+                # figure out an easy way to make a pattern recognize the keys of
+                # MAPPING randomly sprinkled in an import statement.
                 results = self.match(node)
                 if results:
                     self.transform(node, results)

Lib/lib2to3/fixes/fix_itertools_imports.py

 """ Fixer for imports of itertools.(imap|ifilter|izip|ifilterfalse) """
 
 # Local imports
-from .. import fixer_base
-from ..fixer_util import BlankLine
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import BlankLine, syms, token
+
 
 class FixItertoolsImports(fixer_base.BaseFix):
     PATTERN = """
@@ -11,34 +12,40 @@ class FixItertoolsImports(fixer_base.BaseFix):
 
     def transform(self, node, results):
         imports = results['imports']
-        children = imports.children[:] or [imports]
-        for child in children:
-            if not hasattr(child, 'value'):
-                # Handle 'import ... as ...'
-                continue
-            if child.value in ('imap', 'izip', 'ifilter'):
-                # The value must be set to none in case child == import,
-                # so that the test for empty imports will work out
+        if imports.type == syms.import_as_name or not imports.children:
+            children = [imports]
+        else:
+            children = imports.children
+        for child in children[::2]:
+            if child.type == token.NAME:
+                member = child.value
+                name_node = child
+            else:
+                assert child.type == syms.import_as_name
+                name_node = child.children[0]
+            member_name = name_node.value
+            if member_name in ('imap', 'izip', 'ifilter'):
                 child.value = None
                 child.remove()
-            elif child.value == 'ifilterfalse':
+            elif member_name == 'ifilterfalse':
                 node.changed()
-                child.value = 'filterfalse'
+                name_node.value = 'filterfalse'
 
         # Make sure the import statement is still sane
         children = imports.children[:] or [imports]
         remove_comma = True
         for child in children:
-            if remove_comma and getattr(child, 'value', None) == ',':
+            if remove_comma and child.type == token.COMMA:
                 child.remove()
             else:
                 remove_comma ^= True
 
-        if str(children[-1]) == ',':
+        if children[-1].type == token.COMMA:
             children[-1].remove()
 
         # If there are no imports left, just get rid of the entire statement
-        if not (imports.children or getattr(imports, 'value', None)):
+        if not (imports.children or getattr(imports, 'value', None)) or \
+                imports.parent is None:
             p = node.get_prefix()
             node = BlankLine()
             node.prefix = p

Lib/lib2to3/fixes/fix_set_literal.py

@@ -38,7 +38,7 @@ class FixSetLiteral(fixer_base.BaseFix):
         literal.extend(n.clone() for n in items.children)
         literal.append(pytree.Leaf(token.RBRACE, "}"))
         # Set the prefix of the right brace to that of the ')' or ']'
-        literal[-1].set_prefix(items.get_next_sibling().get_prefix())
+        literal[-1].set_prefix(items.next_sibling.get_prefix())
         maker = pytree.Node(syms.dictsetmaker, literal)
         maker.set_prefix(node.get_prefix())
 

Lib/lib2to3/fixes/fix_urllib.py

@@ -12,7 +12,7 @@ from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain
 MAPPING = {'urllib':  [
                 ('urllib.request',
                     ['URLOpener', 'FancyURLOpener', 'urlretrieve',
-                     '_urlopener', 'urlcleanup']),
+                     '_urlopener', 'urlopen', 'urlcleanup']),
                 ('urllib.parse',
                     ['quote', 'quote_plus', 'unquote', 'unquote_plus',
                      'urlencode', 'pathname2url', 'url2pathname', 'splitattr',

Lib/lib2to3/main.py

@@ -10,8 +10,7 @@ import optparse
 
 from . import refactor
 
-
-class StdoutRefactoringTool(refactor.RefactoringTool):
+class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
     """
     Prints output to stdout.
     """
@@ -64,6 +63,8 @@ def main(fixer_pkg, args=None):
                       help="Fix up doctests only")
     parser.add_option("-f", "--fix", action="append", default=[],
                       help="Each FIX specifies a transformation; default: all")
+    parser.add_option("-j", "--processes", action="store", default=1,
+                      type="int", help="Run 2to3 concurrently")
     parser.add_option("-x", "--nofix", action="append", default=[],
                       help="Prevent a fixer from being run.")
     parser.add_option("-l", "--list-fixes", action="store_true",
@@ -126,7 +127,14 @@ def main(fixer_pkg, args=None):
         if refactor_stdin:
             rt.refactor_stdin()
         else:
-            rt.refactor(args, options.write, options.doctests_only)
+            try:
+                rt.refactor(args, options.write, options.doctests_only,
+                            options.processes)
+            except refactor.MultiprocessingUnsupported:
+                assert options.processes > 1
+                print >> sys.stderr, "Sorry, -j isn't " \
+                    "supported on this platform."
+                return 1
         rt.summarize()
 
     # Return error status (0 if rt.errors is zero)

Lib/lib2to3/patcomp.py

@@ -30,7 +30,7 @@ _PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
 
 def tokenize_wrapper(input):
     """Tokenizes a string suppressing significant whitespace."""
-    skip = (token.NEWLINE, token.INDENT, token.DEDENT)
+    skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
     tokens = tokenize.generate_tokens(driver.generate_lines(input).__next__)
     for quintuple in tokens:
         type, value, start, end, line_text = quintuple

Lib/lib2to3/refactor.py

@@ -506,6 +506,63 @@ class RefactoringTool(object):
             yield ""
 
 
+class MultiprocessingUnsupported(Exception):
+    pass
+
+
+class MultiprocessRefactoringTool(RefactoringTool):
+
+    def __init__(self, *args, **kwargs):
+        super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
+        self.queue = None
+
+    def refactor(self, items, write=False, doctests_only=False,
+                 num_processes=1):
+        if num_processes == 1:
+            return super(MultiprocessRefactoringTool, self).refactor(
+                items, write, doctests_only)
+        try:
+            import multiprocessing
+        except ImportError:
+            raise MultiprocessingUnsupported
+        if self.queue is not None:
+            raise RuntimeError("already doing multiple processes")
+        self.queue = multiprocessing.JoinableQueue()
+        processes = [multiprocessing.Process(target=self._child)
+                     for i in xrange(num_processes)]
+        try:
+            for p in processes:
+                p.start()
+            super(MultiprocessRefactoringTool, self).refactor(items, write,
+                                                              doctests_only)
+        finally:
+            self.queue.join()
+            for i in xrange(num_processes):
+                self.queue.put(None)
+            for p in processes:
+                if p.is_alive():
+                    p.join()
+            self.queue = None
+
+    def _child(self):
+        task = self.queue.get()
+        while task is not None:
+            args, kwargs = task
+            try:
+                super(MultiprocessRefactoringTool, self).refactor_file(
+                    *args, **kwargs)
+            finally:
+                self.queue.task_done()
+            task = self.queue.get()
+
+    def refactor_file(self, *args, **kwargs):
+        if self.queue is not None:
+            self.queue.put((args, kwargs))
+        else:
+            return super(MultiprocessRefactoringTool, self).refactor_file(
+                *args, **kwargs)
+
+
 def diff_texts(a, b, filename):
     """Return a unified diff of two strings."""
     a = a.splitlines()

Lib/lib2to3/tests/test_fixers.py

@@ -3404,6 +3404,18 @@ class Test_itertools_imports(FixerTestCase):
         a = "from itertools import bar as bang"
         self.check(b, a)
 
+        b = "from itertools import izip as _zip, imap, bar"
+        a = "from itertools import bar"
+        self.check(b, a)
+
+        b = "from itertools import imap as _map"
+        a = ""
+        self.check(b, a)
+
+        b = "from itertools import imap as _map, izip as _zip"
+        a = ""
+        self.check(b, a)
+
         s = "from itertools import bar as bang"
         self.unchanged(s)
 

Lib/lib2to3/tests/test_pytree.py

@@ -306,36 +306,36 @@ class TestNodes(support.TestCase):
         n2 = pytree.Node(1000, [])
         p1 = pytree.Node(1000, [n1, n2])
 
-        self.failUnless(n1.get_next_sibling() is n2)
-        self.assertEqual(n2.get_next_sibling(), None)
-        self.assertEqual(p1.get_next_sibling(), None)
+        self.failUnless(n1.next_sibling is n2)
+        self.assertEqual(n2.next_sibling, None)
+        self.assertEqual(p1.next_sibling, None)
 
     def testLeafNextSibling(self):
         l1 = pytree.Leaf(100, "a")
         l2 = pytree.Leaf(100, "b")
         p1 = pytree.Node(1000, [l1, l2])
 
-        self.failUnless(l1.get_next_sibling() is l2)
-        self.assertEqual(l2.get_next_sibling(), None)
-        self.assertEqual(p1.get_next_sibling(), None)
+        self.failUnless(l1.next_sibling is l2)
+        self.assertEqual(l2.next_sibling, None)
+        self.assertEqual(p1.next_sibling, None)
 
     def testNodePrevSibling(self):
         n1 = pytree.Node(1000, [])
         n2 = pytree.Node(1000, [])
         p1 = pytree.Node(1000, [n1, n2])
 
-        self.failUnless(n2.get_prev_sibling() is n1)
-        self.assertEqual(n1.get_prev_sibling(), None)
-        self.assertEqual(p1.get_prev_sibling(), None)
+        self.failUnless(n2.prev_sibling is n1)
+        self.assertEqual(n1.prev_sibling, None)
+        self.assertEqual(p1.prev_sibling, None)
 
     def testLeafPrevSibling(self):
         l1 = pytree.Leaf(100, "a")
         l2 = pytree.Leaf(100, "b")
         p1 = pytree.Node(1000, [l1, l2])
 
-        self.failUnless(l2.get_prev_sibling() is l1)
-        self.assertEqual(l1.get_prev_sibling(), None)
-        self.assertEqual(p1.get_prev_sibling(), None)
+        self.failUnless(l2.prev_sibling is l1)
+        self.assertEqual(l1.prev_sibling, None)
+        self.assertEqual(p1.prev_sibling, None)
 
 
 class TestPatterns(support.TestCase):