Issue #13930: Adds ability for 2to3 to write its output to a different

directory tree instead of overwriting the input files.  Adds three command
line options: -o/--output-dir, -W/--write-unchanged-files and --add-suffix.

Feature backports into stable release branches for 2to3 are allowed by
a special exemption:
 http://mail.python.org/pipermail/python-dev/2011-December/115089.html

Doc/library/2to3.rst

@@ -94,6 +94,38 @@ change can also be enabled manually with the :option:`-p` flag.  Use
 :option:`-p` to run fixers on code that already has had its print statements
 converted.
 
+The :option:`-o` or :option:`--output-dir` option allows specification of an
+alternate directory for processed output files to be written to.  The
+:option:`-n` flag is required when using this as backup files do not make sense
+when not overwriting the input files.
+
+.. versionadded:: 3.2.3
+   The :option:`-o` option was added.
+
+The :option:`-W` or :option:`--write-unchanged-files` flag tells 2to3 to always
+write output files even if no changes were required to the file.  This is most
+useful with :option:`-o` so that an entire Python source tree is copied with
+translation from one directory to another.
+This option implies the :option:`-w` flag as it would not make sense otherwise.
+
+.. versionadded:: 3.2.3
+   The :option:`-W` flag was added.
+
+The :option:`--add-suffix` option specifies a string to append to all output
+filenames.  The :option:`-n` flag is required when specifying this as backups
+are not necessary when writing to different filenames.  Example::
+
+   $ 2to3 -n -W --add-suffix=3 example.py
+
+Will cause a converted file named ``example.py3`` to be written.
+
+.. versionadded:: 3.2.3
+   The :option:`--add-suffix` option was added.
+
+To translate an entire project from one directory tree to another use::
+
+   $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode
+
 
 .. _2to3-fixers:
 

Lib/lib2to3/main.py

@@ -25,12 +25,41 @@ def diff_texts(a, b, filename):
 
 class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
     """
+    A refactoring tool that can avoid overwriting its input files.
     Prints output to stdout.
+
+    Output files can optionally be written to a different directory and or
+    have an extra file suffix appended to their name for use in situations
+    where you do not want to replace the input files.
     """
 
-    def __init__(self, fixers, options, explicit, nobackups, show_diffs):
+    def __init__(self, fixers, options, explicit, nobackups, show_diffs,
+                 input_base_dir='', output_dir='', append_suffix=''):
+        """
+        Args:
+            fixers: A list of fixers to import.
+            options: A dict with RefactoringTool configuration.
+            explicit: A list of fixers to run even if they are explicit.
+            nobackups: If true no backup '.bak' files will be created for those
+                files that are being refactored.
+            show_diffs: Should diffs of the refactoring be printed to stdout?
+            input_base_dir: The base directory for all input files.  This class
+                will strip this path prefix off of filenames before substituting
+                it with output_dir.  Only meaningful if output_dir is supplied.
+                All files processed by refactor() must start with this path.
+            output_dir: If supplied, all converted files will be written into
+                this directory tree instead of input_base_dir.
+            append_suffix: If supplied, all files output by this tool will have
+                this appended to their filename.  Useful for changing .py to
+                .py3 for example by passing append_suffix='3'.
+        """
         self.nobackups = nobackups
         self.show_diffs = show_diffs
+        if input_base_dir and not input_base_dir.endswith(os.sep):
+            input_base_dir += os.sep
+        self._input_base_dir = input_base_dir
+        self._output_dir = output_dir
+        self._append_suffix = append_suffix
         super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
 
     def log_error(self, msg, *args, **kwargs):
@@ -38,6 +67,23 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
         self.logger.error(msg, *args, **kwargs)
 
     def write_file(self, new_text, filename, old_text, encoding):
+        orig_filename = filename
+        if self._output_dir:
+            if filename.startswith(self._input_base_dir):
+                filename = os.path.join(self._output_dir,
+                                        filename[len(self._input_base_dir):])
+            else:
+                raise ValueError('filename %s does not start with the '
+                                 'input_base_dir %s' % (
+                                         filename, self._input_base_dir))
+        if self._append_suffix:
+            filename += self._append_suffix
+        if orig_filename != filename:
+          output_dir = os.path.dirname(filename)
+          if not os.path.isdir(output_dir):
+              os.makedirs(output_dir)
+          self.log_message('Writing converted %s to %s.', orig_filename,
+                           filename)
         if not self.nobackups:
             # Make backup
             backup = filename + ".bak"
@@ -55,6 +101,9 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
         write(new_text, filename, old_text, encoding)
         if not self.nobackups:
             shutil.copymode(backup, filename)
+        if orig_filename != filename:
+            # Preserve the file mode in the new output directory.
+            shutil.copymode(orig_filename, filename)
 
     def print_output(self, old, new, filename, equal):
         if equal:
@@ -113,11 +162,33 @@ def main(fixer_pkg, args=None):
                       help="Write back modified files")
     parser.add_option("-n", "--nobackups", action="store_true", default=False,
                       help="Don't write backups for modified files")
+    parser.add_option("-o", "--output-dir", action="store", type="str",
+                      default="", help="Put output files in this directory "
+                      "instead of overwriting the input files.  Requires -n.")
+    parser.add_option("-W", "--write-unchanged-files", action="store_true",
+                      help="Also write files even if no changes were required"
+                      " (useful with --output-dir); implies -w.")
+    parser.add_option("--add-suffix", action="store", type="str", default="",
+                      help="Append this string to all output filenames."
+                      " Requires -n if non-empty.  "
+                      "ex: --add-suffix='3' will generate .py3 files.")
 
     # Parse command line arguments
     refactor_stdin = False
     flags = {}
     options, args = parser.parse_args(args)
+    if options.write_unchanged_files:
+        flags["write_unchanged_files"] = True
+        if not options.write:
+            warn("--write-unchanged-files/-W implies -w.")
+        options.write = True
+    # If we allowed these, the original files would be renamed to backup names
+    # but not replaced.
+    if options.output_dir and not options.nobackups:
+        parser.error("Can't use --output-dir/-o without -n.")
+    if options.add_suffix and not options.nobackups:
+        parser.error("Can't use --add-suffix without -n.")
+
     if not options.write and options.no_diffs:
         warn("not writing files and not printing diffs; that's not very useful")
     if not options.write and options.nobackups:
@@ -143,6 +214,7 @@ def main(fixer_pkg, args=None):
     # Set up logging handler
     level = logging.DEBUG if options.verbose else logging.INFO
     logging.basicConfig(format='%(name)s: %(message)s', level=level)
+    logger = logging.getLogger('lib2to3.main')
 
     # Initialize the refactoring tool
     avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
@@ -159,8 +231,23 @@ def main(fixer_pkg, args=None):
     else:
         requested = avail_fixes.union(explicit)
     fixer_names = requested.difference(unwanted_fixes)
-    rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
-                               options.nobackups, not options.no_diffs)
+    input_base_dir = os.path.commonprefix(args)
+    if (input_base_dir and not input_base_dir.endswith(os.sep)
+        and not os.path.isdir(input_base_dir)):
+        # One or more similar names were passed, their directory is the base.
+        # os.path.commonprefix() is ignorant of path elements, this corrects
+        # for that weird API.
+        input_base_dir = os.path.dirname(input_base_dir)
+    if options.output_dir:
+        input_base_dir = input_base_dir.rstrip(os.sep)
+        logger.info('Output in %r will mirror the input directory %r layout.',
+                    options.output_dir, input_base_dir)
+    rt = StdoutRefactoringTool(
+            sorted(fixer_names), flags, sorted(explicit),
+            options.nobackups, not options.no_diffs,
+            input_base_dir=input_base_dir,
+            output_dir=options.output_dir,
+            append_suffix=options.add_suffix)
 
     # Refactor all files and directories passed as arguments
     if not rt.errors:

Lib/lib2to3/refactor.py

@@ -173,7 +173,8 @@ class FixerError(Exception):
 
 class RefactoringTool(object):
 
-    _default_options = {"print_function" : False}
+    _default_options = {"print_function" : False,
+                        "write_unchanged_files" : False}
 
     CLASS_PREFIX = "Fix" # The prefix for fixer classes
     FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -195,6 +196,10 @@ class RefactoringTool(object):
             self.grammar = pygram.python_grammar_no_print_statement
         else:
             self.grammar = pygram.python_grammar
+        # When this is True, the refactor*() methods will call write_file() for
+        # files processed even if they were not changed during refactoring. If
+        # and only if the refactor method's write parameter was True.
+        self.write_unchanged_files = self.options.get("write_unchanged_files")
         self.errors = []
         self.logger = logging.getLogger("RefactoringTool")
         self.fixer_log = []
@@ -341,13 +346,13 @@ class RefactoringTool(object):
         if doctests_only:
             self.log_debug("Refactoring doctests in %s", filename)
             output = self.refactor_docstring(input, filename)
-            if output != input:
+            if self.write_unchanged_files or output != input:
                 self.processed_file(output, filename, input, write, encoding)
             else:
                 self.log_debug("No doctest changes in %s", filename)
         else:
             tree = self.refactor_string(input, filename)
-            if tree and tree.was_changed:
+            if self.write_unchanged_files or (tree and tree.was_changed):
                 # The [:-1] is to take off the \n we added earlier
                 self.processed_file(str(tree)[:-1], filename,
                                     write=write, encoding=encoding)
@@ -386,13 +391,13 @@ class RefactoringTool(object):
         if doctests_only:
             self.log_debug("Refactoring doctests in stdin")
             output = self.refactor_docstring(input, "<stdin>")
-            if output != input:
+            if self.write_unchanged_files or output != input:
                 self.processed_file(output, "<stdin>", input)
             else:
                 self.log_debug("No doctest changes in stdin")
         else:
             tree = self.refactor_string(input, "<stdin>")
-            if tree and tree.was_changed:
+            if self.write_unchanged_files or (tree and tree.was_changed):
                 self.processed_file(str(tree), "<stdin>", input)
             else:
                 self.log_debug("No changes in stdin")
@@ -502,7 +507,7 @@ class RefactoringTool(object):
     def processed_file(self, new_text, filename, old_text=None, write=False,
                        encoding=None):
         """
-        Called when a file has been refactored, and there are changes.
+        Called when a file has been refactored and there may be changes.
         """
         self.files.append(filename)
         if old_text is None:
@@ -513,6 +518,7 @@ class RefactoringTool(object):
         self.print_output(old_text, new_text, filename, equal)
         if equal:
             self.log_debug("No changes to %s", filename)
+            if not self.write_unchanged_files:
                 return
         if write:
             self.write_file(new_text, filename, old_text, encoding)

Lib/lib2to3/tests/test_main.py

 # -*- coding: utf-8 -*-
-import sys
 import codecs
-import logging
 import io
+import logging
+import os
+import shutil
+import sys
+import tempfile
 import unittest
 
 from lib2to3 import main
 
 
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py")
+
+
 class TestMain(unittest.TestCase):
 
+    def setUp(self):
+        self.temp_dir = None  # tearDown() will rmtree this directory if set.
+
     def tearDown(self):
         # Clean up logging configuration down by main.
         del logging.root.handlers[:]
+        if self.temp_dir:
+            shutil.rmtree(self.temp_dir)
 
     def run_2to3_capture(self, args, in_capture, out_capture, err_capture):
         save_stdin = sys.stdin
@@ -39,3 +51,85 @@ class TestMain(unittest.TestCase):
         self.assertTrue("-print 'nothing'" in output)
         self.assertTrue("WARNING: couldn't encode <stdin>'s diff for "
                         "your terminal" in err.getvalue())
+
+    def setup_test_source_trees(self):
+        """Setup a test source tree and output destination tree."""
+        self.temp_dir = tempfile.mkdtemp()  # tearDown() cleans this up.
+        self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
+        self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
+        os.mkdir(self.py2_src_dir)
+        os.mkdir(self.py3_dest_dir)
+        # Turn it into a package with a few files.
+        self.setup_files = []
+        open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
+        self.setup_files.append("__init__.py")
+        shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
+        self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
+        self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
+        self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
+        with open(self.trivial_py2_file, "w") as trivial:
+            trivial.write("print 'I need a simple conversion.'")
+        self.setup_files.append("trivial.py")
+
+    def test_filename_changing_on_output_single_dir(self):
+        """2to3 a single directory with a new output dir and suffix."""
+        self.setup_test_source_trees()
+        out = io.StringIO()
+        err = io.StringIO()
+        suffix = "TEST"
+        ret = self.run_2to3_capture(
+                ["-n", "--add-suffix", suffix, "--write-unchanged-files",
+                 "--no-diffs", "--output-dir",
+                 self.py3_dest_dir, self.py2_src_dir],
+                io.StringIO(""), out, err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(" implies -w.", stderr)
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(set(name+suffix for name in self.setup_files),
+                         set(os.listdir(self.py3_dest_dir)))
+        for name in self.setup_files:
+            self.assertIn("Writing converted %s to %s" % (
+                    os.path.join(self.py2_src_dir, name),
+                    os.path.join(self.py3_dest_dir, name+suffix)), stderr)
+        self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py")
+        self.assertNotRegex(stderr, r"No changes to .*/trivial\.py")
+
+    def test_filename_changing_on_output_two_files(self):
+        """2to3 two files in one directory with a new output dir."""
+        self.setup_test_source_trees()
+        err = io.StringIO()
+        py2_files = [self.trivial_py2_file, self.init_py2_file]
+        expected_files = set(os.path.basename(name) for name in py2_files)
+        ret = self.run_2to3_capture(
+                ["-n", "-w", "--write-unchanged-files",
+                 "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,
+                io.StringIO(""), io.StringIO(), err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))
+
+    def test_filename_changing_on_output_single_file(self):
+        """2to3 a single file with a new output dir."""
+        self.setup_test_source_trees()
+        err = io.StringIO()
+        ret = self.run_2to3_capture(
+                ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,
+                 self.trivial_py2_file],
+                io.StringIO(""), io.StringIO(), err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),
+                         set(os.listdir(self.py3_dest_dir)))
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/lib2to3/tests/test_refactor.py

@@ -53,6 +53,12 @@ class TestRefactoringTool(unittest.TestCase):
         self.assertTrue(rt.driver.grammar is
                         pygram.python_grammar_no_print_statement)
 
+    def test_write_unchanged_files_option(self):
+        rt = self.rt()
+        self.assertFalse(rt.write_unchanged_files)
+        rt = self.rt({"write_unchanged_files" : True})
+        self.assertTrue(rt.write_unchanged_files)
+
     def test_fixer_loading_helpers(self):
         contents = ["explicit", "first", "last", "parrot", "preorder"]
         non_prefixed = refactor.get_all_fix_names("myfixes")
@@ -176,7 +182,9 @@ from __future__ import print_function"""
                     "<stdin>", False]
         self.assertEqual(results, expected)
 
-    def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
+    def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,
+                               options=None, mock_log_debug=None,
+                               actually_write=True):
         tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")
         self.addCleanup(shutil.rmtree, tmpdir)
         # make a copy of the tested file that we can write to
@@ -189,11 +197,15 @@ from __future__ import print_function"""
                 return fp.read()
 
         old_contents = read_file()
-        rt = self.rt(fixers=fixers)
+        rt = self.rt(fixers=fixers, options=options)
+        if mock_log_debug:
+            rt.log_debug = mock_log_debug
 
         rt.refactor_file(test_file)
         self.assertEqual(old_contents, read_file())
 
+        if not actually_write:
+            return
         rt.refactor_file(test_file, True)
         new_contents = read_file()
         self.assertNotEqual(old_contents, new_contents)
@@ -203,6 +215,26 @@ from __future__ import print_function"""
         test_file = os.path.join(FIXER_DIR, "parrot_example.py")
         self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
 
+    def test_refactor_file_write_unchanged_file(self):
+        test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+        debug_messages = []
+        def recording_log_debug(msg, *args):
+            debug_messages.append(msg % args)
+        self.check_file_refactoring(test_file, fixers=(),
+                                    options={"write_unchanged_files": True},
+                                    mock_log_debug=recording_log_debug,
+                                    actually_write=False)
+        # Testing that it logged this message when write=False was passed is
+        # sufficient to see that it did not bail early after "No changes".
+        message_regex = r"Not writing changes to .*%s%s" % (
+                os.sep, os.path.basename(test_file))
+        for message in debug_messages:
+            if "Not writing changes" in message:
+                self.assertRegexpMatches(message, message_regex)
+                break
+        else:
+            self.fail("%r not matched in %r" % (message_regex, debug_messages))
+
     def test_refactor_dir(self):
         def check(structure, expected):
             def mock_refactor_file(self, f, *args):

Misc/NEWS

@@ -443,6 +443,11 @@ Build
 Tools/Demos
 -----------
 
+- Issue #13930: 2to3 is now able to write its converted output files to another
+  directory tree as well as copying unchanged files and altering the file
+  suffix.  See its new -o, -W and --add-suffix options.  This makes it more
+  useful in many automated code translation workflows.
+
 - Issue #13628: python-gdb.py is now able to retrieve more frames in the Python
   traceback if Python is optimized.