More comprehensive compileall cli tests, and fixes.

Lib/compileall.py

@@ -181,24 +181,29 @@ def main():
                               'of the file'))
     parser.add_argument('-i', metavar='FILE', dest='flist',
                         help='expand the list with the content of FILE.')
-    parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='?')
+    parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*')
     args = parser.parse_args()
 
-    if (args.ddir and args.compile_dest != 1 and
-        not os.path.isdir(args.compile_dest)):
-        raise argparse.ArgumentError(
-            "-d destdir requires exactly one directory argument")
+    compile_dests = args.compile_dest
+
+    if (args.ddir and (len(compile_dests) != 1
+            or not os.path.isdir(compile_dests[0]))):
+        parser.exit('-d destdir requires exactly one directory argument')
     if args.rx:
         import re
         args.rx = re.compile(args.rx)
 
     # if flist is provided then load it
-    compile_dests = [args.compile_dest]
     if args.flist:
-        with open(args.flist) as f:
-            files = f.read().split()
-            compile_dests.extend(files)
-
+        try:
+            with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
+                for line in f:
+                    compile_dests.append(line.strip())
+        except EnvironmentError:
+            print("Error reading file list {}".format(args.flist))
+            return False
+
+    success = True
     try:
         if compile_dests:
             for dest in compile_dests:
@@ -206,17 +211,18 @@ def main():
                     if not compile_dir(dest, args.maxlevels, args.ddir,
                                        args.force, args.rx, args.quiet,
                                        args.legacy):
-                        return 0
+                        success = False
                 else:
                     if not compile_file(dest, args.ddir, args.force, args.rx,
                                         args.quiet, args.legacy):
-                        return 0
+                        success = False
+            return success
         else:
             return compile_path(legacy=args.legacy)
     except KeyboardInterrupt:
         print("\n[interrupted]")
-        return 0
-    return 1
+        return False
+    return True
 
 
 if __name__ == '__main__':

Lib/test/test_compileall.py

@@ -11,7 +11,7 @@ import time
 import unittest
 import io
 
-from test import support
+from test import support, script_helper
 
 class CompileallTests(unittest.TestCase):
 
@@ -124,16 +124,31 @@ class EncodingTest(unittest.TestCase):
 class CommandLineTests(unittest.TestCase):
     """Test compileall's CLI."""
 
+    def assertRunOK(self, *args):
+        rc, out, err = script_helper.assert_python_ok('-m', 'compileall', *args)
+        self.assertEqual(b'', err)
+        return out
+
+    def assertRunNotOK(self, *args):
+        rc, out, err = script_helper.assert_python_failure(
+                         '-m', 'compileall', *args)
+        return rc, out, err
+
+    def assertCompiled(self, fn):
+        self.assertTrue(os.path.exists(imp.cache_from_source(fn)))
+
+    def assertNotCompiled(self, fn):
+        self.assertFalse(os.path.exists(imp.cache_from_source(fn)))
+
     def setUp(self):
         self.addCleanup(self._cleanup)
         self.directory = tempfile.mkdtemp()
         self.pkgdir = os.path.join(self.directory, 'foo')
         os.mkdir(self.pkgdir)
-        # Touch the __init__.py and a package module.
-        with open(os.path.join(self.pkgdir, '__init__.py'), 'w'):
-            pass
-        with open(os.path.join(self.pkgdir, 'bar.py'), 'w'):
-            pass
+        self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__')
+        # Create the __init__.py and a package module.
+        self.initfn = script_helper.make_script(self.pkgdir, '__init__', '')
+        self.barfn = script_helper.make_script(self.pkgdir, 'bar', '')
         sys.path.insert(0, self.directory)
 
     def _cleanup(self):
@@ -149,103 +164,169 @@ class CommandLineTests(unittest.TestCase):
         ('doubleoptimize', 'pyo', ['-OO']),
     ]:
         def f(self, ext=ext, switch=switch):
-            retcode = subprocess.call(
-                [sys.executable] + switch +
-                ['-m', 'compileall', '-q', self.pkgdir])
-            self.assertEqual(retcode, 0)
+            script_helper.assert_python_ok(*(switch +
+                ['-m', 'compileall', '-q', self.pkgdir]))
             # Verify the __pycache__ directory contents.
-            cachedir = os.path.join(self.pkgdir, '__pycache__')
-            self.assertTrue(os.path.exists(cachedir))
+            self.assertTrue(os.path.exists(self.pkgdir_cachedir))
             expected = sorted(base.format(imp.get_tag(), ext) for base in
                               ('__init__.{}.{}', 'bar.{}.{}'))
-            self.assertEqual(sorted(os.listdir(cachedir)), expected)
+            self.assertEqual(sorted(os.listdir(self.pkgdir_cachedir)), expected)
             # Make sure there are no .pyc files in the source directory.
-            self.assertFalse([pyc_file for pyc_file in os.listdir(self.pkgdir)
-                              if pyc_file.endswith(ext)])
+            self.assertFalse([fn for fn in os.listdir(self.pkgdir)
+                              if fn.endswith(ext)])
         locals()['test_pep3147_paths_' + name] = f
 
     def test_legacy_paths(self):
         # Ensure that with the proper switch, compileall leaves legacy
         # pyc/pyo files, and no __pycache__ directory.
-        retcode = subprocess.call(
-            (sys.executable, '-m', 'compileall', '-b', '-q', self.pkgdir))
-        self.assertEqual(retcode, 0)
+        self.assertRunOK('-b', '-q', self.pkgdir)
         # Verify the __pycache__ directory contents.
-        cachedir = os.path.join(self.pkgdir, '__pycache__')
-        self.assertFalse(os.path.exists(cachedir))
+        self.assertFalse(os.path.exists(self.pkgdir_cachedir))
         expected = sorted(['__init__.py', '__init__.pyc', 'bar.py', 'bar.pyc'])
         self.assertEqual(sorted(os.listdir(self.pkgdir)), expected)
 
     def test_multiple_runs(self):
         # Bug 8527 reported that multiple calls produced empty
         # __pycache__/__pycache__ directories.
-        retcode = subprocess.call(
-            (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
-        self.assertEqual(retcode, 0)
+        self.assertRunOK('-q', self.pkgdir)
         # Verify the __pycache__ directory contents.
-        cachedir = os.path.join(self.pkgdir, '__pycache__')
-        self.assertTrue(os.path.exists(cachedir))
-        cachecachedir = os.path.join(cachedir, '__pycache__')
+        self.assertTrue(os.path.exists(self.pkgdir_cachedir))
+        cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__')
         self.assertFalse(os.path.exists(cachecachedir))
         # Call compileall again.
-        retcode = subprocess.call(
-            (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
-        self.assertEqual(retcode, 0)
-        self.assertTrue(os.path.exists(cachedir))
+        self.assertRunOK('-q', self.pkgdir)
+        self.assertTrue(os.path.exists(self.pkgdir_cachedir))
         self.assertFalse(os.path.exists(cachecachedir))
 
     def test_force(self):
-        retcode = subprocess.call(
-            (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
-        self.assertEqual(retcode, 0)
-        pycpath = imp.cache_from_source(os.path.join(self.pkgdir, 'bar.py'))
+        self.assertRunOK('-q', self.pkgdir)
+        pycpath = imp.cache_from_source(self.barfn)
         # set atime/mtime backward to avoid file timestamp resolution issues
         os.utime(pycpath, (time.time()-60,)*2)
-        access = os.stat(pycpath).st_mtime
-        retcode = subprocess.call(
-            (sys.executable, '-m', 'compileall', '-q', '-f', self.pkgdir))
-        self.assertEqual(retcode, 0)
-        access2 = os.stat(pycpath).st_mtime
-        self.assertNotEqual(access, access2)
-
-    def test_legacy(self):
-        # create a new module  XXX could rewrite using self.pkgdir
-        newpackage = os.path.join(self.pkgdir, 'spam')
-        os.mkdir(newpackage)
-        with open(os.path.join(newpackage, '__init__.py'), 'w'):
-            pass
-        with open(os.path.join(newpackage, 'ham.py'), 'w'):
-            pass
-        sourcefile = os.path.join(newpackage, 'ham.py')
-
-        retcode = subprocess.call(
-                (sys.executable, '-m', 'compileall',  '-q', '-l', self.pkgdir))
-        self.assertEqual(retcode, 0)
-        self.assertFalse(os.path.exists(imp.cache_from_source(sourcefile)))
-
-        retcode = subprocess.call(
-                (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
-        self.assertEqual(retcode, 0)
-        self.assertTrue(os.path.exists(imp.cache_from_source(sourcefile)))
+        mtime = os.stat(pycpath).st_mtime
+        # without force, no recompilation
+        self.assertRunOK('-q', self.pkgdir)
+        mtime2 = os.stat(pycpath).st_mtime
+        self.assertEqual(mtime, mtime2)
+        # now force it.
+        self.assertRunOK('-q', '-f', self.pkgdir)
+        mtime2 = os.stat(pycpath).st_mtime
+        self.assertNotEqual(mtime, mtime2)
+
+    def test_recursion_control(self):
+        subpackage = os.path.join(self.pkgdir, 'spam')
+        os.mkdir(subpackage)
+        subinitfn = script_helper.make_script(subpackage, '__init__', '')
+        hamfn = script_helper.make_script(subpackage, 'ham', '')
+        self.assertRunOK('-q', '-l', self.pkgdir)
+        self.assertNotCompiled(subinitfn)
+        self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__')))
+        self.assertRunOK('-q', self.pkgdir)
+        self.assertCompiled(subinitfn)
+        self.assertCompiled(hamfn)
 
     def test_quiet(self):
-        noise = subprocess.check_output(
-             [sys.executable, '-m', 'compileall', self.pkgdir],
-             stderr=subprocess.STDOUT)
-        quiet = subprocess.check_output(
-            [sys.executable, '-m', 'compileall', '-f', '-q', self.pkgdir],
-            stderr=subprocess.STDOUT)
-        self.assertGreater(len(noise), len(quiet))
+        noisy = self.assertRunOK(self.pkgdir)
+        quiet = self.assertRunOK('-q', self.pkgdir)
+        self.assertNotEqual(b'', noisy)
+        self.assertEqual(b'', quiet)
 
     def test_regexp(self):
-        retcode = subprocess.call(
-            (sys.executable, '-m', 'compileall', '-q', '-x', 'bar.*', self.pkgdir))
-        self.assertEqual(retcode, 0)
-
-        sourcefile = os.path.join(self.pkgdir, 'bar.py')
-        self.assertFalse(os.path.exists(imp.cache_from_source(sourcefile)))
-        sourcefile = os.path.join(self.pkgdir, '__init__.py')
-        self.assertTrue(os.path.exists(imp.cache_from_source(sourcefile)))
+        self.assertRunOK('-q', '-x', 'ba.*', self.pkgdir)
+        self.assertNotCompiled(self.barfn)
+        self.assertCompiled(self.initfn)
+
+    def test_multiple_dirs(self):
+        pkgdir2 = os.path.join(self.directory, 'foo2')
+        os.mkdir(pkgdir2)
+        init2fn = script_helper.make_script(pkgdir2, '__init__', '')
+        bar2fn = script_helper.make_script(pkgdir2, 'bar2', '')
+        self.assertRunOK('-q', self.pkgdir, pkgdir2)
+        self.assertCompiled(self.initfn)
+        self.assertCompiled(self.barfn)
+        self.assertCompiled(init2fn)
+        self.assertCompiled(bar2fn)
+
+    def test_d_takes_exactly_one_dir(self):
+        rc, out, err = self.assertRunNotOK('-d', 'foo')
+        self.assertEqual(out, b'')
+        self.assertRegex(err, b'-d')
+        rc, out, err = self.assertRunNotOK('-d', 'foo', 'bar')
+        self.assertEqual(out, b'')
+        self.assertRegex(err, b'-d')
+
+    def test_d_compile_error(self):
+        script_helper.make_script(self.pkgdir, 'crunchyfrog', 'bad(syntax')
+        rc, out, err = self.assertRunNotOK('-q', '-d', 'dinsdale', self.pkgdir)
+        self.assertRegex(out, b'File "dinsdale')
+
+    def test_d_runtime_error(self):
+        bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception')
+        self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir)
+        fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz')
+        pyc = imp.cache_from_source(bazfn)
+        os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc'))
+        os.remove(bazfn)
+        rc, out, err = script_helper.assert_python_failure(fn)
+        self.assertRegex(err, b'File "dinsdale')
+
+    def test_include_bad_file(self):
+        rc, out, err = self.assertRunNotOK(
+            '-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir)
+        self.assertRegex(out, b'rror.*nosuchfile')
+        self.assertNotRegex(err, b'Traceback')
+        self.assertFalse(os.path.exists(imp.cache_from_source(
+                                            self.pkgdir_cachedir)))
+
+    def test_include_file_with_arg(self):
+        f1 = script_helper.make_script(self.pkgdir, 'f1', '')
+        f2 = script_helper.make_script(self.pkgdir, 'f2', '')
+        f3 = script_helper.make_script(self.pkgdir, 'f3', '')
+        f4 = script_helper.make_script(self.pkgdir, 'f4', '')
+        with open(os.path.join(self.directory, 'l1'), 'w') as l1:
+            l1.write(os.path.join(self.pkgdir, 'f1.py')+os.linesep)
+            l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep)
+        self.assertRunOK('-i', os.path.join(self.directory, 'l1'), f4)
+        self.assertCompiled(f1)
+        self.assertCompiled(f2)
+        self.assertNotCompiled(f3)
+        self.assertCompiled(f4)
+
+    def test_include_file_no_arg(self):
+        f1 = script_helper.make_script(self.pkgdir, 'f1', '')
+        f2 = script_helper.make_script(self.pkgdir, 'f2', '')
+        f3 = script_helper.make_script(self.pkgdir, 'f3', '')
+        f4 = script_helper.make_script(self.pkgdir, 'f4', '')
+        with open(os.path.join(self.directory, 'l1'), 'w') as l1:
+            l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep)
+        self.assertRunOK('-i', os.path.join(self.directory, 'l1'))
+        self.assertNotCompiled(f1)
+        self.assertCompiled(f2)
+        self.assertNotCompiled(f3)
+        self.assertNotCompiled(f4)
+
+    def test_include_on_stdin(self):
+        f1 = script_helper.make_script(self.pkgdir, 'f1', '')
+        f2 = script_helper.make_script(self.pkgdir, 'f2', '')
+        f3 = script_helper.make_script(self.pkgdir, 'f3', '')
+        f4 = script_helper.make_script(self.pkgdir, 'f4', '')
+        p = script_helper.spawn_python('-m', 'compileall', '-i', '-')
+        p.stdin.write((f3+os.linesep).encode('ascii'))
+        script_helper.kill_python(p)
+        self.assertNotCompiled(f1)
+        self.assertNotCompiled(f2)
+        self.assertCompiled(f3)
+        self.assertNotCompiled(f4)
+
+    def test_compiles_as_much_as_possible(self):
+        bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error')
+        rc, out, err = self.assertRunNotOK('nosuchfile', self.initfn,
+                                           bingfn, self.barfn)
+        self.assertRegex(b'rror', err)
+        self.assertNotCompiled(bingfn)
+        self.assertCompiled(self.initfn)
+        self.assertCompiled(self.barfn)
+
 
 
 def test_main():