bpo-30540, bpo-30523: Add --matchfile and --list-cases options to regrtest (#2249)

[2.7] bpo-30540, bpo-30523: Add --matchfile and --list-cases options to regrtest

Lib/test/regrtest.py

@@ -38,6 +38,7 @@ Selecting tests
 -x/--exclude    -- arguments are tests to *exclude*
 -s/--single     -- single step through a set of tests (see below)
 -m/--match PAT  -- match test cases and methods with glob pattern PAT
+--matchfile FILENAME -- filters tests using a text file, one pattern per line
 -G/--failfast   -- fail as soon as a test fails (only with -v or -W)
 -u/--use RES1,RES2,...
                 -- specify which special resource intensive tests to run
@@ -64,6 +65,8 @@ Special runs
                    (instead of the Python stdlib test suite)
 --list-tests    -- only write the name of tests that will be run,
                    don't execute them
+--list-cases    -- only write the name of test cases that will be run,
+                   don't execute them
 
 
 Additional Option Details:
@@ -157,6 +160,13 @@ resources to test.  Currently only the following are defined:
 To enable all resources except one, use '-uall,-<resource>'.  For
 example, to run all the tests except for the bsddb tests, give the
 option '-uall,-bsddb'.
+
+--matchfile filters tests using a text file, one pattern per line.
+Pattern examples:
+
+- test method: test_stat_attributes
+- test class: FileTests
+- test identifier: test_os.FileTests.test_stat_attributes
 """
 
 import StringIO
@@ -316,7 +326,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
              'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',
              'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
              'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo',
-             'failfast', 'match=', 'testdir=', 'list-tests', 'coverage'])
+             'failfast', 'match=', 'testdir=', 'list-tests', 'list-cases',
+             'coverage', 'matchfile='])
     except getopt.error, msg:
         usage(2, msg)
 
@@ -327,6 +338,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         use_resources = []
     slaveargs = None
     list_tests = False
+    list_cases_opt = False
     for o, a in opts:
         if o in ('-h', '--help'):
             usage(0)
@@ -354,7 +366,16 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         elif o in ('-f', '--fromfile'):
             fromfile = a
         elif o in ('-m', '--match'):
-            match_tests = a
+            if match_tests is None:
+                match_tests = []
+            match_tests.append(a)
+        elif o == '--matchfile':
+            if match_tests is None:
+                match_tests = []
+            filename = os.path.join(test_support.SAVEDCWD, a)
+            with open(filename) as fp:
+                for line in fp:
+                    match_tests.append(line.strip())
         elif o in ('-l', '--findleaks'):
             findleaks = True
         elif o in ('-L', '--runleaks'):
@@ -416,6 +437,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
             testdir = a
         elif o == '--list-tests':
             list_tests = True
+        elif o == '--list-cases':
+            list_cases_opt = True
         else:
             print >>sys.stderr, ("No handler for option {}.  Please "
                 "report this as a bug at http://bugs.python.org.").format(o)
@@ -534,6 +557,10 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
             print(name)
         sys.exit(0)
 
+    if list_cases_opt:
+        list_cases(testdir, selected)
+        sys.exit(0)
+
     if trace:
         import trace
         tracer = trace.Trace(trace=False, count=True)
@@ -1124,11 +1151,7 @@ def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False, testdir=Non
         try:
             if capture_stdout:
                 sys.stdout = capture_stdout
-            if test.startswith('test.') or testdir:
-                abstest = test
-            else:
-                # Always import it from the test package
-                abstest = 'test.' + test
+            abstest = get_abs_module(testdir, test)
             clear_caches()
             with saved_test_environment(test, verbose, quiet, pgo) as environment:
                 start_time = time.time()
@@ -1452,7 +1475,7 @@ def count(n, word):
     else:
         return "%d %ss" % (n, word)
 
-def printlist(x, width=70, indent=4):
+def printlist(x, width=70, indent=4, file=None):
     """Print the elements of iterable x to stdout.
 
     Optional arg width (default 70) is the maximum line length.
@@ -1463,8 +1486,37 @@ def printlist(x, width=70, indent=4):
     from textwrap import fill
     blanks = ' ' * indent
     # Print the sorted list: 'x' may be a '--random' list or a set()
-    print fill(' '.join(str(elt) for elt in sorted(x)), width,
-               initial_indent=blanks, subsequent_indent=blanks)
+    print >>file, fill(' '.join(str(elt) for elt in sorted(x)), width,
+                       initial_indent=blanks, subsequent_indent=blanks)
+
+def get_abs_module(testdir, test):
+    if test.startswith('test.') or testdir:
+        return test
+    else:
+        # Always import it from the test package
+        return 'test.' + test
+
+def _list_cases(suite):
+    for test in suite:
+        if isinstance(test, unittest.TestSuite):
+            _list_cases(test)
+        elif isinstance(test, unittest.TestCase):
+            print(test.id())
+
+def list_cases(testdir, selected):
+    skipped = []
+    for test in selected:
+        abstest = get_abs_module(testdir, test)
+        try:
+            suite = unittest.defaultTestLoader.loadTestsFromName(abstest)
+            _list_cases(suite)
+        except unittest.SkipTest:
+            skipped.append(test)
+
+    if skipped:
+        print >>sys.stderr
+        print >>sys.stderr, count(len(skipped), "test"), "skipped:"
+        printlist(skipped, file=sys.stderr)
 
 # Map sys.platform to a string containing the basenames of tests
 # expected to be skipped on that platform.

Lib/test/support/__init__.py

@@ -1557,9 +1557,15 @@ def run_unittest(*classes):
     def case_pred(test):
         if match_tests is None:
             return True
-        for name in test.id().split("."):
-            if fnmatch.fnmatchcase(name, match_tests):
+        test_id = test.id()
+
+        for match_test in match_tests:
+            if fnmatch.fnmatchcase(test_id, match_test):
                 return True
+
+            for name in test_id.split("."):
+                if fnmatch.fnmatchcase(name, match_test):
+                    return True
         return False
     _filter_suite(suite, case_pred)
     _run_suite(suite)

Lib/test/test_regrtest.py

@@ -482,6 +482,77 @@ class ArgsTestCase(BaseTestCase):
         self.check_executed_tests(output, tests, failed=crash_test,
                                   randomize=True)
 
+    def parse_methods(self, output):
+        regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
+        return [match.group(1) for match in regex.finditer(output)]
+
+    def test_matchfile(self):
+        # Any code which causes a crash
+        code = textwrap.dedent("""
+            import unittest
+            from test import support
+
+            class Tests(unittest.TestCase):
+                def test_method1(self):
+                    pass
+                def test_method2(self):
+                    pass
+                def test_method3(self):
+                    pass
+                def test_method4(self):
+                    pass
+
+            def test_main():
+                support.run_unittest(Tests)
+        """)
+        all_methods = ['test_method1', 'test_method2',
+                       'test_method3', 'test_method4']
+        testname = self.create_test(code=code)
+
+        # by default, all methods should be run
+        output = self.run_tests("-v", testname)
+        methods = self.parse_methods(output)
+        self.assertEqual(methods, all_methods)
+
+        # only run a subset
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
+
+        subset = [
+            # only match the method name
+            'test_method1',
+            # match the full identifier
+            '%s.Tests.test_method3' % testname]
+        with open(filename, "w") as fp:
+            for name in subset:
+                print(name, file=fp)
+
+        output = self.run_tests("-v", "--matchfile", filename, testname)
+        methods = self.parse_methods(output)
+        subset = ['test_method1', 'test_method3']
+        self.assertEqual(methods, subset)
+
+    def test_list_cases(self):
+        # test --list-cases
+        code = textwrap.dedent("""
+            import unittest
+            from test import support
+
+            class Tests(unittest.TestCase):
+                def test_method1(self):
+                    pass
+                def test_method2(self):
+                    pass
+
+            def test_main():
+                support.run_unittest(Tests)
+        """)
+        testname = self.create_test(code=code)
+        all_methods = ['%s.Tests.test_method1' % testname,
+                       '%s.Tests.test_method2' % testname]
+        output = self.run_tests('--list-cases', testname)
+        self.assertEqual(output.splitlines(), all_methods)
+
 
 def test_main():
     support.run_unittest(ProgramsTestCase, ArgsTestCase)