bpo-36876: Add a tool that identifies unsupported global C variables. (#15877)

Lib/test/test_check_c_globals.py

+import unittest
+import test.test_tools
+
+test.test_tools.skip_if_missing('c-analyzer')
+with test.test_tools.imports_under_tool('c-analyzer'):
+    from c_globals.__main__ import main
+
+
+class ActualChecks(unittest.TestCase):
+
+    # XXX Also run the check in "make check".
+    @unittest.expectedFailure
+    def test_check_c_globals(self):
+        try:
+            main('check', {})
+        except NotImplementedError:
+            raise unittest.SkipTest('not supported on this host')
+
+
+if __name__ == '__main__':
+    # Test needs to be a package, so we can do relative imports.
+    unittest.main()

Lib/test/test_clinic.py

@@ -2,7 +2,7 @@
 # Copyright 2012-2013 by Larry Hastings.
 # Licensed to the PSF under a contributor agreement.
 
-from test import support
+from test import support, test_tools
 from unittest import TestCase
 import collections
 import inspect
@@ -10,17 +10,10 @@ import os.path
 import sys
 import unittest
 
-
-clinic_path = os.path.join(os.path.dirname(__file__), '..', '..', 'Tools', 'clinic')
-clinic_path = os.path.normpath(clinic_path)
-if not os.path.exists(clinic_path):
-    raise unittest.SkipTest(f'{clinic_path!r} path does not exist')
-sys.path.append(clinic_path)
-try:
+test_tools.skip_if_missing('clinic')
+with test_tools.imports_under_tool('clinic'):
     import clinic
     from clinic import DSLParser
-finally:
-    del sys.path[-1]
 
 
 class FakeConverter:

Lib/test/test_tools/__init__.py

 """Support functions for testing scripts in the Tools directory."""
-import os
-import unittest
+import contextlib
 import importlib
+import os.path
+import unittest
 from test import support
 
-basepath = os.path.dirname(                 # <src/install dir>
-                os.path.dirname(                # Lib
-                    os.path.dirname(                # test
-                        os.path.dirname(__file__))))    # test_tools
+basepath = os.path.normpath(
+        os.path.dirname(                 # <src/install dir>
+            os.path.dirname(                # Lib
+                os.path.dirname(                # test
+                    os.path.dirname(__file__)))))    # test_tools
 
 toolsdir = os.path.join(basepath, 'Tools')
 scriptsdir = os.path.join(toolsdir, 'scripts')
 
-def skip_if_missing():
-    if not os.path.isdir(scriptsdir):
-        raise unittest.SkipTest('scripts directory could not be found')
+def skip_if_missing(tool=None):
+    if tool:
+        tooldir = os.path.join(toolsdir, tool)
+    else:
+        tool = 'scripts'
+        tooldir = scriptsdir
+    if not os.path.isdir(tooldir):
+        raise unittest.SkipTest(f'{tool} directory could not be found')
+
+@contextlib.contextmanager
+def imports_under_tool(name, *subdirs):
+    tooldir = os.path.join(toolsdir, name, *subdirs)
+    with support.DirsOnSysPath(tooldir) as cm:
+        yield cm
 
 def import_tool(toolname):
     with support.DirsOnSysPath(scriptsdir):

Lib/test/test_tools/test_c_analyzer/__init__.py

+import contextlib
+import os.path
+import test.test_tools
+from test.support import load_package_tests
+
+
+@contextlib.contextmanager
+def tool_imports_for_tests():
+    test.test_tools.skip_if_missing('c-analyzer')
+    with test.test_tools.imports_under_tool('c-analyzer'):
+        yield
+
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_tools/test_c_analyzer/__main__.py

+from . import load_tests
+import unittest
+
+
+unittest.main()

Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_info.py

+import string
+import unittest
+
+from ..util import PseudoStr, StrProxy, Object
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+
+
+class IDTests(unittest.TestCase):
+
+    VALID_ARGS = (
+            'x/y/z/spam.c',
+            'func',
+            'eggs',
+            )
+    VALID_KWARGS = dict(zip(ID._fields, VALID_ARGS))
+    VALID_EXPECTED = VALID_ARGS
+
+    def test_from_raw(self):
+        tests = [
+            ('', None),
+            (None, None),
+            ('spam', (None, None, 'spam')),
+            (('spam',), (None, None, 'spam')),
+            (('x/y/z/spam.c', 'spam'), ('x/y/z/spam.c', None, 'spam')),
+            (self.VALID_ARGS, self.VALID_EXPECTED),
+            (self.VALID_KWARGS, self.VALID_EXPECTED),
+            ]
+        for raw, expected in tests:
+            with self.subTest(raw):
+                id = ID.from_raw(raw)
+
+                self.assertEqual(id, expected)
+
+    def test_minimal(self):
+        id = ID(
+                filename=None,
+                funcname=None,
+                name='eggs',
+                )
+
+        self.assertEqual(id, (
+                None,
+                None,
+                'eggs',
+                ))
+
+    def test_init_typical_global(self):
+        id = ID(
+                filename='x/y/z/spam.c',
+                funcname=None,
+                name='eggs',
+                )
+
+        self.assertEqual(id, (
+                'x/y/z/spam.c',
+                None,
+                'eggs',
+                ))
+
+    def test_init_typical_local(self):
+        id = ID(
+                filename='x/y/z/spam.c',
+                funcname='func',
+                name='eggs',
+                )
+
+        self.assertEqual(id, (
+                'x/y/z/spam.c',
+                'func',
+                'eggs',
+                ))
+
+    def test_init_all_missing(self):
+        for value in ('', None):
+            with self.subTest(repr(value)):
+                id = ID(
+                        filename=value,
+                        funcname=value,
+                        name=value,
+                        )
+
+                self.assertEqual(id, (
+                        None,
+                        None,
+                        None,
+                        ))
+
+    def test_init_all_coerced(self):
+        tests = [
+            ('str subclass',
+             dict(
+                 filename=PseudoStr('x/y/z/spam.c'),
+                 funcname=PseudoStr('func'),
+                 name=PseudoStr('eggs'),
+                 ),
+             ('x/y/z/spam.c',
+              'func',
+              'eggs',
+              )),
+            ('non-str',
+             dict(
+                 filename=StrProxy('x/y/z/spam.c'),
+                 funcname=Object(),
+                 name=('a', 'b', 'c'),
+                 ),
+             ('x/y/z/spam.c',
+              '<object>',
+              "('a', 'b', 'c')",
+              )),
+            ]
+        for summary, kwargs, expected in tests:
+            with self.subTest(summary):
+                id = ID(**kwargs)
+
+                for field in ID._fields:
+                    value = getattr(id, field)
+                    self.assertIs(type(value), str)
+                self.assertEqual(tuple(id), expected)
+
+    def test_iterable(self):
+        id = ID(**self.VALID_KWARGS)
+
+        filename, funcname, name = id
+
+        values = (filename, funcname, name)
+        for value, expected in zip(values, self.VALID_EXPECTED):
+            self.assertEqual(value, expected)
+
+    def test_fields(self):
+        id = ID('a', 'b', 'z')
+
+        self.assertEqual(id.filename, 'a')
+        self.assertEqual(id.funcname, 'b')
+        self.assertEqual(id.name, 'z')
+
+    def test_validate_typical(self):
+        id = ID(
+                filename='x/y/z/spam.c',
+                funcname='func',
+                name='eggs',
+                )
+
+        id.validate()  # This does not fail.
+
+    def test_validate_missing_field(self):
+        for field in ID._fields:
+            with self.subTest(field):
+                id = ID(**self.VALID_KWARGS)
+                id = id._replace(**{field: None})
+
+                if field == 'funcname':
+                    id.validate()  # The field can be missing (not set).
+                    id = id._replace(filename=None)
+                    id.validate()  # Both fields can be missing (not set).
+                    continue
+
+                with self.assertRaises(TypeError):
+                    id.validate()
+
+    def test_validate_bad_field(self):
+        badch = tuple(c for c in string.punctuation + string.digits)
+        notnames = (
+                '1a',
+                'a.b',
+                'a-b',
+                '&a',
+                'a++',
+                ) + badch
+        tests = [
+            ('filename', ()),  # Any non-empty str is okay.
+            ('funcname', notnames),
+            ('name', notnames),
+            ]
+        seen = set()
+        for field, invalid in tests:
+            for value in invalid:
+                seen.add(value)
+                with self.subTest(f'{field}={value!r}'):
+                    id = ID(**self.VALID_KWARGS)
+                    id = id._replace(**{field: value})
+
+                    with self.assertRaises(ValueError):
+                        id.validate()
+
+        for field, invalid in tests:
+            valid = seen - set(invalid)
+            for value in valid:
+                with self.subTest(f'{field}={value!r}'):
+                    id = ID(**self.VALID_KWARGS)
+                    id = id._replace(**{field: value})
+
+                    id.validate()  # This does not fail.

Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py

+import re
+import textwrap
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser.info import Variable
+    from c_analyzer_common.info import ID
+    from c_analyzer_common.known import from_file
+
+
+class FromFileTests(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_read_tsv = ()
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def _read_tsv(self, *args):
+        self.calls.append(('_read_tsv', args))
+        return self._return_read_tsv
+
+    def test_typical(self):
+        lines = textwrap.dedent('''
+            filename    funcname        name    kind    declaration
+            file1.c     -       var1    variable        static int
+            file1.c     func1   local1  variable        static int
+            file1.c     -       var2    variable        int
+            file1.c     func2   local2  variable        char *
+            file2.c     -       var1    variable        char *
+            ''').strip().splitlines()
+        lines = [re.sub(r'\s+', '\t', line, 4) for line in lines]
+        self._return_read_tsv = [tuple(v.strip() for v in line.split('\t'))
+                                 for line in lines[1:]]
+
+        known = from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(known, {
+            'variables': {v.id: v for v in [
+                Variable.from_parts('file1.c', '', 'var1', 'static int'),
+                Variable.from_parts('file1.c', 'func1', 'local1', 'static int'),
+                Variable.from_parts('file1.c', '', 'var2', 'int'),
+                Variable.from_parts('file1.c', 'func2', 'local2', 'char *'),
+                Variable.from_parts('file2.c', '', 'var1', 'char *'),
+                ]},
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')),
+            ])
+
+    def test_empty(self):
+        self._return_read_tsv = []
+
+        known = from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(known, {
+            'variables': {},
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\tdeclaration')),
+            ])

Lib/test/test_tools/test_c_analyzer/test_c_globals/test___main__.py

+import sys
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common import SOURCE_DIRS
+    from c_analyzer_common.known import DATA_FILE as KNOWN_FILE
+    from c_parser import info
+    import c_globals as cg
+    from c_globals.supported import IGNORED_FILE
+    from c_globals.__main__ import cmd_check, cmd_show, parse_args, main
+
+
+TYPICAL = [
+        (info.Variable.from_parts('src1/spam.c', None, 'var1', 'const char *'),
+         True,
+         ),
+        (info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'int'),
+         True,
+         ),
+        (info.Variable.from_parts('src1/spam.c', None, 'var2', 'PyObject *'),
+         False,
+         ),
+        (info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'int'),
+         True,
+         ),
+        (info.Variable.from_parts('src1/spam.c', None, 'freelist', '(PyTupleObject *)[10]'),
+         False,
+         ),
+        (info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'const char const *'),
+         True,
+         ),
+        (info.Variable.from_parts('src2/jam.c', None, 'var1', 'int'),
+         True,
+         ),
+        (info.Variable.from_parts('src2/jam.c', None, 'var2', 'MyObject *'),
+         False,
+         ),
+        (info.Variable.from_parts('Include/spam.h', None, 'data', 'const int'),
+         True,
+         ),
+        ]
+
+
+class CMDBase(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_find = ()
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def _find(self, *args):
+        self.calls.append(('_find', args))
+        return self._return_find
+
+    def _show(self, *args):
+        self.calls.append(('_show', args))
+
+    def _print(self, *args):
+        self.calls.append(('_print', args))
+
+
+class CheckTests(CMDBase):
+
+    def test_defaults(self):
+        self._return_find = []
+
+        cmd_check('check',
+                  _find=self._find,
+                  _show=self._show,
+                  _print=self._print,
+                  )
+
+        self.assertEqual(self.calls[0], (
+            '_find', (
+                SOURCE_DIRS,
+                KNOWN_FILE,
+                IGNORED_FILE,
+                ),
+            ))
+
+    def test_all_supported(self):
+        self._return_find = [(v, s) for v, s in TYPICAL if s]
+        dirs = ['src1', 'src2', 'Include']
+
+        cmd_check('check',
+                 dirs,
+                 ignored='ignored.tsv',
+                 known='known.tsv',
+                 _find=self._find,
+                 _show=self._show,
+                 _print=self._print,
+                 )
+
+        self.assertEqual(self.calls, [
+            ('_find', (dirs, 'known.tsv', 'ignored.tsv')),
+            #('_print', ('okay',)),
+            ])
+
+    def test_some_unsupported(self):
+        self._return_find = TYPICAL
+        dirs = ['src1', 'src2', 'Include']
+
+        with self.assertRaises(SystemExit) as cm:
+            cmd_check('check',
+                      dirs,
+                      ignored='ignored.tsv',
+                      known='known.tsv',
+                      _find=self._find,
+                      _show=self._show,
+                      _print=self._print,
+                      )
+
+        unsupported = [v for v, s in TYPICAL if not s]
+        self.assertEqual(self.calls, [
+            ('_find', (dirs, 'known.tsv', 'ignored.tsv')),
+            ('_print', ('ERROR: found unsupported global variables',)),
+            ('_print', ()),
+            ('_show', (sorted(unsupported),)),
+            ('_print', (' (3 total)',)),
+            ])
+        self.assertEqual(cm.exception.code, 1)
+
+
+class ShowTests(CMDBase):
+
+    def test_defaults(self):
+        self._return_find = []
+
+        cmd_show('show',
+                 _find=self._find,
+                 _show=self._show,
+                 _print=self._print,
+                 )
+
+        self.assertEqual(self.calls[0], (
+            '_find', (
+                SOURCE_DIRS,
+                KNOWN_FILE,
+                IGNORED_FILE,
+                ),
+            ))
+
+    def test_typical(self):
+        self._return_find = TYPICAL
+        dirs = ['src1', 'src2', 'Include']
+
+        cmd_show('show',
+                 dirs,
+                 known='known.tsv',
+                 ignored='ignored.tsv',
+                 _find=self._find,
+                 _show=self._show,
+                 _print=self._print,
+                 )
+
+        supported = [v for v, s in TYPICAL if s]
+        unsupported = [v for v, s in TYPICAL if not s]
+        self.assertEqual(self.calls, [
+            ('_find', (dirs, 'known.tsv', 'ignored.tsv')),
+            ('_print', ('supported:',)),
+            ('_print', ('----------',)),
+            ('_show', (sorted(supported),)),
+            ('_print', (' (6 total)',)),
+            ('_print', ()),
+            ('_print', ('unsupported:',)),
+            ('_print', ('------------',)),
+            ('_show', (sorted(unsupported),)),
+            ('_print', (' (3 total)',)),
+            ])
+
+
+class ParseArgsTests(unittest.TestCase):
+
+    maxDiff = None
+
+    def test_no_args(self):
+        self.errmsg = None
+        def fail(msg):
+            self.errmsg = msg
+            sys.exit(msg)
+
+        with self.assertRaises(SystemExit):
+            parse_args('cg', [], _fail=fail)
+
+        self.assertEqual(self.errmsg, 'missing command')
+
+    def test_check_no_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'check',
+            ])
+
+        self.assertEqual(cmd, 'check')
+        self.assertEqual(cmdkwargs, {
+            'ignored': IGNORED_FILE,
+            'known': KNOWN_FILE,
+            'dirs': SOURCE_DIRS,
+            })
+
+    def test_check_full_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'check',
+            '--ignored', 'spam.tsv',
+            '--known', 'eggs.tsv',
+            'dir1',
+            'dir2',
+            'dir3',
+            ])
+
+        self.assertEqual(cmd, 'check')
+        self.assertEqual(cmdkwargs, {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3']
+            })
+
+    def test_show_no_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'show',
+            ])
+
+        self.assertEqual(cmd, 'show')
+        self.assertEqual(cmdkwargs, {
+            'ignored': IGNORED_FILE,
+            'known': KNOWN_FILE,
+            'dirs': SOURCE_DIRS,
+            'skip_objects': False,
+            })
+
+    def test_show_full_args(self):
+        cmd, cmdkwargs = parse_args('cg', [
+            'show',
+            '--ignored', 'spam.tsv',
+            '--known', 'eggs.tsv',
+            'dir1',
+            'dir2',
+            'dir3',
+            ])
+
+        self.assertEqual(cmd, 'show')
+        self.assertEqual(cmdkwargs, {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3'],
+            'skip_objects': False,
+            })
+
+
+def new_stub_commands(*names):
+    calls = []
+    def cmdfunc(cmd, **kwargs):
+        calls.append((cmd, kwargs))
+    commands = {name: cmdfunc for name in names}
+    return commands, calls
+
+
+class MainTests(unittest.TestCase):
+
+    def test_no_command(self):
+        with self.assertRaises(ValueError):
+            main(None, {})
+
+    def test_check(self):
+        commands, calls = new_stub_commands('check', 'show')
+
+        cmdkwargs = {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3'],
+            }
+        main('check', cmdkwargs, _COMMANDS=commands)
+
+        self.assertEqual(calls, [
+            ('check', cmdkwargs),
+            ])
+
+    def test_show(self):
+        commands, calls = new_stub_commands('check', 'show')
+
+        cmdkwargs = {
+            'ignored': 'spam.tsv',
+            'known': 'eggs.tsv',
+            'dirs': ['dir1', 'dir2', 'dir3'],
+            }
+        main('show', cmdkwargs, _COMMANDS=commands)
+
+        self.assertEqual(calls, [
+            ('show', cmdkwargs),
+            ])

Lib/test/test_tools/test_c_analyzer/test_c_globals/test_functional.py

+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    pass
+
+
+class SelfCheckTests(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_known(self):
+        # Make sure known macros & vartypes aren't hiding unknown local types.
+        # XXX finish!
+        raise NotImplementedError
+
+    @unittest.expectedFailure
+    def test_compare_nm_results(self):
+        # Make sure the "show" results match the statics found by "nm" command.
+        # XXX Skip if "nm" is not available.
+        # XXX finish!
+        raise NotImplementedError
+
+
+class DummySourceTests(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_check(self):
+        # XXX finish!
+        raise NotImplementedError
+
+    @unittest.expectedFailure
+    def test_show(self):
+        # XXX finish!
+        raise NotImplementedError

Lib/test/test_tools/test_c_analyzer/test_c_globals/test_show.py

+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_parser import info
+    from c_globals.show import basic
+
+
+TYPICAL = [
+        info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'),
+        info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'),
+        info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'),
+        info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'),
+        info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'),
+        info.Variable.from_parts('src1/sub/ham.c', None, 'var1', 'static const char const *'),
+        info.Variable.from_parts('src2/jam.c', None, 'var1', 'static int'),
+        info.Variable.from_parts('src2/jam.c', None, 'var2', 'static MyObject *'),
+        info.Variable.from_parts('Include/spam.h', None, 'data', 'static const int'),
+        ]
+
+
+class BasicTests(unittest.TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.lines = []
+
+    def print(self, line):
+        self.lines.append(line)
+
+    def test_typical(self):
+        basic(TYPICAL,
+              _print=self.print)
+
+        self.assertEqual(self.lines, [
+            'src1/spam.c:var1                                                 static const char *',
+            'src1/spam.c:ham():initialized                                    static int',
+            'src1/spam.c:var2                                                 static PyObject *',
+            'src1/eggs.c:tofu():ready                                         static int',
+            'src1/spam.c:freelist                                             static (PyTupleObject *)[10]',
+            'src1/sub/ham.c:var1                                              static const char const *',
+            'src2/jam.c:var1                                                  static int',
+            'src2/jam.c:var2                                                  static MyObject *',
+            'Include/spam.h:data                                              static const int',
+            ])
+
+    def test_no_rows(self):
+        basic([],
+              _print=self.print)
+
+        self.assertEqual(self.lines, [])

Lib/test/test_tools/test_c_analyzer/test_c_globals/test_supported.py

+import re
+import textwrap
+import unittest
+
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+    from c_parser import info
+    from c_globals.supported import is_supported, ignored_from_file
+
+
+class IsSupportedTests(unittest.TestCase):
+
+    @unittest.expectedFailure
+    def test_supported(self):
+        statics = [
+                info.StaticVar('src1/spam.c', None, 'var1', 'const char *'),
+                info.StaticVar('src1/spam.c', None, 'var1', 'int'),
+                ]
+        for static in statics:
+            with self.subTest(static):
+                result = is_supported(static)
+
+                self.assertTrue(result)
+
+    @unittest.expectedFailure
+    def test_not_supported(self):
+        statics = [
+                info.StaticVar('src1/spam.c', None, 'var1', 'PyObject *'),
+                info.StaticVar('src1/spam.c', None, 'var1', 'PyObject[10]'),
+                ]
+        for static in statics:
+            with self.subTest(static):
+                result = is_supported(static)
+
+                self.assertFalse(result)
+
+
+class IgnoredFromFileTests(unittest.TestCase):
+
+    maxDiff = None
+
+    _return_read_tsv = ()
+
+    @property
+    def calls(self):
+        try:
+            return self._calls
+        except AttributeError:
+            self._calls = []
+            return self._calls
+
+    def _read_tsv(self, *args):
+        self.calls.append(('_read_tsv', args))
+        return self._return_read_tsv
+
+    def test_typical(self):
+        lines = textwrap.dedent('''
+            filename    funcname        name    kind    reason
+            file1.c     -       var1    variable        ...
+            file1.c     func1   local1  variable        |
+            file1.c     -       var2    variable        ???
+            file1.c     func2   local2  variable           |
+            file2.c     -       var1    variable        reasons
+            ''').strip().splitlines()
+        lines = [re.sub(r'\s{1,8}', '\t', line, 4).replace('|', '')
+                 for line in lines]
+        self._return_read_tsv = [tuple(v.strip() for v in line.split('\t'))
+                                 for line in lines[1:]]
+
+        ignored = ignored_from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(ignored, {
+            'variables': {
+                ID('file1.c', '', 'var1'): '...',
+                ID('file1.c', 'func1', 'local1'): '',
+                ID('file1.c', '', 'var2'): '???',
+                ID('file1.c', 'func2', 'local2'): '',
+                ID('file2.c', '', 'var1'): 'reasons',
+                },
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\treason')),
+            ])
+
+    def test_empty(self):
+        self._return_read_tsv = []
+
+        ignored = ignored_from_file('spam.c', _read_tsv=self._read_tsv)
+
+        self.assertEqual(ignored, {
+            'variables': {},
+            })
+        self.assertEqual(self.calls, [
+            ('_read_tsv', ('spam.c', 'filename\tfuncname\tname\tkind\treason')),
+            ])

Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py

+import string
+import unittest
+
+from ..util import PseudoStr, StrProxy, Object
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+    from c_parser.info import (
+        normalize_vartype, Variable,
+        )
+
+
+class NormalizeVartypeTests(unittest.TestCase):
+
+    def test_basic(self):
+        tests = [
+                (None, None),
+                ('', ''),
+                ('int', 'int'),
+                (PseudoStr('int'), 'int'),
+                (StrProxy('int'), 'int'),
+                ]
+        for vartype, expected in tests:
+            with self.subTest(vartype):
+                normalized = normalize_vartype(vartype)
+
+                self.assertEqual(normalized, expected)
+
+
+class VariableTests(unittest.TestCase):
+
+    VALID_ARGS = (
+            ('x/y/z/spam.c', 'func', 'eggs'),
+            'int',
+            )
+    VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS))
+    VALID_EXPECTED = VALID_ARGS
+
+    def test_init_typical_global(self):
+        static = Variable(
+                id=ID(
+                    filename='x/y/z/spam.c',
+                    funcname=None,
+                    name='eggs',
+                    ),
+                vartype='int',
+                )
+
+        self.assertEqual(static, (
+                ('x/y/z/spam.c', None, 'eggs'),
+                'int',
+                ))
+
+    def test_init_typical_local(self):
+        static = Variable(
+                id=ID(
+                    filename='x/y/z/spam.c',
+                    funcname='func',
+                    name='eggs',
+                    ),
+                vartype='int',
+                )
+
+        self.assertEqual(static, (
+                ('x/y/z/spam.c', 'func', 'eggs'),
+                'int',
+                ))
+
+    def test_init_all_missing(self):
+        for value in ('', None):
+            with self.subTest(repr(value)):
+                static = Variable(
+                        id=value,
+                        vartype=value,
+                        )
+
+                self.assertEqual(static, (
+                        None,
+                        None,
+                        ))
+
+    def test_init_all_coerced(self):
+        id = ID('x/y/z/spam.c', 'func', 'spam')
+        tests = [
+            ('str subclass',
+             dict(
+                 id=(
+                    PseudoStr('x/y/z/spam.c'),
+                    PseudoStr('func'),
+                    PseudoStr('spam'),
+                    ),
+                 vartype=PseudoStr('int'),
+                 ),
+             (id,
+              'int',
+              )),
+            ('non-str 1',
+             dict(
+                 id=id,
+                 vartype=Object(),
+                 ),
+             (id,
+              '<object>',
+              )),
+            ('non-str 2',
+             dict(
+                 id=id,
+                 vartype=StrProxy('variable'),
+                 ),
+             (id,
+              'variable',
+              )),
+            ('non-str',
+             dict(
+                 id=id,
+                 vartype=('a', 'b', 'c'),
+                 ),
+             (id,
+              "('a', 'b', 'c')",
+              )),
+            ]
+        for summary, kwargs, expected in tests:
+            with self.subTest(summary):
+                static = Variable(**kwargs)
+
+                for field in Variable._fields:
+                    value = getattr(static, field)
+                    if field == 'id':
+                        self.assertIs(type(value), ID)
+                    else:
+                        self.assertIs(type(value), str)
+                self.assertEqual(tuple(static), expected)
+
+    def test_iterable(self):
+        static = Variable(**self.VALID_KWARGS)
+
+        id, vartype = static
+
+        values = (id, vartype)
+        for value, expected in zip(values, self.VALID_EXPECTED):
+            self.assertEqual(value, expected)
+
+    def test_fields(self):
+        static = Variable(('a', 'b', 'z'), 'x')
+
+        self.assertEqual(static.id, ('a', 'b', 'z'))
+        self.assertEqual(static.vartype, 'x')
+
+    def test___getattr__(self):
+        static = Variable(('a', 'b', 'z'), 'x')
+
+        self.assertEqual(static.filename, 'a')
+        self.assertEqual(static.funcname, 'b')
+        self.assertEqual(static.name, 'z')
+
+    def test_validate_typical(self):
+        static = Variable(
+                id=ID(
+                    filename='x/y/z/spam.c',
+                    funcname='func',
+                    name='eggs',
+                    ),
+                vartype='int',
+                )
+
+        static.validate()  # This does not fail.
+
+    def test_validate_missing_field(self):
+        for field in Variable._fields:
+            with self.subTest(field):
+                static = Variable(**self.VALID_KWARGS)
+                static = static._replace(**{field: None})
+
+                with self.assertRaises(TypeError):
+                    static.validate()
+
+    def test_validate_bad_field(self):
+        badch = tuple(c for c in string.punctuation + string.digits)
+        notnames = (
+                '1a',
+                'a.b',
+                'a-b',
+                '&a',
+                'a++',
+                ) + badch
+        tests = [
+            ('id', ()),  # Any non-empty str is okay.
+            ('vartype', ()),  # Any non-empty str is okay.
+            ]
+        seen = set()
+        for field, invalid in tests:
+            for value in invalid:
+                seen.add(value)
+                with self.subTest(f'{field}={value!r}'):
+                    static = Variable(**self.VALID_KWARGS)
+                    static = static._replace(**{field: value})
+
+                    with self.assertRaises(ValueError):
+                        static.validate()
+
+        for field, invalid in tests:
+            valid = seen - set(invalid)
+            for value in valid:
+                with self.subTest(f'{field}={value!r}'):
+                    static = Variable(**self.VALID_KWARGS)
+                    static = static._replace(**{field: value})
+
+                    static.validate()  # This does not fail.

Lib/test/test_tools/test_c_analyzer/test_c_symbols/test_info.py

+import string
+import unittest
+
+from ..util import PseudoStr, StrProxy, Object
+from .. import tool_imports_for_tests
+with tool_imports_for_tests():
+    from c_analyzer_common.info import ID
+    from c_symbols.info import Symbol
+
+
+class SymbolTests(unittest.TestCase):
+
+    VALID_ARGS = (
+            ID('x/y/z/spam.c', 'func', 'eggs'),
+            Symbol.KIND.VARIABLE,
+            False,
+            )
+    VALID_KWARGS = dict(zip(Symbol._fields, VALID_ARGS))
+    VALID_EXPECTED = VALID_ARGS
+
+    def test_init_typical_binary_local(self):
+        id = ID(None, None, 'spam')
+        symbol = Symbol(
+                id=id,
+                kind=Symbol.KIND.VARIABLE,
+                external=False,
+                )
+
+        self.assertEqual(symbol, (
+            id,
+            Symbol.KIND.VARIABLE,
+            False,
+            ))
+
+    def test_init_typical_binary_global(self):
+        id = ID('Python/ceval.c', None, 'spam')
+        symbol = Symbol(
+                id=id,
+                kind=Symbol.KIND.VARIABLE,
+                external=False,
+                )
+
+        self.assertEqual(symbol, (
+            id,
+            Symbol.KIND.VARIABLE,
+            False,
+            ))
+
+    def test_init_coercion(self):
+        tests = [
+            ('str subclass',
+             dict(
+                 id=PseudoStr('eggs'),
+                 kind=PseudoStr('variable'),
+                 external=0,
+                 ),
+             (ID(None, None, 'eggs'),
+              Symbol.KIND.VARIABLE,
+              False,
+              )),
+            ('with filename',
+             dict(
+                 id=('x/y/z/spam.c', 'eggs'),
+                 kind=PseudoStr('variable'),
+                 external=0,
+                 ),
+             (ID('x/y/z/spam.c', None, 'eggs'),
+              Symbol.KIND.VARIABLE,
+              False,
+              )),
+            ('non-str 1',
+             dict(
+                 id=('a', 'b', 'c'),
+                 kind=StrProxy('variable'),
+                 external=0,
+                 ),
+             (ID('a', 'b', 'c'),
+              Symbol.KIND.VARIABLE,
+              False,
+              )),
+            ('non-str 2',
+             dict(
+                 id=('a', 'b', 'c'),
+                 kind=Object(),
+                 external=0,
+                 ),
+             (ID('a', 'b', 'c'),
+              '<object>',
+              False,
+              )),
+            ]
+        for summary, kwargs, expected in tests:
+            with self.subTest(summary):
+                symbol = Symbol(**kwargs)
+
+                for field in Symbol._fields:
+                    value = getattr(symbol, field)
+                    if field == 'external':
+                        self.assertIs(type(value), bool)
+                    elif field == 'id':
+                        self.assertIs(type(value), ID)
+                    else:
+                        self.assertIs(type(value), str)
+                self.assertEqual(tuple(symbol), expected)
+
+    def test_init_all_missing(self):
+        id = ID(None, None, 'spam')
+
+        symbol = Symbol(id)
+
+        self.assertEqual(symbol, (
+            id,
+            Symbol.KIND.VARIABLE,
+            None,
+            ))
+
+    def test_fields(self):
+        id = ID('z', 'x', 'a')
+
+        symbol = Symbol(id, 'b', False)
+
+        self.assertEqual(symbol.id, id)
+        self.assertEqual(symbol.kind, 'b')
+        self.assertIs(symbol.external, False)
+
+    def test___getattr__(self):
+        id = ID('z', 'x', 'a')
+        symbol = Symbol(id, 'b', False)
+
+        filename = symbol.filename
+        funcname = symbol.funcname
+        name = symbol.name
+
+        self.assertEqual(filename, 'z')
+        self.assertEqual(funcname, 'x')
+        self.assertEqual(name, 'a')
+
+    def test_validate_typical(self):
+        id = ID('z', 'x', 'a')
+
+        symbol = Symbol(
+                id=id,
+                kind=Symbol.KIND.VARIABLE,
+                external=False,
+                )
+
+        symbol.validate()  # This does not fail.
+
+    def test_validate_missing_field(self):
+        for field in Symbol._fields:
+            with self.subTest(field):
+                symbol = Symbol(**self.VALID_KWARGS)
+                symbol = symbol._replace(**{field: None})
+
+                with self.assertRaises(TypeError):
+                    symbol.validate()
+
+    def test_validate_bad_field(self):
+        badch = tuple(c for c in string.punctuation + string.digits)
+        notnames = (
+                '1a',
+                'a.b',
+                'a-b',
+                '&a',
+                'a++',
+                ) + badch
+        tests = [
+            ('id', notnames),
+            ('kind', ('bogus',)),
+            ]
+        seen = set()
+        for field, invalid in tests:
+            for value in invalid:
+                if field != 'kind':
+                    seen.add(value)
+                with self.subTest(f'{field}={value!r}'):
+                    symbol = Symbol(**self.VALID_KWARGS)
+                    symbol = symbol._replace(**{field: value})
+
+                    with self.assertRaises(ValueError):
+                        symbol.validate()
+
+        for field, invalid in tests:
+            if field == 'kind':
+                continue
+            valid = seen - set(invalid)
+            for value in valid:
+                with self.subTest(f'{field}={value!r}'):
+                    symbol = Symbol(**self.VALID_KWARGS)
+                    symbol = symbol._replace(**{field: value})
+
+                    symbol.validate()  # This does not fail.

Lib/test/test_tools/test_c_analyzer/util.py

+import itertools
+
+
+class PseudoStr(str):
+    pass
+
+
+class StrProxy:
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return self.value
+    def __bool__(self):
+        return bool(self.value)
+
+
+class Object:
+    def __repr__(self):
+        return '<object>'
+
+
+def wrapped_arg_combos(*args,
+                       wrappers=(PseudoStr, StrProxy),
+                       skip=(lambda w, i, v: not isinstance(v, str)),
+                       ):
+    """Yield every possible combination of wrapped items for the given args.
+
+    Effectively, the wrappers are applied to the args according to the
+    powerset of the args indicies.  So the result includes the args
+    completely unwrapped.
+
+    If "skip" is supplied (default is to skip all non-str values) and
+    it returns True for a given arg index/value then that arg will
+    remain unwrapped,
+
+    Only unique results are returned.  If an arg was skipped for one
+    of the combinations then it could end up matching one of the other
+    combinations.  In that case only one of them will be yielded.
+    """
+    if not args:
+        return
+    indices = list(range(len(args)))
+    # The powerset (from recipe in the itertools docs).
+    combos = itertools.chain.from_iterable(itertools.combinations(indices, r)
+                                           for r in range(len(indices)+1))
+    seen = set()
+    for combo in combos:
+        for wrap in wrappers:
+            indexes = []
+            applied = list(args)
+            for i in combo:
+                arg = args[i]
+                if skip and skip(wrap, i, arg):
+                    continue
+                indexes.append(i)
+                applied[i] = wrap(arg)
+            key = (wrap, tuple(indexes))
+            if key not in seen:
+                yield tuple(applied)
+                seen.add(key)

Tools/c-analyzer/c-globals.py

+# This is a script equivalent of running "python -m test.test_c_globals.cg".
+
+from c_globals.__main__ import parse_args, main
+
+
+# This is effectively copied from cg/__main__.py:
+if __name__ == '__main__':
+    cmd, cmdkwargs = parse_args()
+    main(cmd, cmdkwargs)

Tools/c-analyzer/c_analyzer_common/__init__.py

+import os.path
+
+
+PKG_ROOT = os.path.dirname(__file__)
+DATA_DIR = os.path.dirname(PKG_ROOT)
+REPO_ROOT = os.path.dirname(
+        os.path.dirname(DATA_DIR))
+
+SOURCE_DIRS = [os.path.join(REPO_ROOT, name) for name in [
+        'Include',
+        'Python',
+        'Parser',
+        'Objects',
+        'Modules',
+        ]]
+
+
+# Clean up the namespace.
+del os

Tools/c-analyzer/c_analyzer_common/files.py

+import glob
+import os
+import os.path
+
+from . import SOURCE_DIRS, REPO_ROOT
+
+
+C_SOURCE_SUFFIXES = ('.c', '.h')
+
+
+def _walk_tree(root, *,
+               _walk=os.walk,
+               ):
+    # A wrapper around os.walk that resolves the filenames.
+    for parent, _, names in _walk(root):
+        for name in names:
+            yield os.path.join(parent, name)
+
+
+def walk_tree(root, *,
+              suffix=None,
+              walk=_walk_tree,
+              ):
+    """Yield each file in the tree under the given directory name.
+
+    If "suffix" is provided then only files with that suffix will
+    be included.
+    """
+    if suffix and not isinstance(suffix, str):
+        raise ValueError('suffix must be a string')
+
+    for filename in walk(root):
+        if suffix and not filename.endswith(suffix):
+            continue
+        yield filename
+
+
+def glob_tree(root, *,
+              suffix=None,
+              _glob=glob.iglob,
+              ):
+    """Yield each file in the tree under the given directory name.
+
+    If "suffix" is provided then only files with that suffix will
+    be included.
+    """
+    suffix = suffix or ''
+    if not isinstance(suffix, str):
+        raise ValueError('suffix must be a string')
+
+    for filename in _glob(f'{root}/*{suffix}'):
+        yield filename
+    for filename in _glob(f'{root}/**/*{suffix}'):
+        yield filename
+
+
+def iter_files(root, suffix=None, relparent=None, *,
+               get_files=os.walk,
+               _glob=glob_tree,
+               _walk=walk_tree,
+               ):
+    """Yield each file in the tree under the given directory name.
+
+    If "root" is a non-string iterable then do the same for each of
+    those trees.
+
+    If "suffix" is provided then only files with that suffix will
+    be included.
+
+    if "relparent" is provided then it is used to resolve each
+    filename as a relative path.
+    """
+    if not isinstance(root, str):
+        roots = root
+        for root in roots:
+            yield from iter_files(root, suffix, relparent,
+                                  get_files=get_files,
+                                  _glob=_glob, _walk=_walk)
+        return
+
+    # Use the right "walk" function.
+    if get_files in (glob.glob, glob.iglob, glob_tree):
+        get_files = _glob
+    else:
+        _files = _walk_tree if get_files in (os.walk, walk_tree) else get_files
+        get_files = (lambda *a, **k: _walk(*a, walk=_files, **k))
+
+    # Handle a single suffix.
+    if suffix and not isinstance(suffix, str):
+        filenames = get_files(root)
+        suffix = tuple(suffix)
+    else:
+        filenames = get_files(root, suffix=suffix)
+        suffix = None
+
+    for filename in filenames:
+        if suffix and not isinstance(suffix, str):  # multiple suffixes
+            if not filename.endswith(suffix):
+                continue
+        if relparent:
+            filename = os.path.relpath(filename, relparent)
+        yield filename
+
+
+def iter_files_by_suffix(root, suffixes, relparent=None, *,
+                         walk=walk_tree,
+                         _iter_files=iter_files,
+                         ):
+    """Yield each file in the tree that has the given suffixes.
+
+    Unlike iter_files(), the results are in the original suffix order.
+    """
+    if isinstance(suffixes, str):
+        suffixes = [suffixes]
+    # XXX Ignore repeated suffixes?
+    for suffix in suffixes:
+        yield from _iter_files(root, suffix, relparent)
+
+
+def iter_cpython_files(*,
+                       walk=walk_tree,
+                       _files=iter_files_by_suffix,
+                       ):
+    """Yield each file in the tree for each of the given directory names."""
+    excludedtrees = [
+        os.path.join('Include', 'cpython', ''),
+        ]
+    def is_excluded(filename):
+        for root in excludedtrees:
+            if filename.startswith(root):
+                return True
+        return False
+    for filename in _files(SOURCE_DIRS, C_SOURCE_SUFFIXES, REPO_ROOT,
+                           walk=walk,
+                           ):
+        if is_excluded(filename):
+            continue
+        yield filename

Tools/c-analyzer/c_globals/show.py

+
+def basic(globals, *,
+          _print=print):
+    """Print each row simply."""
+    for variable in globals:
+        if variable.funcname:
+            line = f'{variable.filename}:{variable.funcname}():{variable.name}'
+        else:
+            line = f'{variable.filename}:{variable.name}'
+        vartype = variable.vartype
+        #if vartype.startswith('static '):
+        #    vartype = vartype.partition(' ')[2]
+        #else:
+        #    vartype = '=' + vartype
+        line = f'{line:<64} {vartype}'
+        _print(line)

Tools/c-analyzer/ignored.tsv

+filename	funcname	name	kind	reason