gh-90110: Fix the c-analyzer Tool (gh-96731)

This includes:

* update the whitelists
* fixes so we can stop ignoring some of the files
* ensure Include/cpython/*.h get analyzed

Author: ericsnowcurrently

Include/internal/pycore_interp.h

@@ -18,7 +18,6 @@ extern "C" {
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
-#include "pycore_gil.h"           // struct _gil_runtime_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_tuple.h"         // struct _Py_tuple_state

Tools/c-analyzer/c_common/fsutil.py

@@ -104,6 +104,25 @@ def format_filename(filename, relroot=USE_CWD, *,
     return filename
 
 
+def match_path_tail(path1, path2):
+    """Return True if one path ends the other."""
+    if path1 == path2:
+        return True
+    if os.path.isabs(path1):
+        if os.path.isabs(path2):
+            return False
+        return _match_tail(path1, path2)
+    elif os.path.isabs(path2):
+        return _match_tail(path2, path1)
+    else:
+        return _match_tail(path1, path2) or _match_tail(path2, path1)
+
+
+def _match_tail(path, tail):
+    assert not os.path.isabs(tail), repr(tail)
+    return path.endswith(os.path.sep + tail)
+
+
 ##################################
 # find files
 

Tools/c-analyzer/c_parser/__init__.py

@@ -22,8 +22,12 @@ def parse_files(filenames, *,
     if get_file_preprocessor is None:
         get_file_preprocessor = _get_preprocessor()
     for filename in filenames:
-        yield from _parse_file(
-                filename, match_kind, get_file_preprocessor, file_maxsizes)
+        try:
+            yield from _parse_file(
+                    filename, match_kind, get_file_preprocessor, file_maxsizes)
+        except Exception:
+            print(f'# requested file: <{filename}>')
+            raise  # re-raise
 
 
 def _parse_file(filename, match_kind, get_file_preprocessor, maxsizes):

Tools/c-analyzer/c_parser/preprocessor/__init__.py

@@ -35,27 +35,39 @@
 
 def preprocess(source, *,
                incldirs=None,
+               includes=None,
                macros=None,
                samefiles=None,
                filename=None,
+               cwd=None,
                tool=True,
                ):
     """...
 
     CWD should be the project root and "source" should be relative.
     """
     if tool:
-        logger.debug(f'CWD: {os.getcwd()!r}')
-        logger.debug(f'incldirs: {incldirs!r}')
-        logger.debug(f'macros: {macros!r}')
+        if not cwd:
+            cwd = os.getcwd()
+        logger.debug(f'CWD:       {cwd!r}')
+        logger.debug(f'incldirs:  {incldirs!r}')
+        logger.debug(f'includes:  {includes!r}')
+        logger.debug(f'macros:    {macros!r}')
         logger.debug(f'samefiles: {samefiles!r}')
         _preprocess = _get_preprocessor(tool)
         with _good_file(source, filename) as source:
-            return _preprocess(source, incldirs, macros, samefiles) or ()
+            return _preprocess(
+                source,
+                incldirs,
+                includes,
+                macros,
+                samefiles,
+                cwd,
+            ) or ()
     else:
         source, filename = _resolve_source(source, filename)
         # We ignore "includes", "macros", etc.
-        return _pure.preprocess(source, filename)
+        return _pure.preprocess(source, filename, cwd)
 
     # if _run() returns just the lines:
 #    text = _run(source)
@@ -72,6 +84,7 @@ def preprocess(source, *,
 
 def get_preprocessor(*,
                      file_macros=None,
+                     file_includes=None,
                      file_incldirs=None,
                      file_same=None,
                      ignore_exc=False,
@@ -80,27 +93,39 @@ def get_preprocessor(*,
     _preprocess = preprocess
     if file_macros:
         file_macros = tuple(_parse_macros(file_macros))
+    if file_includes:
+        file_includes = tuple(_parse_includes(file_includes))
     if file_incldirs:
         file_incldirs = tuple(_parse_incldirs(file_incldirs))
     if file_same:
-        file_same = tuple(file_same)
+        file_same = dict(file_same or ())
     if not callable(ignore_exc):
         ignore_exc = (lambda exc, _ig=ignore_exc: _ig)
 
     def get_file_preprocessor(filename):
         filename = filename.strip()
         if file_macros:
             macros = list(_resolve_file_values(filename, file_macros))
+        if file_includes:
+            # There's a small chance we could need to filter out any
+            # includes that import "filename".  It isn't clear that it's
+            # a problem any longer.  If we do end up filtering then
+            # it may make sense to use c_common.fsutil.match_path_tail().
+            includes = [i for i, in _resolve_file_values(filename, file_includes)]
         if file_incldirs:
             incldirs = [v for v, in _resolve_file_values(filename, file_incldirs)]
+        if file_same:
+            samefiles = _resolve_samefiles(filename, file_same)
 
         def preprocess(**kwargs):
             if file_macros and 'macros' not in kwargs:
                 kwargs['macros'] = macros
+            if file_includes and 'includes' not in kwargs:
+                kwargs['includes'] = includes
             if file_incldirs and 'incldirs' not in kwargs:
-                kwargs['incldirs'] = [v for v, in _resolve_file_values(filename, file_incldirs)]
-            if file_same and 'file_same' not in kwargs:
-                kwargs['samefiles'] = file_same
+                kwargs['incldirs'] = incldirs
+            if file_same and 'samefiles' not in kwargs:
+                kwargs['samefiles'] = samefiles
             kwargs.setdefault('filename', filename)
             with handling_errors(ignore_exc, log_err=log_err):
                 return _preprocess(filename, **kwargs)
@@ -120,6 +145,11 @@ def _parse_macros(macros):
         yield row
 
 
+def _parse_includes(includes):
+    for row, srcfile in _parse_table(includes, '\t', 'glob\tinclude', default=None):
+        yield row
+
+
 def _parse_incldirs(incldirs):
     for row, srcfile in _parse_table(incldirs, '\t', 'glob\tdirname', default=None):
         glob, dirname = row
@@ -130,6 +160,43 @@ def _parse_incldirs(incldirs):
         yield row
 
 
+def _resolve_samefiles(filename, file_same):
+    assert '*' not in filename, (filename,)
+    assert os.path.normpath(filename) == filename, (filename,)
+    _, suffix = os.path.splitext(filename)
+    samefiles = []
+    for patterns, in _resolve_file_values(filename, file_same.items()):
+        for pattern in patterns:
+            same = _resolve_samefile(filename, pattern, suffix)
+            if not same:
+                continue
+            samefiles.append(same)
+    return samefiles
+
+
+def _resolve_samefile(filename, pattern, suffix):
+    if pattern == filename:
+        return None
+    if pattern.endswith(os.path.sep):
+        pattern += f'*{suffix}'
+    assert os.path.normpath(pattern) == pattern, (pattern,)
+    if '*' in os.path.dirname(pattern):
+        raise NotImplementedError((filename, pattern))
+    if '*' not in os.path.basename(pattern):
+        return pattern
+
+    common = os.path.commonpath([filename, pattern])
+    relpattern = pattern[len(common) + len(os.path.sep):]
+    relpatterndir = os.path.dirname(relpattern)
+    relfile = filename[len(common) + len(os.path.sep):]
+    if os.path.basename(pattern) == '*':
+        return os.path.join(common, relpatterndir, relfile)
+    elif os.path.basename(relpattern) == '*' + suffix:
+        return os.path.join(common, relpatterndir, relfile)
+    else:
+        raise NotImplementedError((filename, pattern))
+
+
 @contextlib.contextmanager
 def handling_errors(ignore_exc=None, *, log_err=None):
     try:

Tools/c-analyzer/c_parser/preprocessor/common.py

@@ -44,7 +44,7 @@ def run_cmd(argv, *,
     return proc.stdout
 
 
-def preprocess(tool, filename, **kwargs):
+def preprocess(tool, filename, cwd=None, **kwargs):
     argv = _build_argv(tool, filename, **kwargs)
     logger.debug(' '.join(shlex.quote(v) for v in argv))
 
@@ -59,19 +59,24 @@ def preprocess(tool, filename, **kwargs):
         # distutil compiler object's preprocess() method, since that
         # one writes to stdout/stderr and it's simpler to do it directly
         # through subprocess.
-        return run_cmd(argv)
+        return run_cmd(argv, cwd=cwd)
 
 
 def _build_argv(
     tool,
     filename,
     incldirs=None,
+    includes=None,
     macros=None,
     preargs=None,
     postargs=None,
     executable=None,
     compiler=None,
 ):
+    if includes:
+        includes = tuple(f'-include{i}' for i in includes)
+        postargs = (includes + postargs) if postargs else includes
+
     compiler = distutils.ccompiler.new_compiler(
         compiler=compiler or tool,
     )