pythongh-133228: c-analyzer clang preprocessor (pythonGH-133229)

* impl
* included 2 failures to tsvs next to similar entries
* added fix/hack for curses.h fails
* fix leftover from debug

Author: dg-pb

Tools/c-analyzer/c_parser/preprocessor/__init__.py

@@ -16,6 +16,7 @@
 from . import (
     pure as _pure,
     gcc as _gcc,
+    clang as _clang,
 )
 
 
@@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None):
     'bcpp': None,
     # aliases/extras:
     'gcc': _gcc.preprocess,
-    'clang': None,
+    'clang': _clang.preprocess,
 }
 
 

Tools/c-analyzer/c_parser/preprocessor/clang.py

@@ -0,0 +1,104 @@
+import os.path
+import re, sys
+
+from . import common as _common
+from . import gcc as _gcc
+
+_normpath = _gcc._normpath
+
+TOOL = 'clang'
+
+META_FILES = {
+    '<built-in>',
+    '<command line>',
+}
+
+
+def preprocess(filename,
+               incldirs=None,
+               includes=None,
+               macros=None,
+               samefiles=None,
+               cwd=None,
+               ):
+    if not cwd or not os.path.isabs(cwd):
+        cwd = os.path.abspath(cwd or '.')
+    filename = _normpath(filename, cwd)
+
+    postargs = _gcc.POST_ARGS
+    basename = os.path.basename(filename)
+    dirname = os.path.basename(os.path.dirname(filename))
+    if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI
+       and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI):
+        postargs += ('-DPy_BUILD_CORE=1',)
+
+    text = _common.preprocess(
+        TOOL,
+        filename,
+        incldirs=incldirs,
+        includes=includes,
+        macros=macros,
+        #preargs=PRE_ARGS,
+        postargs=postargs,
+        executable=['clang'],
+        compiler='unix',
+        cwd=cwd,
+    )
+    return _iter_lines(text, filename, samefiles, cwd)
+
+
+EXIT_MARKERS = {'# 2 "<built-in>" 2', '# 3 "<built-in>" 2', '# 4 "<built-in>" 2'}
+
+
+def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
+    lines = iter(text.splitlines())
+
+    # The first line is special.
+    # The subsequent lines are consistent.
+    firstlines = [
+        f'# 1 "{reqfile}"',
+        '# 1 "<built-in>" 1',
+        '# 1 "<built-in>" 3',
+        '# 370 "<built-in>" 3',
+        '# 1 "<command line>" 1',
+        '# 1 "<built-in>" 2',
+    ]
+    for expected in firstlines:
+        line = next(lines)
+        if line != expected:
+            raise NotImplementedError((line, expected))
+
+    # Do all the CLI-provided includes.
+    filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles))
+    make_info = (lambda lno: _common.FileInfo(reqfile, lno))
+    last = None
+    for line in lines:
+        assert last != reqfile, (last,)
+        # NOTE:condition is clang specific
+        if not line:
+            continue
+        lno, included, flags = _gcc._parse_marker_line(line, reqfile)
+        if not included:
+            raise NotImplementedError((line,))
+        if included == reqfile:
+            # This will be the last one.
+            assert 2 in flags, (line, flags)
+        else:
+            # NOTE:first condition is specific to clang
+            if _normpath(included, cwd) == reqfile:
+                assert 1 in flags or 2 in flags, (line, flags, included, reqfile)
+            else:
+                assert 1 in flags, (line, flags, included, reqfile)
+        yield from _gcc._iter_top_include_lines(
+            lines,
+            _normpath(included, cwd),
+            cwd,
+            filter_reqfile,
+            make_info,
+            raw,
+            EXIT_MARKERS
+        )
+        last = included
+    # The last one is always the requested file.
+    # NOTE:_normpath is clang specific
+    assert _normpath(included, cwd) == reqfile, (line,)

Tools/c-analyzer/c_parser/preprocessor/gcc.py

@@ -65,6 +65,8 @@
     '-E',
 )
 
+EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'}
+
 
 def preprocess(filename,
                incldirs=None,
@@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
             filter_reqfile,
             make_info,
             raw,
+            EXIT_MARKERS
         )
         last = included
     # The last one is always the requested file.
@@ -146,20 +149,28 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
 
 def _iter_top_include_lines(lines, topfile, cwd,
                             filter_reqfile, make_info,
-                            raw):
+                            raw, exit_markers):
     partial = 0  # depth
     files = [topfile]
     # We start at 1 in case there are source lines (including blank ones)
     # before the first marker line.  Also, we already verified in
     # _parse_marker_line() that the preprocessor reported lno as 1.
     lno = 1
     for line in lines:
-        if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2':
+        if line in exit_markers:
             # We're done with this top-level include.
             return
 
         _lno, included, flags = _parse_marker_line(line)
         if included:
+            # HACK:
+            # Mixes curses.h and ncurses.h marker lines
+            # gcc silently passes this, while clang fails
+            # See: /Include/py_curses.h #if-elif directives
+            # And compare with preprocessor output
+            if os.path.basename(included) == 'curses.h':
+                included = os.path.join(os.path.dirname(included), 'ncurses.h')
+
             lno = _lno
             included = _normpath(included, cwd)
             # We hit a marker line.

Tools/c-analyzer/cpython/_parser.py

@@ -340,6 +340,10 @@ def format_tsv_lines(lines):
 
     # Catch-alls:
     _abs('Include/**/*.h'): (5_000, 500),
+
+    # Specific to clang
+    _abs('Modules/selectmodule.c'): (40_000, 3000),
+    _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000),
 }
 
 

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -400,6 +400,7 @@ Modules/_tkinter.c	-	tcl_lock	-
 Modules/_tkinter.c	-	excInCmd	-
 Modules/_tkinter.c	-	valInCmd	-
 Modules/_tkinter.c	-	trbInCmd	-
+Modules/socketmodule.c 	- 	netdb_lock 	-
 
 
 ##################################

Tools/c-analyzer/cpython/ignored.tsv

@@ -16,6 +16,7 @@ filename	funcname	name	reason
 ## indicators for resource availability/capability
 # (set during first init)
 Python/bootstrap_hash.c	py_getrandom	getrandom_works	-
+Python/bootstrap_hash.c	py_getentropy	getentropy_works	-
 Python/fileutils.c	-	_Py_open_cloexec_works	-
 Python/fileutils.c	set_inheritable	ioctl_works	-
 # (set lazily, *after* first init)