From 8396e79f0db925e3c3f3157ef9792d30d23a0eba Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 20 Jan 2024 21:17:43 -0600 Subject: [PATCH 01/10] init commit of cherry-picked changes from #341 --- fortls/__init__.py | 6 +- fortls/langserver.py | 21 +++- fortls/parsers/internal/parser.py | 104 ++++++++++++++---- fortls/regex_patterns.py | 15 ++- test/test_preproc.py | 15 ++- test/test_server.py | 5 + test/test_source/pp/.pp_conf.json | 2 +- test/test_source/pp/include/indent.h | 22 ++++ .../pp/preproc_spacing_arg_defs.F90 | 42 +++++++ 9 files changed, 196 insertions(+), 36 deletions(-) create mode 100644 test/test_source/pp/include/indent.h create mode 100644 test/test_source/pp/preproc_spacing_arg_defs.F90 diff --git a/fortls/__init__.py b/fortls/__init__.py index faebde88..a4fc3209 100644 --- a/fortls/__init__.py +++ b/fortls/__init__.py @@ -501,7 +501,11 @@ def locate_config(root: str) -> str | None: error_exit(f"Reading file failed: {err_str}") print(f" Detected format: {'fixed' if file_obj.fixed else 'free'}") print("\n=========\nParser Output\n=========\n") - file_ast = file_obj.parse(debug=True, pp_defs=pp_defs, include_dirs=include_dirs) + file_ast = file_obj.parse( + debug=True, + pp_defs=pp_defs, + include_dirs=include_dirs, + ) print("\n=========\nObject Tree\n=========\n") for obj in file_ast.get_scopes(): print("{}: {}".format(obj.get_type(), obj.FQSN)) diff --git a/fortls/langserver.py b/fortls/langserver.py index 422061d9..ce79ec3b 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -751,11 +751,18 @@ def get_definition( return None # Search in Preprocessor defined variables if def_name in def_file.pp_defs: + def_value = def_file.pp_defs.get(def_name) + def_arg_str = "" + if isinstance(def_value, tuple): + def_arg_str, def_value = def_value + def_arg_str = ", ".join([x.strip() for x in def_arg_str.split(",")]) + def_arg_str = f"({def_arg_str})" + var = Variable( def_file.ast, def_line + 1, def_name, - f"#define {def_name} {def_file.pp_defs.get(def_name)}", + f"#define {def_name}{def_arg_str} {def_value}", [], ) return var @@ -1316,7 +1323,9 @@ def serve_onChange(self, request: dict): # Update inheritance (currently file only) # tmp_file.ast.resolve_links(self.obj_tree, self.link_version) elif file_obj.preproc: - file_obj.preprocess(pp_defs=self.pp_defs) + file_obj.preprocess( + pp_defs=self.pp_defs, + ) self.pp_defs = {**self.pp_defs, **file_obj.pp_defs} def serve_onOpen(self, request: dict): @@ -1389,7 +1398,8 @@ def update_workspace_file( if not file_changed: return False, None ast_new = file_obj.parse( - pp_defs=self.pp_defs, include_dirs=self.include_dirs + pp_defs=self.pp_defs, + include_dirs=self.include_dirs, ) # Add the included read in pp_defs from to the ones specified in the # configuration file @@ -1453,7 +1463,10 @@ def file_init( # This is a bypass. # For more see on SO: shorturl.at/hwAG1 set_keyword_ordering(sort) - file_ast = file_obj.parse(pp_defs=pp_defs, include_dirs=include_dirs) + file_ast = file_obj.parse( + pp_defs=pp_defs, + include_dirs=include_dirs, + ) except: log.error("Error while parsing file %s", filepath, exc_info=True) return "Error during parsing" diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 9c4c4558..cca275dc 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -1176,7 +1176,10 @@ def find_word_in_code_line( return line_no, word_range def preprocess( - self, pp_defs: dict = None, include_dirs: set = None, debug: bool = False + self, + pp_defs: dict = None, + include_dirs: set = None, + debug: bool = False, ) -> tuple[list, list]: if pp_defs is None: pp_defs = {} @@ -1265,7 +1268,9 @@ def parse( if self.preproc: log.debug("=== PreProc Pass ===\n") pp_skips, pp_defines = self.preprocess( - pp_defs=pp_defs, include_dirs=include_dirs, debug=debug + pp_defs=pp_defs, + include_dirs=include_dirs, + debug=debug, ) for pp_reg in pp_skips: file_ast.start_ppif(pp_reg[0]) @@ -2038,6 +2043,7 @@ def replace_ops(expr: str): expr = expr.replace("!=", " <> ") expr = expr.replace("!", " not ") expr = expr.replace(" <> ", " != ") + return expr def replace_defined(line: str): @@ -2070,7 +2076,9 @@ def replace_vars(line: str): if defs is None: defs = {} - out_line = replace_defined(text) + + out_line = text + out_line = replace_defined(out_line) out_line = replace_vars(out_line) try: line_res = eval(replace_ops(out_line)) @@ -2098,26 +2106,27 @@ def replace_vars(line: str): if def_cont_name is not None: output_file.append("") if line.rstrip()[-1] != "\\": - defs_tmp[def_cont_name] += line.strip() + append_multiline_macro(defs_tmp, def_cont_name, line.strip()) def_cont_name = None else: - defs_tmp[def_cont_name] += line[0:-1].strip() + append_multiline_macro(defs_tmp, def_cont_name, line[0:-1].strip()) + continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) - if match: + if match and check_pp_prefix(match.group(1)): output_file.append(line) def_name = None if_start = False # Opening conditional statements - if match.group(1) == "if ": - is_path = eval_pp_if(line[match.end(1) :], defs_tmp) + if match.group(2).lower() == "if ": + is_path = eval_pp_if(line[match.end(2) :], defs_tmp) if_start = True - elif match.group(1) == "ifdef": + elif match.group(2).lower() == "ifdef": if_start = True def_name = line[match.end(0) :].strip() is_path = def_name in defs_tmp - elif match.group(1) == "ifndef": + elif match.group(2).lower() == "ifndef": if_start = True def_name = line[match.end(0) :].strip() is_path = not (def_name in defs_tmp) @@ -2135,7 +2144,7 @@ def replace_vars(line: str): inc_start = False exc_start = False exc_continue = False - if match.group(1) == "elif": + if match.group(2).lower() == "elif": if (not pp_stack_group) or (pp_stack_group[-1][0] != len(pp_stack)): # First elif statement for this elif group if pp_stack[-1][0] < 0: @@ -2147,7 +2156,7 @@ def replace_vars(line: str): exc_continue = True if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 - elif eval_pp_if(line[match.end(1) :], defs_tmp): + elif eval_pp_if(line[match.end(2) :], defs_tmp): pp_stack[-1][1] = i + 1 pp_skips.append(pp_stack.pop()) pp_stack_group[-1][1] = True @@ -2155,7 +2164,7 @@ def replace_vars(line: str): inc_start = True else: exc_start = True - elif match.group(1) == "else": + elif match.group(2).lower() == "else": if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 exc_start = True @@ -2171,7 +2180,7 @@ def replace_vars(line: str): pp_skips.append(pp_stack.pop()) pp_stack.append([-1, -1]) inc_start = True - elif match.group(1) == "endif": + elif match.group(2).lower() == "endif": if pp_stack_group and (pp_stack_group[-1][0] == len(pp_stack)): pp_stack_group.pop() if pp_stack[-1][0] < 0: @@ -2192,10 +2201,12 @@ def replace_vars(line: str): continue # Handle variable/macro definitions files match = FRegex.PP_DEF.match(line) - if (match is not None) and ((len(pp_stack) == 0) or (pp_stack[-1][0] < 0)): + if (match is not None and check_pp_prefix(match.group(1))) and ( + (len(pp_stack) == 0) or (pp_stack[-1][0] < 0) + ): output_file.append(line) pp_defines.append(i + 1) - def_name = match.group(2) + def_name = match.group(3) # If this is an argument list of a function add them to the name # get_definition will only return the function name upon hover # hence if the argument list is appended in the def_name then @@ -2204,18 +2215,29 @@ def replace_vars(line: str): # This also does not allow for multiline argument list definitions. # if match.group(3): # def_name += match.group(3) - if (match.group(1) == "define") and (def_name not in defs_tmp): + if (match.group(2) == "define") and (def_name not in defs_tmp): eq_ind = line[match.end(0) :].find(" ") + if eq_ind < 0: + eq_ind = line[match.end(0) :].find("\t") + if eq_ind >= 0: # Handle multiline macros if line.rstrip()[-1] == "\\": - defs_tmp[def_name] = line[match.end(0) + eq_ind : -1].strip() + def_value = line[match.end(0) + eq_ind : -1].strip() def_cont_name = def_name else: - defs_tmp[def_name] = line[match.end(0) + eq_ind :].strip() + def_value = line[match.end(0) + eq_ind :].strip() else: - defs_tmp[def_name] = "True" - elif (match.group(1) == "undef") and (def_name in defs_tmp): + def_value = "True" + + # are there arguments to parse? + if match.group(4): + def_value = (match.group(5), def_value) + + defs_tmp[def_name] = def_value + elif ( + match.group(2) == "undef" + ) and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue @@ -2265,8 +2287,16 @@ def replace_vars(line: str): continue def_regex = def_regexes.get(def_tmp) if def_regex is None: - def_regex = re.compile(rf"\b{def_tmp}\b") + if isinstance(value, tuple): + def_regex = expand_def_func_macro(def_tmp, value) + else: + def_regex = re.compile(rf"\b{def_tmp}\b") + def_regexes[def_tmp] = def_regex + + if isinstance(def_regex, tuple): + def_regex, value = def_regex + line_new, nsubs = def_regex.subn(value, line) if nsubs > 0: log.debug( @@ -2275,3 +2305,33 @@ def replace_vars(line: str): line = line_new output_file.append(line) return output_file, pp_skips, pp_defines, defs_tmp + + +def expand_def_func_macro(def_name: str, def_value: tuple[str, str]): + def_args, sub = def_value + def_args = def_args.split(",") + regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") + + for i, arg in enumerate(def_args): + arg = arg.strip() + sub = re.sub(rf"\b({arg})\b", rf"\\{i + 1}", sub) + + return regex, sub + + +def append_multiline_macro(pp_defs: dict, def_name: str, line: str): + def_value = pp_defs[def_name] + def_args = None + if isinstance(def_value, tuple): + def_args, def_value = def_value + + def_value += line + + if def_args is not None: + def_value = (def_args, def_value) + + pp_defs[def_name] = def_value + + +def check_pp_prefix(prefix: str): + return prefix == "#" diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 69c10f48..55cd2421 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,11 +124,18 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#(if |ifdef|ifndef|else|elif|endif)") - PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I) + PP_REGEX: Pattern = compile( + r"[ ]*(#)[ ]*(if |ifdef|ifndef|else|elif|endif)", + I, + ) + PP_DEF: Pattern = compile( + r"[ ]*(#)[ ]*(define|undef|undefined)" + r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + I, + ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#include[ ]*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^#:?\w+)") + PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]+([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^[ ]*(?:#)[ ]*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_preproc.py b/test/test_preproc.py index 50f50607..7d2e77b9 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -42,6 +42,10 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_if_elif_skip.F90" string += hover_req(file_path, 30, 23) + file_path = root_dir / "preproc_spacing_arg_defs.F90" + string += hover_req(file_path, 11, 20) + string += hover_req(file_path, 20, 17) + string += hover_req(file_path, 22, 13) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -52,12 +56,12 @@ def check_return(result_array, checks): "```fortran90\n#define PETSC_ERR_INT_OVERFLOW 84\n```", "```fortran90\n#define varVar 55\n```", ( - "```fortran90\n#define ewrite if (priority <= 3) write((priority)," - " format)\n```" + "```fortran90\n#define ewrite(priority, format)" + " if (priority <= 3) write((priority), format)\n```" ), ( - "```fortran90\n#define ewrite2 if (priority <= 3) write((priority)," - " format)\n```" + "```fortran90\n#define ewrite2(priority, format)" + " if (priority <= 3) write((priority), format)\n```" ), "```fortran90\n#define SUCCESS .true.\n```", "```fortran90\nREAL, CONTIGUOUS, POINTER, DIMENSION(:) :: var1\n```", @@ -68,6 +72,9 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 0+1+0+0\n```", "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", + "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", + "```fortran90\nSUBROUTINE test_type_set_test()\n```", + "```fortran90\n#define MACROARGS(x, y) x + y\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) diff --git a/test/test_server.py b/test/test_server.py index 639ef427..c3a35dc6 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -175,6 +175,7 @@ def test_workspace_symbols(): def check_return(result_array): # Expected objects objs = ( + ["argtest", 13, 16], ["test", 6, 7], ["test_abstract", 2, 0], ["test_associate_block", 2, 0], @@ -196,7 +197,11 @@ def check_return(result_array): ["test_str1", 13, 5], ["test_str2", 13, 5], ["test_sub", 6, 8], + ["test_type", 5, 5], + ["test_type_set_test", 6, 25], ["test_vis_mod", 2, 0], + ["the_test", 13, 15], + ["wrap_test_type_set_test", 6, 33], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/pp/.pp_conf.json b/test/test_source/pp/.pp_conf.json index 0cf75a8a..246c4b53 100644 --- a/test/test_source/pp/.pp_conf.json +++ b/test/test_source/pp/.pp_conf.json @@ -7,5 +7,5 @@ "pp_suffixes": [".h", ".F90"], "incl_suffixes": [".h"], "include_dirs": ["include"], - "pp_defs": { "HAVE_CONTIGUOUS": "" } + "pp_defs": { "HAVE_CONTIGUOUS": "" }, } diff --git a/test/test_source/pp/include/indent.h b/test/test_source/pp/include/indent.h new file mode 100644 index 00000000..f477f249 --- /dev/null +++ b/test/test_source/pp/include/indent.h @@ -0,0 +1,22 @@ +!! sample code adapted from json-fortran/json_macros.inc + + # define SPACING_TEST + # define FILE_ENCODING ,encoding='UTF-8' + +# ifdef __GFORTRAN__ +! gfortran uses cpp in old-school compatibility mode so +! the # stringify and ## concatenate operators don't work +! but we can use C/C++ style comment to ensure PROCEDURE is +! correctly tokenized and prepended with 'wrap_' when the +! macro is expanded +# define MAYBEWRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE +# else +! Intel's fpp does support the more contemporary ## concatenation +! operator, but doesn't treat the C/C++ comments the same way. +! If you use the gfortran approach and pass the -noB switch to +! fpp, the macro will expand, but with a space between wrap_ and +! whatever PROCEDURE expands to +# define MAYBEWRAP(PROCEDURE) PROCEDURE +# endif + +# define MACROARGS( x , y ) x + y diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 new file mode 100644 index 00000000..0ecc0f02 --- /dev/null +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -0,0 +1,42 @@ +program preprocessor_spacing_arg_defs + implicit none + + #include "indent.h" + + type :: test_type + private + integer, public :: test_int + + contains + generic, public :: set_test => MAYBEWRAP(test_type_set_test) + procedure :: MAYBEWRAP(test_type_set_test) + + end type test_type + + type(test_type) :: the_test + integer :: argtest + + INTEGER (KIND=4), PARAMETER :: C_LONG = 4 + + call the_test%set_test() + + argtest = MACROARGS(the_test%test_int, C_LONG) + +contains + subroutine test_type_set_test(me) + implicit none + + class(test_type), intent(inout) :: me + + me%test_int = 3 + end subroutine test_type_set_test + + subroutine wrap_test_type_set_test(me) + implicit none + + class(test_type), intent(inout) :: me + + me%test_int = 5 + end subroutine wrap_test_type_set_test + +end program preprocessor_spacing_arg_defs From 5987186697cb838d864c3558c4aab3ff1f960eaa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 21 Jan 2024 03:26:31 +0000 Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- fortls/parsers/internal/parser.py | 4 +--- fortls/regex_patterns.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index cca275dc..1288a5cd 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2235,9 +2235,7 @@ def replace_vars(line: str): def_value = (match.group(5), def_value) defs_tmp[def_name] = def_value - elif ( - match.group(2) == "undef" - ) and (def_name in defs_tmp): + elif (match.group(2) == "undef") and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 55cd2421..656cfed1 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -129,8 +129,7 @@ class FortranRegularExpressions: I, ) PP_DEF: Pattern = compile( - r"[ ]*(#)[ ]*(define|undef|undefined)" - r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + r"[ ]*(#)[ ]*(define|undef|undefined)" r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) From 6e9f5f751534e43e250a512e47211d1e8cd80911 Mon Sep 17 00:00:00 2001 From: gnikit Date: Sat, 30 Mar 2024 21:26:38 +0000 Subject: [PATCH 03/10] refactor: edit incoming commits from stale PR --- fortls/__init__.py | 6 +-- fortls/langserver.py | 12 ++--- fortls/parsers/internal/parser.py | 81 ++++++++++++------------------- fortls/regex_patterns.py | 11 ++--- test/test_source/pp/.pp_conf.json | 2 +- 5 files changed, 39 insertions(+), 73 deletions(-) diff --git a/fortls/__init__.py b/fortls/__init__.py index a4fc3209..faebde88 100644 --- a/fortls/__init__.py +++ b/fortls/__init__.py @@ -501,11 +501,7 @@ def locate_config(root: str) -> str | None: error_exit(f"Reading file failed: {err_str}") print(f" Detected format: {'fixed' if file_obj.fixed else 'free'}") print("\n=========\nParser Output\n=========\n") - file_ast = file_obj.parse( - debug=True, - pp_defs=pp_defs, - include_dirs=include_dirs, - ) + file_ast = file_obj.parse(debug=True, pp_defs=pp_defs, include_dirs=include_dirs) print("\n=========\nObject Tree\n=========\n") for obj in file_ast.get_scopes(): print("{}: {}".format(obj.get_type(), obj.FQSN)) diff --git a/fortls/langserver.py b/fortls/langserver.py index ce79ec3b..484ea765 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -1323,9 +1323,7 @@ def serve_onChange(self, request: dict): # Update inheritance (currently file only) # tmp_file.ast.resolve_links(self.obj_tree, self.link_version) elif file_obj.preproc: - file_obj.preprocess( - pp_defs=self.pp_defs, - ) + file_obj.preprocess(pp_defs=self.pp_defs) self.pp_defs = {**self.pp_defs, **file_obj.pp_defs} def serve_onOpen(self, request: dict): @@ -1398,8 +1396,7 @@ def update_workspace_file( if not file_changed: return False, None ast_new = file_obj.parse( - pp_defs=self.pp_defs, - include_dirs=self.include_dirs, + pp_defs=self.pp_defs, include_dirs=self.include_dirs ) # Add the included read in pp_defs from to the ones specified in the # configuration file @@ -1463,10 +1460,7 @@ def file_init( # This is a bypass. # For more see on SO: shorturl.at/hwAG1 set_keyword_ordering(sort) - file_ast = file_obj.parse( - pp_defs=pp_defs, - include_dirs=include_dirs, - ) + file_ast = file_obj.parse(pp_defs=pp_defs, include_dirs=include_dirs) except: log.error("Error while parsing file %s", filepath, exc_info=True) return "Error during parsing" diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 1288a5cd..878c22ff 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -1176,10 +1176,7 @@ def find_word_in_code_line( return line_no, word_range def preprocess( - self, - pp_defs: dict = None, - include_dirs: set = None, - debug: bool = False, + self, pp_defs: dict = None, include_dirs: set = None, debug: bool = False ) -> tuple[list, list]: if pp_defs is None: pp_defs = {} @@ -1268,9 +1265,7 @@ def parse( if self.preproc: log.debug("=== PreProc Pass ===\n") pp_skips, pp_defines = self.preprocess( - pp_defs=pp_defs, - include_dirs=include_dirs, - debug=debug, + pp_defs=pp_defs, include_dirs=include_dirs, debug=debug ) for pp_reg in pp_skips: file_ast.start_ppif(pp_reg[0]) @@ -2043,7 +2038,6 @@ def replace_ops(expr: str): expr = expr.replace("!=", " <> ") expr = expr.replace("!", " not ") expr = expr.replace(" <> ", " != ") - return expr def replace_defined(line: str): @@ -2076,9 +2070,7 @@ def replace_vars(line: str): if defs is None: defs = {} - - out_line = text - out_line = replace_defined(out_line) + out_line = replace_defined(text) out_line = replace_vars(out_line) try: line_res = eval(replace_ops(out_line)) @@ -2106,27 +2098,30 @@ def replace_vars(line: str): if def_cont_name is not None: output_file.append("") if line.rstrip()[-1] != "\\": - append_multiline_macro(defs_tmp, def_cont_name, line.strip()) + defs_tmp[def_cont_name] = append_multiline_macro( + defs_tmp[def_cont_name], line.strip() + ) def_cont_name = None else: - append_multiline_macro(defs_tmp, def_cont_name, line[0:-1].strip()) - + defs_tmp[def_cont_name] = append_multiline_macro( + defs_tmp[def_cont_name], line[0:-1].strip() + ) continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) - if match and check_pp_prefix(match.group(1)): + if match: output_file.append(line) def_name = None if_start = False # Opening conditional statements - if match.group(2).lower() == "if ": - is_path = eval_pp_if(line[match.end(2) :], defs_tmp) + if match.group(1).lower() == "if ": + is_path = eval_pp_if(line[match.end(1) :], defs_tmp) if_start = True - elif match.group(2).lower() == "ifdef": + elif match.group(1).lower() == "ifdef": if_start = True def_name = line[match.end(0) :].strip() is_path = def_name in defs_tmp - elif match.group(2).lower() == "ifndef": + elif match.group(1).lower() == "ifndef": if_start = True def_name = line[match.end(0) :].strip() is_path = not (def_name in defs_tmp) @@ -2144,7 +2139,7 @@ def replace_vars(line: str): inc_start = False exc_start = False exc_continue = False - if match.group(2).lower() == "elif": + if match.group(1).lower() == "elif": if (not pp_stack_group) or (pp_stack_group[-1][0] != len(pp_stack)): # First elif statement for this elif group if pp_stack[-1][0] < 0: @@ -2156,7 +2151,7 @@ def replace_vars(line: str): exc_continue = True if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 - elif eval_pp_if(line[match.end(2) :], defs_tmp): + elif eval_pp_if(line[match.end(1) :], defs_tmp): pp_stack[-1][1] = i + 1 pp_skips.append(pp_stack.pop()) pp_stack_group[-1][1] = True @@ -2164,7 +2159,7 @@ def replace_vars(line: str): inc_start = True else: exc_start = True - elif match.group(2).lower() == "else": + elif match.group(1).lower() == "else": if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 exc_start = True @@ -2180,7 +2175,7 @@ def replace_vars(line: str): pp_skips.append(pp_stack.pop()) pp_stack.append([-1, -1]) inc_start = True - elif match.group(2).lower() == "endif": + elif match.group(1).lower() == "endif": if pp_stack_group and (pp_stack_group[-1][0] == len(pp_stack)): pp_stack_group.pop() if pp_stack[-1][0] < 0: @@ -2201,12 +2196,10 @@ def replace_vars(line: str): continue # Handle variable/macro definitions files match = FRegex.PP_DEF.match(line) - if (match is not None and check_pp_prefix(match.group(1))) and ( - (len(pp_stack) == 0) or (pp_stack[-1][0] < 0) - ): + if (match is not None) and ((len(pp_stack) == 0) or (pp_stack[-1][0] < 0)): output_file.append(line) pp_defines.append(i + 1) - def_name = match.group(3) + def_name = match.group(2) # If this is an argument list of a function add them to the name # get_definition will only return the function name upon hover # hence if the argument list is appended in the def_name then @@ -2215,11 +2208,8 @@ def replace_vars(line: str): # This also does not allow for multiline argument list definitions. # if match.group(3): # def_name += match.group(3) - if (match.group(2) == "define") and (def_name not in defs_tmp): + if (match.group(1) == "define") and (def_name not in defs_tmp): eq_ind = line[match.end(0) :].find(" ") - if eq_ind < 0: - eq_ind = line[match.end(0) :].find("\t") - if eq_ind >= 0: # Handle multiline macros if line.rstrip()[-1] == "\\": @@ -2231,11 +2221,11 @@ def replace_vars(line: str): def_value = "True" # are there arguments to parse? - if match.group(4): - def_value = (match.group(5), def_value) + if match.group(3): + def_value = (match.group(4), def_value) defs_tmp[def_name] = def_value - elif (match.group(2) == "undef") and (def_name in defs_tmp): + elif (match.group(1) == "undef") and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue @@ -2286,10 +2276,9 @@ def replace_vars(line: str): def_regex = def_regexes.get(def_tmp) if def_regex is None: if isinstance(value, tuple): - def_regex = expand_def_func_macro(def_tmp, value) + def_regex = expand_func_macro(def_tmp, value) else: def_regex = re.compile(rf"\b{def_tmp}\b") - def_regexes[def_tmp] = def_regex if isinstance(def_regex, tuple): @@ -2305,7 +2294,7 @@ def replace_vars(line: str): return output_file, pp_skips, pp_defines, defs_tmp -def expand_def_func_macro(def_name: str, def_value: tuple[str, str]): +def expand_func_macro(def_name: str, def_value: tuple[str, str]): def_args, sub = def_value def_args = def_args.split(",") regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") @@ -2317,19 +2306,9 @@ def expand_def_func_macro(def_name: str, def_value: tuple[str, str]): return regex, sub -def append_multiline_macro(pp_defs: dict, def_name: str, line: str): - def_value = pp_defs[def_name] - def_args = None +def append_multiline_macro(def_value: str | tuple, line: str): if isinstance(def_value, tuple): def_args, def_value = def_value - - def_value += line - - if def_args is not None: - def_value = (def_args, def_value) - - pp_defs[def_name] = def_value - - -def check_pp_prefix(prefix: str): - return prefix == "#" + def_value += line + return (def_args, def_value) + return def_value + line diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 656cfed1..46cc5d28 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,17 +124,14 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile( - r"[ ]*(#)[ ]*(if |ifdef|ifndef|else|elif|endif)", - I, - ) + PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)", I) PP_DEF: Pattern = compile( - r"[ ]*(#)[ ]*(define|undef|undefined)" r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + r"[ ]*#[ ]*(define|undef|undefined)[ ]*(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]+([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^[ ]*(?:#)[ ]*\w*:?\w+)") + PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]*([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"^[ ]*#:?[ ]*(\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_source/pp/.pp_conf.json b/test/test_source/pp/.pp_conf.json index 246c4b53..0cf75a8a 100644 --- a/test/test_source/pp/.pp_conf.json +++ b/test/test_source/pp/.pp_conf.json @@ -7,5 +7,5 @@ "pp_suffixes": [".h", ".F90"], "incl_suffixes": [".h"], "include_dirs": ["include"], - "pp_defs": { "HAVE_CONTIGUOUS": "" }, + "pp_defs": { "HAVE_CONTIGUOUS": "" } } From 4e27b16684033112350b72d75cbb525290654532 Mon Sep 17 00:00:00 2001 From: gnikit Date: Mon, 1 Apr 2024 09:03:30 +0100 Subject: [PATCH 04/10] refactor: simplify multiline macro parsing --- fortls/parsers/internal/parser.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 878c22ff..1fe2f307 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2097,15 +2097,13 @@ def replace_vars(line: str): # Handle multiline macro continuation if def_cont_name is not None: output_file.append("") - if line.rstrip()[-1] != "\\": - defs_tmp[def_cont_name] = append_multiline_macro( - defs_tmp[def_cont_name], line.strip() - ) + is_multiline = line.strip()[-1] != "\\" + line_to_append = line.strip() if is_multiline else line[0:-1].strip() + defs_tmp[def_cont_name] = append_multiline_macro( + defs_tmp[def_cont_name], line_to_append + ) + if is_multiline: def_cont_name = None - else: - defs_tmp[def_cont_name] = append_multiline_macro( - defs_tmp[def_cont_name], line[0:-1].strip() - ) continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) From 0d3921df98cce0599a72884d3a1d7a170956912b Mon Sep 17 00:00:00 2001 From: gnikit Date: Mon, 1 Apr 2024 09:07:55 +0100 Subject: [PATCH 05/10] refactor(pp): make pp specific functions inner --- fortls/parsers/internal/parser.py | 38 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 1fe2f307..bc218546 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2079,6 +2079,24 @@ def replace_vars(line: str): else: return line_res + def expand_func_macro(def_name: str, def_value: tuple[str, str]): + def_args, sub = def_value + def_args = def_args.split(",") + regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") + + for i, arg in enumerate(def_args): + arg = arg.strip() + sub = re.sub(rf"\b({arg})\b", rf"\\{i + 1}", sub) + + return regex, sub + + def append_multiline_macro(def_value: str | tuple, line: str): + if isinstance(def_value, tuple): + def_args, def_value = def_value + def_value += line + return (def_args, def_value) + return def_value + line + if pp_defs is None: pp_defs = {} if include_dirs is None: @@ -2290,23 +2308,3 @@ def replace_vars(line: str): line = line_new output_file.append(line) return output_file, pp_skips, pp_defines, defs_tmp - - -def expand_func_macro(def_name: str, def_value: tuple[str, str]): - def_args, sub = def_value - def_args = def_args.split(",") - regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") - - for i, arg in enumerate(def_args): - arg = arg.strip() - sub = re.sub(rf"\b({arg})\b", rf"\\{i + 1}", sub) - - return regex, sub - - -def append_multiline_macro(def_value: str | tuple, line: str): - if isinstance(def_value, tuple): - def_args, def_value = def_value - def_value += line - return (def_args, def_value) - return def_value + line From aa18a6819b71c912335e5fa185fcc777da8423ac Mon Sep 17 00:00:00 2001 From: gnikit Date: Sat, 13 Apr 2024 17:22:50 +0100 Subject: [PATCH 06/10] refactor: remove preprocessor whitespace test This test is problematic for a few reasons and is in need of refactoring. A summary of the issues are: - macro expansions are already covered in another test - test too big - it pollutes the global workspace of symbols - names of objects are too generic - tests indirectly the preprocessor Its purpose in this PR is to test the preprocessor REGEX patterns work with leading whitespaces and can perform macro expansions. I have opted to remove and write a more compact test. --- test/test_preproc.py | 7 ---- test/test_server.py | 5 --- test/test_source/pp/include/indent.h | 22 ---------- .../pp/preproc_spacing_arg_defs.F90 | 42 ------------------- 4 files changed, 76 deletions(-) delete mode 100644 test/test_source/pp/include/indent.h delete mode 100644 test/test_source/pp/preproc_spacing_arg_defs.F90 diff --git a/test/test_preproc.py b/test/test_preproc.py index 7d2e77b9..15b26026 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -42,10 +42,6 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_if_elif_skip.F90" string += hover_req(file_path, 30, 23) - file_path = root_dir / "preproc_spacing_arg_defs.F90" - string += hover_req(file_path, 11, 20) - string += hover_req(file_path, 20, 17) - string += hover_req(file_path, 22, 13) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -72,9 +68,6 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 0+1+0+0\n```", "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", - "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", - "```fortran90\nSUBROUTINE test_type_set_test()\n```", - "```fortran90\n#define MACROARGS(x, y) x + y\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) diff --git a/test/test_server.py b/test/test_server.py index c3a35dc6..639ef427 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -175,7 +175,6 @@ def test_workspace_symbols(): def check_return(result_array): # Expected objects objs = ( - ["argtest", 13, 16], ["test", 6, 7], ["test_abstract", 2, 0], ["test_associate_block", 2, 0], @@ -197,11 +196,7 @@ def check_return(result_array): ["test_str1", 13, 5], ["test_str2", 13, 5], ["test_sub", 6, 8], - ["test_type", 5, 5], - ["test_type_set_test", 6, 25], ["test_vis_mod", 2, 0], - ["the_test", 13, 15], - ["wrap_test_type_set_test", 6, 33], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/pp/include/indent.h b/test/test_source/pp/include/indent.h deleted file mode 100644 index f477f249..00000000 --- a/test/test_source/pp/include/indent.h +++ /dev/null @@ -1,22 +0,0 @@ -!! sample code adapted from json-fortran/json_macros.inc - - # define SPACING_TEST - # define FILE_ENCODING ,encoding='UTF-8' - -# ifdef __GFORTRAN__ -! gfortran uses cpp in old-school compatibility mode so -! the # stringify and ## concatenate operators don't work -! but we can use C/C++ style comment to ensure PROCEDURE is -! correctly tokenized and prepended with 'wrap_' when the -! macro is expanded -# define MAYBEWRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE -# else -! Intel's fpp does support the more contemporary ## concatenation -! operator, but doesn't treat the C/C++ comments the same way. -! If you use the gfortran approach and pass the -noB switch to -! fpp, the macro will expand, but with a space between wrap_ and -! whatever PROCEDURE expands to -# define MAYBEWRAP(PROCEDURE) PROCEDURE -# endif - -# define MACROARGS( x , y ) x + y diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 deleted file mode 100644 index 0ecc0f02..00000000 --- a/test/test_source/pp/preproc_spacing_arg_defs.F90 +++ /dev/null @@ -1,42 +0,0 @@ -program preprocessor_spacing_arg_defs - implicit none - - #include "indent.h" - - type :: test_type - private - integer, public :: test_int - - contains - generic, public :: set_test => MAYBEWRAP(test_type_set_test) - procedure :: MAYBEWRAP(test_type_set_test) - - end type test_type - - type(test_type) :: the_test - integer :: argtest - - INTEGER (KIND=4), PARAMETER :: C_LONG = 4 - - call the_test%set_test() - - argtest = MACROARGS(the_test%test_int, C_LONG) - -contains - subroutine test_type_set_test(me) - implicit none - - class(test_type), intent(inout) :: me - - me%test_int = 3 - end subroutine test_type_set_test - - subroutine wrap_test_type_set_test(me) - implicit none - - class(test_type), intent(inout) :: me - - me%test_int = 5 - end subroutine wrap_test_type_set_test - -end program preprocessor_spacing_arg_defs From ee9c39a2e94cd4db723df85e36eaf22c540045ff Mon Sep 17 00:00:00 2001 From: gnikit Date: Sat, 13 Apr 2024 17:26:10 +0100 Subject: [PATCH 07/10] test(pp): add test for preprocessor whitespaces --- test/test_preproc_parser.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/test_preproc_parser.py diff --git a/test/test_preproc_parser.py b/test/test_preproc_parser.py new file mode 100644 index 00000000..955e3671 --- /dev/null +++ b/test/test_preproc_parser.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from fortls.parsers.internal.parser import preprocess_file + + +def test_pp_leading_spaces(): + lines = [ + " #define LEADING_SPACES_INDENT 1", + " # define LEADING_SPACES_INDENT2", + " # define FILE_ENCODING ,encoding='UTF-8'", + "program pp_intentation", + " implicit none", + " print*, LEADING_SPACES_INDENT", + " open(unit=1,file='somefile.txt' FILE_ENCODING)", + "end program pp_intentation", + ] + _, _, _, defs = preprocess_file(lines) + ref = { + "LEADING_SPACES_INDENT": "1", + "LEADING_SPACES_INDENT2": "True", + "FILE_ENCODING": ",encoding='UTF-8'", + } + assert defs == ref From 90b8ac5d0f98349d13f734b18aae654bd2f3a5c1 Mon Sep 17 00:00:00 2001 From: gnikit Date: Sat, 13 Apr 2024 19:44:57 +0100 Subject: [PATCH 08/10] refactor(pp): improve code readability --- fortls/parsers/internal/parser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index bc218546..2077ec99 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2084,9 +2084,8 @@ def expand_func_macro(def_name: str, def_value: tuple[str, str]): def_args = def_args.split(",") regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") - for i, arg in enumerate(def_args): - arg = arg.strip() - sub = re.sub(rf"\b({arg})\b", rf"\\{i + 1}", sub) + for i, arg in enumerate(def_args, start=1): + sub = re.sub(rf"\b({arg.strip()})\b", rf"\\{i}", sub) return regex, sub From 008fe949ab1f07e873918a68bc2e4a5b42c2bcff Mon Sep 17 00:00:00 2001 From: gnikit Date: Sat, 13 Apr 2024 19:46:21 +0100 Subject: [PATCH 09/10] test(pp): add macro expansion test We don't actually do the full C expansion, as it can be seen by the C-style comment character still present in the output, but it's better than no expansion at all. --- test/test_preproc_parser.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test_preproc_parser.py b/test/test_preproc_parser.py index 955e3671..c897669a 100644 --- a/test/test_preproc_parser.py +++ b/test/test_preproc_parser.py @@ -21,3 +21,18 @@ def test_pp_leading_spaces(): "FILE_ENCODING": ",encoding='UTF-8'", } assert defs == ref + + +def test_pp_macro_expansion(): + lines = [ + "# define WRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE", + "generic, public :: set => WRAP(abc)", + "procedure :: WRAP(abc)", + ] + ref = [ + "# define WRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE", + "generic, public :: set => abc , wrap_/**/abc", + "procedure :: abc , wrap_/**/abc", + ] + output, _, _, _ = preprocess_file(lines) + assert output == ref From b2e74ad4a8d7384f65d3b6e19da5613c37ba45f8 Mon Sep 17 00:00:00 2001 From: gnikit Date: Sat, 13 Apr 2024 19:55:01 +0100 Subject: [PATCH 10/10] docs: updated CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b78a65e..ded8e3d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ### Added +- Added support for preprocessor macro expansions + ([#368](https://github.com/fortran-lang/fortls/pull/368)) +- Added support for leading white spaces in preprocessor directives + ([#297](https://github.com/fortran-lang/fortls/issues/297)) - Added hover messages for Types and Modules ([#208](https://github.com/fortran-lang/fortls/issues/208)) - Added support for Markdown intrinsics from the M_intrinsics repository