From ce0606a756a6f0c09fd4e07ce820d7ac14838aa4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 14 Jun 2016 12:36:41 -0500 Subject: [PATCH 1/8] [build tools] Added better support for features and recursive configs per @screamerbg --- tools/build_api.py | 53 ++++++++++++++++++++++-------- tools/config.py | 63 ++++++++++++++++++++++++------------ tools/toolchains/__init__.py | 19 ++++++++++- 3 files changed, 101 insertions(+), 34 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index e34e74d6e6f..d9c6835b6cf 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -195,16 +195,30 @@ def build_project(src_path, build_path, target, toolchain_name, else: resources.inc_dirs.append(inc_dirs) - # Update the configuration with any .json files found while scanning - config.add_config_files(resources.json_files) + # Update configuration files until added features creates no changes + prev_features = set() + while True: + # Update the configuration with any .json files found while scanning + config.add_config_files(resources.json_files) + + # Add features while we find new ones + features = config.get_features() + if features == prev_features: + break + + for feature in features: + if feature not in resources.features: + raise KeyError("Feature %s is unavailable" % feature) + resources += resources.features[feature] + + prev_features = features + # And add the configuration macros to the toolchain toolchain.add_macros(config.get_config_data_macros()) # Compile Sources - for path in src_paths: - src = toolchain.scan_resources(path) - objects = toolchain.compile_sources(src, build_path, resources.inc_dirs) - resources.objects.extend(objects) + objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs) + resources.objects.extend(objects) # Link Program res, _ = toolchain.link_program(resources, build_path, name) @@ -237,7 +251,7 @@ def build_project(src_path, build_path, target, toolchain_name, add_result_to_report(report, cur_result) # Let Exception propagate - raise e + raise def build_library(src_paths, build_path, target, toolchain_name, dependencies_paths=None, options=None, name=None, clean=False, archive=True, @@ -346,16 +360,29 @@ def build_library(src_paths, build_path, target, toolchain_name, # Handle configuration config = Config(target) - # Update the configuration with any .json files found while scanning - config.add_config_files(resources.json_files) + + # Update configuration files until added features creates no changes + prev_features = set() + while True: + # Update the configuration with any .json files found while scanning + config.add_config_files(resources.json_files) + + # Add features while we find new ones + features = config.get_features() + if features == prev_features: + break + + for feature in features: + resources += resources.features[feature] + + prev_features = features + # And add the configuration macros to the toolchain toolchain.add_macros(config.get_config_data_macros()) # Compile Sources - for path in src_paths: - src = toolchain.scan_resources(path) - objects = toolchain.compile_sources(src, abspath(tmp_path), resources.inc_dirs) - resources.objects.extend(objects) + objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs) + resources.objects.extend(objects) if archive: toolchain.build_library(objects, build_path, name) diff --git a/tools/config.py b/tools/config.py index 66b10350367..e11bb13c476 100644 --- a/tools/config.py +++ b/tools/config.py @@ -176,7 +176,9 @@ def __init__(self, target, top_level_dirs = []): self.processed_configs = {} self.target = target if isinstance(target, str) else target.name self.target_labels = Target.get_target(self.target).get_labels() - self.target_instance = Target.get_target(self.target) + self.added_features = set() + self.removed_features = set() + self.removed_unecessary_features = False # Add one or more configuration files def add_config_files(self, flist): @@ -212,6 +214,23 @@ def _process_config_parameters(self, data, params, unit_name, unit_kind): params[full_name] = ConfigParameter(name, v if isinstance(v, dict) else {"value": v}, unit_name, unit_kind) return params + # Add features to the available features + def remove_features(self, features): + for feature in features: + if feature in self.added_features: + raise ConfigException("Configuration conflict. Feature %s both added and removed." % feature) + + self.removed_features |= set(features) + + # Remove features from the available features + def add_features(self, features): + for feature in features: + if (feature in self.removed_features + or (self.removed_unecessary_features and feature not in self.added_features)): + raise ConfigException("Configuration conflict. Feature %s both added and removed." % feature) + + self.added_features |= set(features) + # Helper function: process "config_parameters" and "target_config_overrides" in a given dictionary # data: the configuration data of the library/appliation # params: storage for the discovered configuration parameters @@ -222,25 +241,21 @@ def _process_config_and_overrides(self, data, params, unit_name, unit_kind): for label, overrides in data.get("target_overrides", {}).items(): # If the label is defined by the target or it has the special value "*", process the overrides if (label == '*') or (label in self.target_labels): - # Parse out cumulative attributes - for attr in Target._Target__cumulative_attributes: - attrs = getattr(self.target_instance, attr) - - if attr in overrides: - del attrs[:] - attrs.extend(overrides[attr]) - del overrides[attr] - - if attr+'_add' in overrides: - attrs.extend(overrides[attr+'_add']) - del overrides[attr+'_add'] - - if attr+'_remove' in overrides: - for a in overrides[attr+'_remove']: - attrs.remove(a) - del overrides[attr+'_remove'] - - setattr(self.target_instance, attr, attrs) + # Parse out features + if 'features' in overrides: + features = overrides['features'] + self.remove_features(list(set(self.added_features) - set(features))) + self.add_features(features) + self.removed_unecessary_features = True + del overrides['features'] + + if 'features_add' in overrides: + self.add_features(overrides['features_add']) + del overrides['features_add'] + + if 'features_remove' in overrides: + self.remove_features(overrides['features_remove']) + del overrides['features_remove'] # Consider the others as overrides for name, v in overrides.items(): @@ -345,3 +360,11 @@ def get_config_data_macros(self): params, macros = self.get_config_data() self._check_required_parameters(params) return macros + self.parameters_to_macros(params) + + # Returns any features in the configuration data + def get_features(self): + params, _ = self.get_config_data() + self._check_required_parameters(params) + return ((set(Target.get_target(self.target).features) + | self.added_features) - self.removed_features) + diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 0d4468134f2..043acaa2798 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -89,6 +89,9 @@ def __init__(self, base_path=None): self.bin_files = [] self.json_files = [] + # Features + self.features = {} + def __add__(self, resources): if resources is None: return self @@ -126,6 +129,8 @@ def add(self, resources): self.bin_files += resources.bin_files self.json_files += resources.json_files + self.features.update(resources.features) + return self def relative_to(self, base, dot=False): @@ -135,6 +140,10 @@ def relative_to(self, base, dot=False): 'hex_files', 'bin_files', 'json_files']: v = [rel_path(f, base, dot) for f in getattr(self, field)] setattr(self, field, v) + + for f in self.features: + self.features[f] = rel_path(self.features[f], base, dot) + if self.linker_script is not None: self.linker_script = rel_path(self.linker_script, base, dot) @@ -145,6 +154,10 @@ def win_to_unix(self): 'hex_files', 'bin_files', 'json_files']: v = [f.replace('\\', '/') for f in getattr(self, field)] setattr(self, field, v) + + for f in self.features: + self.features[f] = self.features[f].replace('\\', '/') + if self.linker_script is not None: self.linker_script = self.linker_script.replace('\\', '/') @@ -165,6 +178,8 @@ def __str__(self): ('Hex files', self.hex_files), ('Bin files', self.bin_files), + + ('Features', self.features), ): if resources: s.append('%s:\n ' % label + '\n '.join(resources)) @@ -425,11 +440,13 @@ def scan_resources(self, path, exclude_paths=None): if ((d.startswith('.') or d in self.legacy_ignore_dirs) or (d.startswith('TARGET_') and d[7:] not in labels['TARGET']) or - (d.startswith('FEATURE_') and d[8:] not in labels['FEATURE']) or (d.startswith('TOOLCHAIN_') and d[10:] not in labels['TOOLCHAIN']) or (d == 'TESTS')): dirs.remove(d) + if (d.startswith('FEATURE_')): + resources.features[d[8:]] = self.scan_resources(dir_path) + dirs.remove(d) # Remove dirs that already match the ignorepatterns # to avoid travelling into them and to prevent them From d9749b0447fb6ec0c745533f32380e3c6523dd92 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 14 Jun 2016 12:41:29 -0500 Subject: [PATCH 2/8] [build tools] Added list of supported features per @screamerbg --- tools/config.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/config.py b/tools/config.py index e11bb13c476..f0257d73ffb 100644 --- a/tools/config.py +++ b/tools/config.py @@ -147,6 +147,11 @@ class Config: "application": set(["config", "custom_targets", "target_overrides", "macros", "__config_path"]) } + # Allowed features in configurations + __allowed_features = [ + "UVISOR", "BLE", "CLIENT", "IPV4", "IPV6" + ] + # The initialization arguments for Config are: # target: the name of the mbed target used for this configuration instance # top_level_dirs: a list of top level source directories (where mbed_abb_config.json could be found) @@ -365,6 +370,12 @@ def get_config_data_macros(self): def get_features(self): params, _ = self.get_config_data() self._check_required_parameters(params) - return ((set(Target.get_target(self.target).features) + features = ((set(Target.get_target(self.target).features) | self.added_features) - self.removed_features) + + for feature in features: + if feature not in self.__allowed_features: + raise ConfigException("Feature '%s' is not a supported features" % feature) + + return features From 6da324fa3fe114f03120aa9ff5024615256f77d3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 14 Jun 2016 14:17:38 -0500 Subject: [PATCH 3/8] [build tools] Added test coverage for config features feature --- tools/build_api.py | 30 ++++++++++++---- tools/test/config_test/config_test.py | 36 ++++++++++++------- tools/test/config_test/test21/mbed_app.json | 7 ++++ tools/test/config_test/test21/test_data.py | 8 +++++ .../config_test/test22/lib1/mbed_lib.json | 8 +++++ tools/test/config_test/test22/mbed_app.json | 7 ++++ tools/test/config_test/test22/test_data.py | 8 +++++ .../config_test/test23/lib1/mbed_lib.json | 8 +++++ .../config_test/test23/lib2/mbed_lib.json | 8 +++++ tools/test/config_test/test23/mbed_app.json | 7 ++++ tools/test/config_test/test23/test_data.py | 8 +++++ .../test24/FEATURE_IPV4/lib1/mbed_lib.json | 8 +++++ .../test24/FEATURE_IPV6/lib2/mbed_lib.json | 8 +++++ tools/test/config_test/test24/mbed_app.json | 7 ++++ tools/test/config_test/test24/test_data.py | 8 +++++ .../FEATURE_IPV4/lib1/mbed_lib.json | 8 +++++ .../test25/FEATURE_IPV6/lib2/mbed_lib.json | 8 +++++ tools/test/config_test/test25/mbed_app.json | 7 ++++ tools/test/config_test/test25/test_data.py | 8 +++++ 19 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 tools/test/config_test/test21/mbed_app.json create mode 100644 tools/test/config_test/test21/test_data.py create mode 100644 tools/test/config_test/test22/lib1/mbed_lib.json create mode 100644 tools/test/config_test/test22/mbed_app.json create mode 100644 tools/test/config_test/test22/test_data.py create mode 100644 tools/test/config_test/test23/lib1/mbed_lib.json create mode 100644 tools/test/config_test/test23/lib2/mbed_lib.json create mode 100644 tools/test/config_test/test23/mbed_app.json create mode 100644 tools/test/config_test/test23/test_data.py create mode 100644 tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json create mode 100644 tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json create mode 100644 tools/test/config_test/test24/mbed_app.json create mode 100644 tools/test/config_test/test24/test_data.py create mode 100644 tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json create mode 100644 tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json create mode 100644 tools/test/config_test/test25/mbed_app.json create mode 100644 tools/test/config_test/test25/test_data.py diff --git a/tools/build_api.py b/tools/build_api.py index d9c6835b6cf..a5f6dc93b7e 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -105,8 +105,26 @@ def get_config(src_path, target, toolchain_name): for path in src_paths[1:]: resources.add(toolchain.scan_resources(path)) - config.add_config_files(resources.json_files) - return config.get_config_data() + # Update configuration files until added features creates no changes + prev_features = set() + while True: + # Update the configuration with any .json files found while scanning + config.add_config_files(resources.json_files) + + # Add features while we find new ones + features = config.get_features() + if features == prev_features: + break + + for feature in features: + if feature in resources.features: + resources += resources.features[feature] + + prev_features = features + + cfg, macros = config.get_config_data() + features = config.get_features() + return cfg, macros, features def build_project(src_path, build_path, target, toolchain_name, libraries_paths=None, options=None, linker_script=None, @@ -207,9 +225,8 @@ def build_project(src_path, build_path, target, toolchain_name, break for feature in features: - if feature not in resources.features: - raise KeyError("Feature %s is unavailable" % feature) - resources += resources.features[feature] + if feature in resources.features: + resources += resources.features[feature] prev_features = features @@ -373,7 +390,8 @@ def build_library(src_paths, build_path, target, toolchain_name, break for feature in features: - resources += resources.features[feature] + if feature in resources.features: + resources += resources.features[feature] prev_features = features diff --git a/tools/test/config_test/config_test.py b/tools/test/config_test/config_test.py index a520c2006da..a0e5eab8ff9 100644 --- a/tools/test/config_test/config_test.py +++ b/tools/test/config_test/config_test.py @@ -28,7 +28,7 @@ def compare_config(cfg, expected): except KeyError: return "Unexpected key '%s' in configuration data" % k for k in expected: - if k != "desc" and k != "expected_macros" and not k in cfg: + if k not in ["desc", "expected_macros", "expected_features"] + cfg.keys(): return "Expected key '%s' was not found in configuration data" % k return "" @@ -43,7 +43,7 @@ def test_tree(full_name, name): sys.stdout.flush() err_msg = None try: - cfg, macros = get_config(full_name, target, "GCC_ARM") + cfg, macros, features = get_config(full_name, target, "GCC_ARM") except ConfigException as e: err_msg = e.message if err_msg: @@ -63,23 +63,33 @@ def test_tree(full_name, name): failed += 1 else: res = compare_config(cfg, expected) + expected_macros = expected.get("expected_macros", None) + expected_features = expected.get("expected_features", None) + if res: print "FAILED!" sys.stdout.write(" " + res + "\n") failed += 1 - else: - expected_macros = expected.get("expected_macros", None) - if expected_macros is not None: - if sorted(expected_macros) != sorted(macros): - print "FAILED!" - sys.stderr.write(" List of macros doesn't match\n") - sys.stderr.write(" Expected: '%s'\n" % ",".join(sorted(expected_macros))) - sys.stderr.write(" Got: '%s'\n" % ",".join(sorted(expected_macros))) - failed += 1 - else: - print "OK" + elif expected_macros is not None: + if sorted(expected_macros) != sorted(macros): + print "FAILED!" + sys.stderr.write(" List of macros doesn't match\n") + sys.stderr.write(" Expected: '%s'\n" % ",".join(sorted(expected_macros))) + sys.stderr.write(" Got: '%s'\n" % ",".join(sorted(expected_macros))) + failed += 1 else: print "OK" + elif expected_features is not None: + if sorted(expected_features) != sorted(features): + print "FAILED!" + sys.stderr.write(" List of features doesn't match\n") + sys.stderr.write(" Expected: '%s'\n" % ",".join(sorted(expected_features))) + sys.stderr.write(" Got: '%s'\n" % ",".join(sorted(expected_features))) + failed += 1 + else: + print "OK" + else: + print "OK" sys.path.remove(full_name) return failed diff --git a/tools/test/config_test/test21/mbed_app.json b/tools/test/config_test/test21/mbed_app.json new file mode 100644 index 00000000000..cb3c7ed133f --- /dev/null +++ b/tools/test/config_test/test21/mbed_app.json @@ -0,0 +1,7 @@ +{ + "target_overrides": { + "*": { + "features": ["IPV4", "IPV6"] + } + } +} diff --git a/tools/test/config_test/test21/test_data.py b/tools/test/config_test/test21/test_data.py new file mode 100644 index 00000000000..ccacf26faff --- /dev/null +++ b/tools/test/config_test/test21/test_data.py @@ -0,0 +1,8 @@ +# Testing basic features + +expected_results = { + "K64F": { + "desc": "test basic features", + "expected_features": ["IPV4", "IPV6"] + } +} diff --git a/tools/test/config_test/test22/lib1/mbed_lib.json b/tools/test/config_test/test22/lib1/mbed_lib.json new file mode 100644 index 00000000000..b35e4d56c0a --- /dev/null +++ b/tools/test/config_test/test22/lib1/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib1", + "target_overrides": { + "*": { + "features_add": ["IPV4"] + } + } +} diff --git a/tools/test/config_test/test22/mbed_app.json b/tools/test/config_test/test22/mbed_app.json new file mode 100644 index 00000000000..d5126abf78c --- /dev/null +++ b/tools/test/config_test/test22/mbed_app.json @@ -0,0 +1,7 @@ +{ + "target_overrides": { + "*": { + "features_add": ["IPV6"] + } + } +} diff --git a/tools/test/config_test/test22/test_data.py b/tools/test/config_test/test22/test_data.py new file mode 100644 index 00000000000..6fcd5089618 --- /dev/null +++ b/tools/test/config_test/test22/test_data.py @@ -0,0 +1,8 @@ +# Testing when adding two features + +expected_results = { + "K64F": { + "desc": "test composing features", + "expected_features": ["IPV4", "IPV6"] + } +} diff --git a/tools/test/config_test/test23/lib1/mbed_lib.json b/tools/test/config_test/test23/lib1/mbed_lib.json new file mode 100644 index 00000000000..b35e4d56c0a --- /dev/null +++ b/tools/test/config_test/test23/lib1/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib1", + "target_overrides": { + "*": { + "features_add": ["IPV4"] + } + } +} diff --git a/tools/test/config_test/test23/lib2/mbed_lib.json b/tools/test/config_test/test23/lib2/mbed_lib.json new file mode 100644 index 00000000000..6215ff57d7e --- /dev/null +++ b/tools/test/config_test/test23/lib2/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib2", + "target_overrides": { + "*": { + "features_remove": ["IPV4"] + } + } +} diff --git a/tools/test/config_test/test23/mbed_app.json b/tools/test/config_test/test23/mbed_app.json new file mode 100644 index 00000000000..d5126abf78c --- /dev/null +++ b/tools/test/config_test/test23/mbed_app.json @@ -0,0 +1,7 @@ +{ + "target_overrides": { + "*": { + "features_add": ["IPV6"] + } + } +} diff --git a/tools/test/config_test/test23/test_data.py b/tools/test/config_test/test23/test_data.py new file mode 100644 index 00000000000..0d1105a08a7 --- /dev/null +++ b/tools/test/config_test/test23/test_data.py @@ -0,0 +1,8 @@ +# Testing when two features collide + +expected_results = { + "K64F": { + "desc": "test feature collisions", + "exception_msg": "Configuration conflict. Feature IPV4 both added and removed." + } +} diff --git a/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json b/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json new file mode 100644 index 00000000000..6b23b3450a4 --- /dev/null +++ b/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib1", + "target_overrides": { + "*": { + "features_add": ["IPV6"] + } + } +} diff --git a/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json b/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json new file mode 100644 index 00000000000..6acafdcdc88 --- /dev/null +++ b/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib2", + "target_overrides": { + "*": { + "features_add": ["UVISOR"] + } + } +} diff --git a/tools/test/config_test/test24/mbed_app.json b/tools/test/config_test/test24/mbed_app.json new file mode 100644 index 00000000000..58a9d08845a --- /dev/null +++ b/tools/test/config_test/test24/mbed_app.json @@ -0,0 +1,7 @@ +{ + "target_overrides": { + "*": { + "features_add": ["IPV4"] + } + } +} diff --git a/tools/test/config_test/test24/test_data.py b/tools/test/config_test/test24/test_data.py new file mode 100644 index 00000000000..97ebe5bc7c8 --- /dev/null +++ b/tools/test/config_test/test24/test_data.py @@ -0,0 +1,8 @@ +# Testing if features can enable other features + +expected_results = { + "K64F": { + "desc": "test recursive features", + "expected_features": ["IPV4", "IPV6", "UVISOR"] + } +} diff --git a/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json b/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json new file mode 100644 index 00000000000..6b23b3450a4 --- /dev/null +++ b/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib1", + "target_overrides": { + "*": { + "features_add": ["IPV6"] + } + } +} diff --git a/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json b/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json new file mode 100644 index 00000000000..6acafdcdc88 --- /dev/null +++ b/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib2", + "target_overrides": { + "*": { + "features_add": ["UVISOR"] + } + } +} diff --git a/tools/test/config_test/test25/mbed_app.json b/tools/test/config_test/test25/mbed_app.json new file mode 100644 index 00000000000..cb3c7ed133f --- /dev/null +++ b/tools/test/config_test/test25/mbed_app.json @@ -0,0 +1,7 @@ +{ + "target_overrides": { + "*": { + "features": ["IPV4", "IPV6"] + } + } +} diff --git a/tools/test/config_test/test25/test_data.py b/tools/test/config_test/test25/test_data.py new file mode 100644 index 00000000000..1ebfc24924d --- /dev/null +++ b/tools/test/config_test/test25/test_data.py @@ -0,0 +1,8 @@ +# Testing if feature collisions are detected accross recursive features + +expected_results = { + "K64F": { + "desc": "test recursive feature collisions", + "exception_msg": "Configuration conflict. Feature UVISOR both added and removed." + } +} From e70efe3a415f20879c1e449887a6ac15c3291847 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 14 Jun 2016 19:42:31 -0500 Subject: [PATCH 4/8] [build tools] Made config settings compatible with features system per @screamerbg --- tools/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/config.py b/tools/config.py index f0257d73ffb..dabf2a44de5 100644 --- a/tools/config.py +++ b/tools/config.py @@ -266,10 +266,8 @@ def _process_config_and_overrides(self, data, params, unit_name, unit_kind): for name, v in overrides.items(): # Get the full name of the parameter full_name = ConfigParameter.get_full_name(name, unit_name, unit_kind, label) - # If an attempt is made to override a parameter that isn't defined, raise an error - if not full_name in params: - raise ConfigException("Attempt to override undefined parameter '%s' in '%s'" % (full_name, ConfigParameter.get_display_name(unit_name, unit_kind, label))) - params[full_name].set_value(v, unit_name, unit_kind, label) + if full_name in params: + params[full_name].set_value(v, unit_name, unit_kind, label) return params # Read and interpret configuration data defined by targets From f45dc15aa848de1603facdd0447de25fd1737115 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 14 Jun 2016 19:51:35 -0500 Subject: [PATCH 5/8] [build tools] Added test for configurations set from same level as feature --- .../config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json | 8 ++++++++ .../config_test/test26/FEATURE_IPV6/lib2/mbed_lib.json | 8 ++++++++ tools/test/config_test/test26/mbed_app.json | 8 ++++++++ tools/test/config_test/test26/test_data.py | 8 ++++++++ 4 files changed, 32 insertions(+) create mode 100644 tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json create mode 100644 tools/test/config_test/test26/FEATURE_IPV6/lib2/mbed_lib.json create mode 100644 tools/test/config_test/test26/mbed_app.json create mode 100644 tools/test/config_test/test26/test_data.py diff --git a/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json b/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json new file mode 100644 index 00000000000..6b23b3450a4 --- /dev/null +++ b/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib1", + "target_overrides": { + "*": { + "features_add": ["IPV6"] + } + } +} diff --git a/tools/test/config_test/test26/FEATURE_IPV6/lib2/mbed_lib.json b/tools/test/config_test/test26/FEATURE_IPV6/lib2/mbed_lib.json new file mode 100644 index 00000000000..5fd4a81cce4 --- /dev/null +++ b/tools/test/config_test/test26/FEATURE_IPV6/lib2/mbed_lib.json @@ -0,0 +1,8 @@ +{ + "name": "lib2", + "config": { + "test": { + "value": "BAD" + } + } +} diff --git a/tools/test/config_test/test26/mbed_app.json b/tools/test/config_test/test26/mbed_app.json new file mode 100644 index 00000000000..92331b1f62e --- /dev/null +++ b/tools/test/config_test/test26/mbed_app.json @@ -0,0 +1,8 @@ +{ + "target_overrides": { + "*": { + "features_add": ["IPV4"], + "lib2.test": "GOOD" + } + } +} diff --git a/tools/test/config_test/test26/test_data.py b/tools/test/config_test/test26/test_data.py new file mode 100644 index 00000000000..08300ab45ae --- /dev/null +++ b/tools/test/config_test/test26/test_data.py @@ -0,0 +1,8 @@ +# Testing if config settings work in recursive features + +expected_results = { + "K64F": { + "desc": "test recursive feature configurations", + "lib2.test": "GOOD" + } +} From 41daccf8d90a12e2c6e33348787d9bf06c1107b4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 15 Jun 2016 07:10:32 -0500 Subject: [PATCH 6/8] [build tools] Fixed masked out configuration error --- tools/build_api.py | 3 +++ tools/config.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/tools/build_api.py b/tools/build_api.py index a5f6dc93b7e..478353e3988 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -121,6 +121,7 @@ def get_config(src_path, target, toolchain_name): resources += resources.features[feature] prev_features = features + config.validate_config() cfg, macros = config.get_config_data() features = config.get_features() @@ -229,6 +230,7 @@ def build_project(src_path, build_path, target, toolchain_name, resources += resources.features[feature] prev_features = features + config.validate_config() # And add the configuration macros to the toolchain toolchain.add_macros(config.get_config_data_macros()) @@ -394,6 +396,7 @@ def build_library(src_paths, build_path, target, toolchain_name, resources += resources.features[feature] prev_features = features + config.validate_config() # And add the configuration macros to the toolchain toolchain.add_macros(config.get_config_data_macros()) diff --git a/tools/config.py b/tools/config.py index dabf2a44de5..91a35542e63 100644 --- a/tools/config.py +++ b/tools/config.py @@ -40,6 +40,7 @@ def __init__(self, name, data, unit_name, unit_kind): self.value = data.get("value", None) self.required = data.get("required", False) self.macro_name = data.get("macro_name", "MBED_CONF_%s" % self.sanitize(self.name.upper())) + self.config_errors = [] # Return the full (prefixed) name of a parameter. # If the parameter already has a prefix, check if it is valid @@ -242,6 +243,7 @@ def add_features(self, features): # unit_name: the unit (library/application) that defines this parameter # unit_kind: the kind of the unit ("library" or "application") def _process_config_and_overrides(self, data, params, unit_name, unit_kind): + self.config_errors = [] self._process_config_parameters(data.get("config", {}), params, unit_name, unit_kind) for label, overrides in data.get("target_overrides", {}).items(): # If the label is defined by the target or it has the special value "*", process the overrides @@ -268,6 +270,9 @@ def _process_config_and_overrides(self, data, params, unit_name, unit_kind): full_name = ConfigParameter.get_full_name(name, unit_name, unit_kind, label) if full_name in params: params[full_name].set_value(v, unit_name, unit_kind, label) + else: + self.config_errors.append(ConfigException("Attempt to override undefined parameter '%s' in '%s'" + % (full_name, ConfigParameter.get_display_name(unit_name, unit_kind, label)))) return params # Read and interpret configuration data defined by targets @@ -376,4 +381,10 @@ def get_features(self): raise ConfigException("Feature '%s' is not a supported features" % feature) return features + + # Validate configuration settings. This either returns True or raises an exception + def validate_config(self): + if self.config_errors: + raise self.config_errors[0] + return True From 99ed4e03f1328920475e228bc8825d45fd66435c Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Wed, 15 Jun 2016 17:18:40 +0100 Subject: [PATCH 7/8] Restore original compile sources routine --- tools/build_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index 478353e3988..e3247b37fc2 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -236,8 +236,10 @@ def build_project(src_path, build_path, target, toolchain_name, toolchain.add_macros(config.get_config_data_macros()) # Compile Sources - objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs) - resources.objects.extend(objects) + for path in src_paths: + src = toolchain.scan_resources(path) + objects = toolchain.compile_sources(src, build_path, resources.inc_dirs) + resources.objects.extend(objects) # Link Program res, _ = toolchain.link_program(resources, build_path, name) @@ -402,8 +404,10 @@ def build_library(src_paths, build_path, target, toolchain_name, toolchain.add_macros(config.get_config_data_macros()) # Compile Sources - objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs) - resources.objects.extend(objects) + for path in src_paths: + src = toolchain.scan_resources(path) + objects = toolchain.compile_sources(src, abspath(tmp_path), resources.inc_dirs) + resources.objects.extend(objects) if archive: toolchain.build_library(objects, build_path, name) From 0d9d4639cd8cfeacad2b2c97b3b34d4d1dffd350 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 15 Jun 2016 10:25:31 -0500 Subject: [PATCH 8/8] [build tools] Added proper namespace for features per @bogdanm --- tools/config.py | 20 +++++++++---------- tools/test/config_test/test21/mbed_app.json | 2 +- .../config_test/test22/lib1/mbed_lib.json | 2 +- tools/test/config_test/test22/mbed_app.json | 2 +- .../config_test/test23/lib1/mbed_lib.json | 2 +- .../config_test/test23/lib2/mbed_lib.json | 2 +- tools/test/config_test/test23/mbed_app.json | 2 +- .../test24/FEATURE_IPV4/lib1/mbed_lib.json | 2 +- .../test24/FEATURE_IPV6/lib2/mbed_lib.json | 2 +- tools/test/config_test/test24/mbed_app.json | 2 +- .../FEATURE_IPV4/lib1/mbed_lib.json | 2 +- .../test25/FEATURE_IPV6/lib2/mbed_lib.json | 2 +- tools/test/config_test/test25/mbed_app.json | 2 +- .../test26/FEATURE_IPV4/lib1/mbed_lib.json | 2 +- tools/test/config_test/test26/mbed_app.json | 2 +- 15 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tools/config.py b/tools/config.py index 91a35542e63..c95b0206c75 100644 --- a/tools/config.py +++ b/tools/config.py @@ -249,20 +249,20 @@ def _process_config_and_overrides(self, data, params, unit_name, unit_kind): # If the label is defined by the target or it has the special value "*", process the overrides if (label == '*') or (label in self.target_labels): # Parse out features - if 'features' in overrides: - features = overrides['features'] - self.remove_features(list(set(self.added_features) - set(features))) + if 'target.features' in overrides: + features = overrides['target.features'] + self.remove_features(self.added_features - set(features)) self.add_features(features) self.removed_unecessary_features = True - del overrides['features'] + del overrides['target.features'] - if 'features_add' in overrides: - self.add_features(overrides['features_add']) - del overrides['features_add'] + if 'target.features_add' in overrides: + self.add_features(overrides['target.features_add']) + del overrides['target.features_add'] - if 'features_remove' in overrides: - self.remove_features(overrides['features_remove']) - del overrides['features_remove'] + if 'target.features_remove' in overrides: + self.remove_features(overrides['target.features_remove']) + del overrides['target.features_remove'] # Consider the others as overrides for name, v in overrides.items(): diff --git a/tools/test/config_test/test21/mbed_app.json b/tools/test/config_test/test21/mbed_app.json index cb3c7ed133f..d1a874a23d8 100644 --- a/tools/test/config_test/test21/mbed_app.json +++ b/tools/test/config_test/test21/mbed_app.json @@ -1,7 +1,7 @@ { "target_overrides": { "*": { - "features": ["IPV4", "IPV6"] + "target.features": ["IPV4", "IPV6"] } } } diff --git a/tools/test/config_test/test22/lib1/mbed_lib.json b/tools/test/config_test/test22/lib1/mbed_lib.json index b35e4d56c0a..07e6e626b59 100644 --- a/tools/test/config_test/test22/lib1/mbed_lib.json +++ b/tools/test/config_test/test22/lib1/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib1", "target_overrides": { "*": { - "features_add": ["IPV4"] + "target.features_add": ["IPV4"] } } } diff --git a/tools/test/config_test/test22/mbed_app.json b/tools/test/config_test/test22/mbed_app.json index d5126abf78c..a070a2d5565 100644 --- a/tools/test/config_test/test22/mbed_app.json +++ b/tools/test/config_test/test22/mbed_app.json @@ -1,7 +1,7 @@ { "target_overrides": { "*": { - "features_add": ["IPV6"] + "target.features_add": ["IPV6"] } } } diff --git a/tools/test/config_test/test23/lib1/mbed_lib.json b/tools/test/config_test/test23/lib1/mbed_lib.json index b35e4d56c0a..07e6e626b59 100644 --- a/tools/test/config_test/test23/lib1/mbed_lib.json +++ b/tools/test/config_test/test23/lib1/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib1", "target_overrides": { "*": { - "features_add": ["IPV4"] + "target.features_add": ["IPV4"] } } } diff --git a/tools/test/config_test/test23/lib2/mbed_lib.json b/tools/test/config_test/test23/lib2/mbed_lib.json index 6215ff57d7e..c635513a56f 100644 --- a/tools/test/config_test/test23/lib2/mbed_lib.json +++ b/tools/test/config_test/test23/lib2/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib2", "target_overrides": { "*": { - "features_remove": ["IPV4"] + "target.features_remove": ["IPV4"] } } } diff --git a/tools/test/config_test/test23/mbed_app.json b/tools/test/config_test/test23/mbed_app.json index d5126abf78c..a070a2d5565 100644 --- a/tools/test/config_test/test23/mbed_app.json +++ b/tools/test/config_test/test23/mbed_app.json @@ -1,7 +1,7 @@ { "target_overrides": { "*": { - "features_add": ["IPV6"] + "target.features_add": ["IPV6"] } } } diff --git a/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json b/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json index 6b23b3450a4..539d8ccc221 100644 --- a/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json +++ b/tools/test/config_test/test24/FEATURE_IPV4/lib1/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib1", "target_overrides": { "*": { - "features_add": ["IPV6"] + "target.features_add": ["IPV6"] } } } diff --git a/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json b/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json index 6acafdcdc88..73b57765aa1 100644 --- a/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json +++ b/tools/test/config_test/test24/FEATURE_IPV6/lib2/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib2", "target_overrides": { "*": { - "features_add": ["UVISOR"] + "target.features_add": ["UVISOR"] } } } diff --git a/tools/test/config_test/test24/mbed_app.json b/tools/test/config_test/test24/mbed_app.json index 58a9d08845a..56af1f52840 100644 --- a/tools/test/config_test/test24/mbed_app.json +++ b/tools/test/config_test/test24/mbed_app.json @@ -1,7 +1,7 @@ { "target_overrides": { "*": { - "features_add": ["IPV4"] + "target.features_add": ["IPV4"] } } } diff --git a/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json b/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json index 6b23b3450a4..539d8ccc221 100644 --- a/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json +++ b/tools/test/config_test/test25/FEATURE_IPV6/FEATURE_IPV4/lib1/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib1", "target_overrides": { "*": { - "features_add": ["IPV6"] + "target.features_add": ["IPV6"] } } } diff --git a/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json b/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json index 6acafdcdc88..73b57765aa1 100644 --- a/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json +++ b/tools/test/config_test/test25/FEATURE_IPV6/lib2/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib2", "target_overrides": { "*": { - "features_add": ["UVISOR"] + "target.features_add": ["UVISOR"] } } } diff --git a/tools/test/config_test/test25/mbed_app.json b/tools/test/config_test/test25/mbed_app.json index cb3c7ed133f..d1a874a23d8 100644 --- a/tools/test/config_test/test25/mbed_app.json +++ b/tools/test/config_test/test25/mbed_app.json @@ -1,7 +1,7 @@ { "target_overrides": { "*": { - "features": ["IPV4", "IPV6"] + "target.features": ["IPV4", "IPV6"] } } } diff --git a/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json b/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json index 6b23b3450a4..539d8ccc221 100644 --- a/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json +++ b/tools/test/config_test/test26/FEATURE_IPV4/lib1/mbed_lib.json @@ -2,7 +2,7 @@ "name": "lib1", "target_overrides": { "*": { - "features_add": ["IPV6"] + "target.features_add": ["IPV6"] } } } diff --git a/tools/test/config_test/test26/mbed_app.json b/tools/test/config_test/test26/mbed_app.json index 92331b1f62e..1d926c1abec 100644 --- a/tools/test/config_test/test26/mbed_app.json +++ b/tools/test/config_test/test26/mbed_app.json @@ -1,7 +1,7 @@ { "target_overrides": { "*": { - "features_add": ["IPV4"], + "target.features_add": ["IPV4"], "lib2.test": "GOOD" } }