diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index 8f6f8eb170b3b..fb9e6768651c6 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -12,6 +12,7 @@ import json import logging import sys +import glob from pathlib import Path from git import Repo from west.manifest import Manifest @@ -19,10 +20,10 @@ if "ZEPHYR_BASE" not in os.environ: exit("$ZEPHYR_BASE environment variable undefined.") -repository_path = Path(os.environ['ZEPHYR_BASE']) +zephyr_base = Path(os.environ['ZEPHYR_BASE']) logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) -sys.path.append(os.path.join(repository_path, 'scripts')) +sys.path.append(os.path.join(zephyr_base, 'scripts')) import list_boards def _get_match_fn(globs, regexes): @@ -86,15 +87,23 @@ def __repr__(self): return "".format(self.name) class Filters: - def __init__(self, modified_files, pull_request=False, platforms=[]): + def __init__(self, modified_files, pull_request=False, platforms=[], no_path_name = False, ignore_path=None, alt_tags=None, testsuite_root=None): self.modified_files = modified_files + self.resolved_files = [] + self.testsuite_root = testsuite_root self.twister_options = [] self.full_twister = False self.all_tests = [] self.tag_options = [] self.pull_request = pull_request self.platforms = platforms - self.default_run = False + self.no_path_name = no_path_name + self.tag_cfg_file = os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml') + if alt_tags: + self.tag_cfg_file = alt_tags + self.ignore_path = f"{zephyr_base}/scripts/ci/twister_ignore.txt" + if ignore_path: + self.ignore_path = ignore_path def process(self): self.find_modules() @@ -103,15 +112,16 @@ def process(self): if not self.platforms: self.find_archs() self.find_boards() + self.find_excludes() - if self.default_run: - self.find_excludes(skip=["tests/*", "boards/*/*/*"]) - else: - self.find_excludes() - - def get_plan(self, options, integration=False): + def get_plan(self, options, integration=False, use_testsuite_root=True): fname = "_test_plan_partial.json" - cmd = ["scripts/twister", "-c"] + options + ["--save-tests", fname ] + cmd = [f"{zephyr_base}/scripts/twister", "-c"] + options + ["--save-tests", fname ] + if self.testsuite_root and use_testsuite_root: + for root in self.testsuite_root: + cmd+=["-T", root] + if self.no_path_name: + cmd += ["--no-path-name"] if integration: cmd.append("--integration") @@ -128,7 +138,7 @@ def find_modules(self): if 'west.yml' in self.modified_files: print(f"Manifest file 'west.yml' changed") print("=========") - old_manifest_content = repo.git.show(f"{args.commits[:-2]}:west.yml") + old_manifest_content = repo_to_scan.git.show(f"{args.commits[:-2]}:west.yml") with open("west_old.yml", "w") as manifest: manifest.write(old_manifest_content) old_manifest = Manifest.from_file("west_old.yml") @@ -179,6 +189,8 @@ def find_archs(self): archs.add('riscv64') else: archs.add(p.group(1)) + # Modified file is treated as resolved, since a matching scope was found + self.resolved_files.append(f) _options = [] for arch in archs: @@ -197,6 +209,7 @@ def find_archs(self): def find_boards(self): boards = set() all_boards = set() + resolved = [] for f in self.modified_files: if f.endswith(".rst") or f.endswith(".png") or f.endswith(".jpg"): @@ -204,9 +217,14 @@ def find_boards(self): p = re.match(r"^boards\/[^/]+\/([^/]+)\/", f) if p and p.groups(): boards.add(p.group(1)) + resolved.append(f) + + roots = [zephyr_base] + if repository_path != zephyr_base: + roots.append(repository_path) - # Limit search to $ZEPHYR_BASE since this is where the changed files are - lb_args = argparse.Namespace(**{ 'arch_roots': [repository_path], 'board_roots': [repository_path] }) + # Look for boards in monitored repositories + lb_args = argparse.Namespace(**{ 'arch_roots': roots, 'board_roots': roots}) known_boards = list_boards.find_boards(lb_args) for b in boards: name_re = re.compile(b) @@ -214,10 +232,16 @@ def find_boards(self): if name_re.search(kb.name): all_boards.add(kb.name) + # If modified file is catched by "find_boards" workflow (change in "boards" dir AND board recognized) + # it means a proper testing scope for this file was found and this file can be removed + # from further consideration + for board in all_boards: + self.resolved_files.extend(list(filter(lambda f: board in f, resolved))) + _options = [] if len(all_boards) > 20: logging.warning(f"{len(boards)} boards changed, this looks like a global change, skipping test handling, revert to default.") - self.default_run = True + self.full_twister = True return for board in all_boards: @@ -233,11 +257,25 @@ def find_tests(self): if f.endswith(".rst"): continue d = os.path.dirname(f) - while d: + scope_found = False + while not scope_found and d: + head, tail = os.path.split(d) if os.path.exists(os.path.join(d, "testcase.yaml")) or \ os.path.exists(os.path.join(d, "sample.yaml")): tests.add(d) - break + # Modified file is treated as resolved, since a matching scope was found + self.resolved_files.append(f) + scope_found = True + elif tail == "common": + # Look for yamls in directories collocated with common + + yamls_found = [yaml for yaml in glob.iglob(head + '/**/testcase.yaml', recursive=True)] + yamls_found.extend([yaml for yaml in glob.iglob(head + '/**/sample.yaml', recursive=True)]) + if yamls_found: + for yaml in yamls_found: + tests.add(os.path.dirname(yaml)) + self.resolved_files.append(f) + scope_found = True else: d = os.path.dirname(d) @@ -247,7 +285,7 @@ def find_tests(self): if len(tests) > 20: logging.warning(f"{len(tests)} tests changed, this looks like a global change, skipping test handling, revert to default") - self.default_run = True + self.full_twister = True return if _options: @@ -257,12 +295,11 @@ def find_tests(self): _options.extend(["-p", platform]) else: _options.append("--all") - self.get_plan(_options) + self.get_plan(_options, use_testsuite_root=False) def find_tags(self): - tag_cfg_file = os.path.join(repository_path, 'scripts', 'ci', 'tags.yaml') - with open(tag_cfg_file, 'r') as ymlfile: + with open(self.tag_cfg_file, 'r') as ymlfile: tags_config = yaml.safe_load(ymlfile) tags = {} @@ -299,26 +336,27 @@ def find_tags(self): logging.info(f'Potential tag based filters: {exclude_tags}') def find_excludes(self, skip=[]): - with open("scripts/ci/twister_ignore.txt", "r") as twister_ignore: + with open(self.ignore_path, "r") as twister_ignore: ignores = twister_ignore.read().splitlines() ignores = filter(lambda x: not x.startswith("#"), ignores) found = set() - files = list(filter(lambda x: x, self.modified_files)) + files_not_resolved = list(filter(lambda x: x not in self.resolved_files, self.modified_files)) for pattern in ignores: - if pattern in skip: - continue if pattern: - found.update(fnmatch.filter(files, pattern)) + found.update(fnmatch.filter(files_not_resolved, pattern)) logging.debug(found) - logging.debug(files) + logging.debug(files_not_resolved) - if sorted(files) != sorted(found): + # Full twister run can be ordered by detecting great number of tests/boards changed + # or if not all modified files were resolved (corresponding scope found) + self.full_twister = self.full_twister or sorted(files_not_resolved) != sorted(found) + + if self.full_twister: _options = [] logging.info(f'Need to run full or partial twister...') - self.full_twister = True if self.platforms: for platform in self.platforms: _options.extend(["-p", platform]) @@ -349,6 +387,20 @@ def parse_args(): help="Number of tests per builder") parser.add_argument('-n', '--default-matrix', default=10, type=int, help="Number of tests per builder") + parser.add_argument('-r', '--repo-to-scan', default=None, + help="Repo to scan") + parser.add_argument('--no-path-name', action="store_true", + help="Don't put paths into test suites' names ") + parser.add_argument('--ignore-path', default=None, + help="Path to a text file with patterns of files to be matched against changed files") + parser.add_argument('--alt-tags', default=None, + help="Path to a file describing relations between directories and tags") + parser.add_argument( + "-T", "--testsuite-root", action="append", default=[], + help="Base directory to recursively search for test cases. All " + "testcase.yaml files under here will be processed. May be " + "called multiple times. Defaults to the 'samples/' and " + "'tests/' directories at the base of the Zephyr tree.") return parser.parse_args() @@ -358,9 +410,12 @@ def parse_args(): args = parse_args() files = [] errors = 0 + repository_path = zephyr_base + if args.repo_to_scan: + repository_path = Path(args.repo_to_scan) if args.commits: - repo = Repo(repository_path) - commit = repo.git.diff("--name-only", args.commits) + repo_to_scan = Repo(repository_path) + commit = repo_to_scan.git.diff("--name-only", args.commits) files = commit.split("\n") elif args.modified_files: with open(args.modified_files, "r") as fp: @@ -371,8 +426,7 @@ def parse_args(): print("\n".join(files)) print("=========") - - f = Filters(files, args.pull_request, args.platform) + f = Filters(files, args.pull_request, args.platform, args.no_path_name, args.ignore_path, args.alt_tags, args.testsuite_root) f.process() # remove dupes and filtered cases diff --git a/scripts/ci/twister_ignore.txt b/scripts/ci/twister_ignore.txt index 59f735495ca02..4a0589f291cca 100644 --- a/scripts/ci/twister_ignore.txt +++ b/scripts/ci/twister_ignore.txt @@ -17,25 +17,6 @@ CODEOWNERS MAINTAINERS.yml LICENSE Makefile -tests/* -samples/* -boards/*/*/* -arch/xtensa/* -arch/x86/* -arch/posix/* -arch/arc/* -arch/sparc/* -arch/arm/* -arch/nios2/* -arch/riscv/* -include/arch/xtensa/* -include/arch/x86/* -include/arch/posix/* -include/arch/arc/* -include/arch/sparc/* -include/arch/arm/* -include/arch/nios2/* -include/arch/riscv/* doc/* # GH action have no impact on code .github/* diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 160233a66fe8b..73777c0a1b5a3 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -437,6 +437,10 @@ def add_parse_arguments(parser = None): help="Re-use the outdir before building. Will result in " "faster compilation since builds will be incremental.") + parser.add_argument( + "--no-path-name", action="store_true", + help="Don't put paths into test suites' names ") + # To be removed in favor of --detailed-skipped-report parser.add_argument( "--no-skipped-report", action="store_true", diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 85340885dbb6d..8bd8db84b17d4 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -522,7 +522,7 @@ def add_testsuites(self, testsuite_filter=[]): for name in parsed_data.scenarios.keys(): suite_dict = parsed_data.get_scenario(name) - suite = TestSuite(root, suite_path, name, data=suite_dict) + suite = TestSuite(root, suite_path, name, data=suite_dict, no_path_name=self.options.no_path_name) suite.add_subcases(suite_dict, subcases, ztest_suite_names) if testsuite_filter: if suite.name and suite.name in testsuite_filter: diff --git a/scripts/pylib/twister/twisterlib/testsuite.py b/scripts/pylib/twister/twisterlib/testsuite.py index 3f4a71e4a8c5f..11e0e5a7ba8ac 100644 --- a/scripts/pylib/twister/twisterlib/testsuite.py +++ b/scripts/pylib/twister/twisterlib/testsuite.py @@ -348,7 +348,7 @@ class TestSuite(DisablePyTestCollectionMixin): """Class representing a test application """ - def __init__(self, suite_root, suite_path, name, data=None): + def __init__(self, suite_root, suite_path, name, data=None, no_path_name=False): """TestSuite constructor. This gets called by TestPlan as it finds and reads test yaml files. @@ -369,7 +369,9 @@ def __init__(self, suite_root, suite_path, name, data=None): """ workdir = os.path.relpath(suite_path, suite_root) - self.name = self.get_unique(suite_root, workdir, name) + + assert self.check_suite_name(name, suite_root, workdir) + self.name = name if no_path_name else self.get_unique(suite_root, workdir, name) self.id = name self.source_dir = suite_path @@ -425,10 +427,14 @@ def get_unique(testsuite_root, workdir, name): # workdir can be "." unique = os.path.normpath(os.path.join(relative_ts_root, workdir, name)) + return unique + + @staticmethod + def check_suite_name(name, testsuite_root, workdir): check = name.split(".") if len(check) < 2: raise TwisterException(f"""bad test name '{name}' in {testsuite_root}/{workdir}. \ Tests should reference the category and subsystem with a dot as a separator. """ ) - return unique + return True diff --git a/scripts/tests/twister/test_testsuite.py b/scripts/tests/twister/test_testsuite.py index c954208d75a7b..45495cc2aa190 100644 --- a/scripts/tests/twister/test_testsuite.py +++ b/scripts/tests/twister/test_testsuite.py @@ -110,3 +110,42 @@ def test_get_unique(testsuite_root, suite_path, name, expected): '''Test to check if the unique name is given for each testsuite root and workdir''' suite = TestSuite(testsuite_root, suite_path, name) assert suite.name == expected + +TESTDATA_2 = [ + ( + ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites', + ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/tests/test_a', + 'test_a.check_1', + 'test_a.check_1' + ), + ( + ZEPHYR_BASE, + ZEPHYR_BASE, + 'test_a.check_1', + 'test_a.check_1' + ), + ( + ZEPHYR_BASE, + ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/test_b', + 'test_b.check_1', + 'test_b.check_1' + ), + ( + os.path.join(ZEPHYR_BASE, 'scripts/tests'), + os.path.join(ZEPHYR_BASE, 'scripts/tests'), + 'test_b.check_1', + 'test_b.check_1' + ), + ( + ZEPHYR_BASE, + ZEPHYR_BASE, + 'test_a.check_1.check_2', + 'test_a.check_1.check_2' + ), +] +@pytest.mark.parametrize("testsuite_root, suite_path, name, expected", TESTDATA_2) +def test_get_no_path_name(testsuite_root, suite_path, name, expected): + '''Test to check if the name without path is given for each testsuite''' + suite = TestSuite(testsuite_root, suite_path, name, no_path_name=True) + print(suite.name) + assert suite.name == expected