Test plan up

scripts/ci/test_plan.py

@@ -12,17 +12,18 @@
 import json
 import logging
 import sys
+import glob
 from pathlib import Path
 from git import Repo
 from west.manifest import Manifest
 
 if "ZEPHYR_BASE" not in os.environ:
     exit("$ZEPHYR_BASE environment variable undefined.")
 
-repository_path = Path(os.environ['ZEPHYR_BASE'])
+zephyr_base = Path(os.environ['ZEPHYR_BASE'])
 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
 
-sys.path.append(os.path.join(repository_path, 'scripts'))
+sys.path.append(os.path.join(zephyr_base, 'scripts'))
 import list_boards
 
 def _get_match_fn(globs, regexes):
@@ -86,15 +87,23 @@ def __repr__(self):
         return "<Tag {}>".format(self.name)
 
 class Filters:
-    def __init__(self, modified_files, pull_request=False, platforms=[]):
+    def __init__(self, modified_files, pull_request=False, platforms=[], no_path_name = False, ignore_path=None, alt_tags=None, testsuite_root=None):
         self.modified_files = modified_files
+        self.resolved_files = []
+        self.testsuite_root = testsuite_root
         self.twister_options = []
         self.full_twister = False
         self.all_tests = []
         self.tag_options = []
         self.pull_request = pull_request
         self.platforms = platforms
-        self.default_run = False
+        self.no_path_name = no_path_name
+        self.tag_cfg_file = os.path.join(zephyr_base, 'scripts', 'ci', 'tags.yaml')
+        if alt_tags:
+            self.tag_cfg_file = alt_tags
+        self.ignore_path = f"{zephyr_base}/scripts/ci/twister_ignore.txt"
+        if ignore_path:
+            self.ignore_path = ignore_path
 
     def process(self):
         self.find_modules()
@@ -103,15 +112,16 @@ def process(self):
         if not self.platforms:
             self.find_archs()
             self.find_boards()
+        self.find_excludes()
 
-        if self.default_run:
-            self.find_excludes(skip=["tests/*", "boards/*/*/*"])
-        else:
-            self.find_excludes()
-
-    def get_plan(self, options, integration=False):
+    def get_plan(self, options, integration=False, use_testsuite_root=True):
         fname = "_test_plan_partial.json"
-        cmd = ["scripts/twister", "-c"] + options + ["--save-tests", fname ]
+        cmd = [f"{zephyr_base}/scripts/twister", "-c"] + options + ["--save-tests", fname ]
+        if self.testsuite_root and use_testsuite_root:
+            for root in self.testsuite_root:
+                cmd+=["-T", root]
+        if self.no_path_name:
+            cmd += ["--no-path-name"]
         if integration:
             cmd.append("--integration")
 
@@ -128,7 +138,7 @@ def find_modules(self):
         if 'west.yml' in self.modified_files:
             print(f"Manifest file 'west.yml' changed")
             print("=========")
-            old_manifest_content = repo.git.show(f"{args.commits[:-2]}:west.yml")
+            old_manifest_content = repo_to_scan.git.show(f"{args.commits[:-2]}:west.yml")
             with open("west_old.yml", "w") as manifest:
                 manifest.write(old_manifest_content)
             old_manifest = Manifest.from_file("west_old.yml")
@@ -179,6 +189,8 @@ def find_archs(self):
                         archs.add('riscv64')
                     else:
                         archs.add(p.group(1))
+                    # Modified file is treated as resolved, since a matching scope was found
+                    self.resolved_files.append(f)
 
         _options = []
         for arch in archs:
@@ -197,27 +209,39 @@ def find_archs(self):
     def find_boards(self):
         boards = set()
         all_boards = set()
+        resolved = []
 
         for f in self.modified_files:
             if f.endswith(".rst") or f.endswith(".png") or f.endswith(".jpg"):
                 continue
             p = re.match(r"^boards\/[^/]+\/([^/]+)\/", f)
             if p and p.groups():
                 boards.add(p.group(1))
+                resolved.append(f)
+
+        roots = [zephyr_base]
+        if repository_path != zephyr_base:
+            roots.append(repository_path)
 
-        # Limit search to $ZEPHYR_BASE since this is where the changed files are
-        lb_args = argparse.Namespace(**{ 'arch_roots': [repository_path], 'board_roots': [repository_path] })
+        # Look for boards in monitored repositories
+        lb_args = argparse.Namespace(**{ 'arch_roots': roots, 'board_roots': roots})
         known_boards = list_boards.find_boards(lb_args)
         for b in boards:
             name_re = re.compile(b)
             for kb in known_boards:
                 if name_re.search(kb.name):
                     all_boards.add(kb.name)
 
+        # If modified file is catched by "find_boards" workflow (change in "boards" dir AND board recognized)
+        # it means a proper testing scope for this file was found and this file can be removed
+        # from further consideration
+        for board in all_boards:
+            self.resolved_files.extend(list(filter(lambda f: board in f, resolved)))
+
         _options = []
         if len(all_boards) > 20:
             logging.warning(f"{len(boards)} boards changed, this looks like a global change, skipping test handling, revert to default.")
-            self.default_run = True
+            self.full_twister = True
             return
 
         for board in all_boards:
@@ -233,11 +257,25 @@ def find_tests(self):
             if f.endswith(".rst"):
                 continue
             d = os.path.dirname(f)
-            while d:
+            scope_found = False
+            while not scope_found and d:
+                head, tail = os.path.split(d)
                 if os.path.exists(os.path.join(d, "testcase.yaml")) or \
                     os.path.exists(os.path.join(d, "sample.yaml")):
                     tests.add(d)
-                    break
+                    # Modified file is treated as resolved, since a matching scope was found
+                    self.resolved_files.append(f)
+                    scope_found = True
+                elif tail == "common":
+                    # Look for yamls in directories collocated with common
+
+                    yamls_found = [yaml for yaml in glob.iglob(head + '/**/testcase.yaml', recursive=True)]
+                    yamls_found.extend([yaml for yaml in glob.iglob(head + '/**/sample.yaml', recursive=True)])
+                    if yamls_found:
+                        for yaml in yamls_found:
+                            tests.add(os.path.dirname(yaml))
+                        self.resolved_files.append(f)
+                        scope_found = True
                 else:
                     d = os.path.dirname(d)
 
@@ -247,7 +285,7 @@ def find_tests(self):
 
         if len(tests) > 20:
             logging.warning(f"{len(tests)} tests changed, this looks like a global change, skipping test handling, revert to default")
-            self.default_run = True
+            self.full_twister = True
             return
 
         if _options:
@@ -257,12 +295,11 @@ def find_tests(self):
                     _options.extend(["-p", platform])
             else:
                 _options.append("--all")
-            self.get_plan(_options)
+            self.get_plan(_options, use_testsuite_root=False)
 
     def find_tags(self):
 
-        tag_cfg_file = os.path.join(repository_path, 'scripts', 'ci', 'tags.yaml')
-        with open(tag_cfg_file, 'r') as ymlfile:
+        with open(self.tag_cfg_file, 'r') as ymlfile:
             tags_config = yaml.safe_load(ymlfile)
 
         tags = {}
@@ -299,26 +336,27 @@ def find_tags(self):
             logging.info(f'Potential tag based filters: {exclude_tags}')
 
     def find_excludes(self, skip=[]):
-        with open("scripts/ci/twister_ignore.txt", "r") as twister_ignore:
+        with open(self.ignore_path, "r") as twister_ignore:
             ignores = twister_ignore.read().splitlines()
             ignores = filter(lambda x: not x.startswith("#"), ignores)
 
         found = set()
-        files = list(filter(lambda x: x, self.modified_files))
+        files_not_resolved = list(filter(lambda x: x not in self.resolved_files, self.modified_files))
 
         for pattern in ignores:
-            if pattern in skip:
-                continue
             if pattern:
-                found.update(fnmatch.filter(files, pattern))
+                found.update(fnmatch.filter(files_not_resolved, pattern))
 
         logging.debug(found)
-        logging.debug(files)
+        logging.debug(files_not_resolved)
 
-        if sorted(files) != sorted(found):
+        # Full twister run can be ordered by detecting great number of tests/boards changed
+        # or if not all modified files were resolved (corresponding scope found)
+        self.full_twister = self.full_twister or sorted(files_not_resolved) != sorted(found)
+
+        if self.full_twister:
             _options = []
             logging.info(f'Need to run full or partial twister...')
-            self.full_twister = True
             if self.platforms:
                 for platform in self.platforms:
                     _options.extend(["-p", platform])
@@ -349,6 +387,20 @@ def parse_args():
             help="Number of tests per builder")
     parser.add_argument('-n', '--default-matrix', default=10, type=int,
             help="Number of tests per builder")
+    parser.add_argument('-r', '--repo-to-scan', default=None,
+                        help="Repo to scan")
+    parser.add_argument('--no-path-name', action="store_true",
+            help="Don't put paths into test suites' names ")
+    parser.add_argument('--ignore-path', default=None,
+            help="Path to a text file with patterns of files to be matched against changed files")
+    parser.add_argument('--alt-tags', default=None,
+            help="Path to a file describing relations between directories and tags")
+    parser.add_argument(
+        "-T", "--testsuite-root", action="append", default=[],
+        help="Base directory to recursively search for test cases. All "
+             "testcase.yaml files under here will be processed. May be "
+             "called multiple times. Defaults to the 'samples/' and "
+             "'tests/' directories at the base of the Zephyr tree.")
 
     return parser.parse_args()
 
@@ -358,9 +410,12 @@ def parse_args():
     args = parse_args()
     files = []
     errors = 0
+    repository_path = zephyr_base
+    if args.repo_to_scan:
+        repository_path = Path(args.repo_to_scan)
     if args.commits:
-        repo = Repo(repository_path)
-        commit = repo.git.diff("--name-only", args.commits)
+        repo_to_scan = Repo(repository_path)
+        commit = repo_to_scan.git.diff("--name-only", args.commits)
         files = commit.split("\n")
     elif args.modified_files:
         with open(args.modified_files, "r") as fp:
@@ -371,8 +426,7 @@ def parse_args():
         print("\n".join(files))
         print("=========")
 
-
-    f = Filters(files, args.pull_request, args.platform)
+    f = Filters(files, args.pull_request, args.platform, args.no_path_name, args.ignore_path, args.alt_tags, args.testsuite_root)
     f.process()
 
     # remove dupes and filtered cases

scripts/ci/twister_ignore.txt

@@ -17,25 +17,6 @@ CODEOWNERS
 MAINTAINERS.yml
 LICENSE
 Makefile
-tests/*
-samples/*
-boards/*/*/*
-arch/xtensa/*
-arch/x86/*
-arch/posix/*
-arch/arc/*
-arch/sparc/*
-arch/arm/*
-arch/nios2/*
-arch/riscv/*
-include/arch/xtensa/*
-include/arch/x86/*
-include/arch/posix/*
-include/arch/arc/*
-include/arch/sparc/*
-include/arch/arm/*
-include/arch/nios2/*
-include/arch/riscv/*
 doc/*
 # GH action have no impact on code
 .github/*

scripts/pylib/twister/twisterlib/environment.py

@@ -437,6 +437,10 @@ def add_parse_arguments(parser = None):
         help="Re-use the outdir before building. Will result in "
              "faster compilation since builds will be incremental.")
 
+    parser.add_argument(
+        "--no-path-name", action="store_true",
+        help="Don't put paths into test suites' names ")
+
     # To be removed in favor of --detailed-skipped-report
     parser.add_argument(
         "--no-skipped-report", action="store_true",

scripts/pylib/twister/twisterlib/testplan.py

@@ -522,7 +522,7 @@ def add_testsuites(self, testsuite_filter=[]):
 
                     for name in parsed_data.scenarios.keys():
                         suite_dict = parsed_data.get_scenario(name)
-                        suite = TestSuite(root, suite_path, name, data=suite_dict)
+                        suite = TestSuite(root, suite_path, name, data=suite_dict, no_path_name=self.options.no_path_name)
                         suite.add_subcases(suite_dict, subcases, ztest_suite_names)
                         if testsuite_filter:
                             if suite.name and suite.name in testsuite_filter:

scripts/pylib/twister/twisterlib/testsuite.py

@@ -348,7 +348,7 @@ class TestSuite(DisablePyTestCollectionMixin):
     """Class representing a test application
     """
 
-    def __init__(self, suite_root, suite_path, name, data=None):
+    def __init__(self, suite_root, suite_path, name, data=None, no_path_name=False):
         """TestSuite constructor.
 
         This gets called by TestPlan as it finds and reads test yaml files.
@@ -369,7 +369,9 @@ def __init__(self, suite_root, suite_path, name, data=None):
         """
 
         workdir = os.path.relpath(suite_path, suite_root)
-        self.name = self.get_unique(suite_root, workdir, name)
+
+        assert self.check_suite_name(name, suite_root, workdir)
+        self.name = name if no_path_name else self.get_unique(suite_root, workdir, name)
         self.id = name
 
         self.source_dir = suite_path
@@ -425,10 +427,14 @@ def get_unique(testsuite_root, workdir, name):
 
         # workdir can be "."
         unique = os.path.normpath(os.path.join(relative_ts_root, workdir, name))
+        return unique
+
+    @staticmethod
+    def check_suite_name(name, testsuite_root, workdir):
         check = name.split(".")
         if len(check) < 2:
             raise TwisterException(f"""bad test name '{name}' in {testsuite_root}/{workdir}. \
 Tests should reference the category and subsystem with a dot as a separator.
                     """
                     )
-        return unique
+        return True

scripts/tests/twister/test_testsuite.py

@@ -110,3 +110,42 @@ def test_get_unique(testsuite_root, suite_path, name, expected):
     '''Test to check if the unique name is given for each testsuite root and workdir'''
     suite = TestSuite(testsuite_root, suite_path, name)
     assert suite.name == expected
+
+TESTDATA_2 = [
+        (
+            ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites',
+            ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/tests/test_a',
+            'test_a.check_1',
+            'test_a.check_1'
+        ),
+        (
+            ZEPHYR_BASE,
+            ZEPHYR_BASE,
+            'test_a.check_1',
+            'test_a.check_1'
+        ),
+        (
+            ZEPHYR_BASE,
+            ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/test_b',
+            'test_b.check_1',
+            'test_b.check_1'
+        ),
+        (
+            os.path.join(ZEPHYR_BASE, 'scripts/tests'),
+            os.path.join(ZEPHYR_BASE, 'scripts/tests'),
+            'test_b.check_1',
+            'test_b.check_1'
+        ),
+        (
+            ZEPHYR_BASE,
+            ZEPHYR_BASE,
+            'test_a.check_1.check_2',
+            'test_a.check_1.check_2'
+        ),
+]
+@pytest.mark.parametrize("testsuite_root, suite_path, name, expected", TESTDATA_2)
+def test_get_no_path_name(testsuite_root, suite_path, name, expected):
+    '''Test to check if the name without path is given for each testsuite'''
+    suite = TestSuite(testsuite_root, suite_path, name, no_path_name=True)
+    print(suite.name)
+    assert suite.name == expected