Add support for required key

sample.redhat-ci.yml

@@ -79,6 +79,14 @@ branches:
 # 'Red Hat CI'.
 context: 'CI Tester'
 
+# OPTIONAL
+# Mark this testsuite as required. This causes a special
+# "required" context to be reported to GitHub on branch
+# tests. The result is set to successful only if all
+# testsuites marked as required are also successful. If
+# omitted, defaults to false.
+required: true
+
 # OPTIONAL
 # Additional YUM repositories to inject during provisioning.
 extra-repos:

spawner.py

@@ -21,17 +21,19 @@ def main():
     "Main entry point."
 
     try:
-        n = parse_suites()
+        suites = parse_suites()
     except ScannerError:
-        update_gh('error', "YAML syntax error.")
+        update_gh('error', "Red Hat CI", "YAML syntax error.")
     except SchemaError as e:
         # print the error to give feedback in the logs, but exit nicely
         traceback.print_exc()
-        update_gh('error', "YAML semantic error.")
+        update_gh('error', "Red Hat CI", "YAML semantic error.")
     else:
+        n = len(suites)
         if n > 0:
             spawn_testrunners(n)
-            count_failures(n)
+            inspect_suite_failures(suites)
+            update_required_context(suites)
         else:
             print("INFO: No testsuites to run.")
 
@@ -45,21 +47,21 @@ def parse_suites():
     # this should have been checked already
     assert os.path.isfile(yml_file)
 
-    nsuites = 0
+    suites = []
     branch = os.environ.get('github_branch')
-    for idx, suite in enumerate(parser.load_suites(yml_file)):
+    suite_parser = parser.SuiteParser(yml_file)
+    for idx, suite in enumerate(suite_parser.parse()):
         if len(os.environ.get('RHCI_DEBUG_ALWAYS_RUN', '')) == 0:
             branches = suite.get('branches', ['master'])
             if branch is not None and branch not in branches:
                 print("INFO: %s suite not defined to run for branch %s." %
                       (common.ordinal(idx + 1), branch))
                 continue
-        suite_dir = 'state/suite-%d/parsed' % nsuites
+        suite_dir = 'state/suite-%d/parsed' % len(suites)
         parser.flush_suite(suite, suite_dir)
-        nsuites += 1
+        suites.append(suite)
 
-    # return the number of testsuites
-    return nsuites
+    return suites
 
 
 def spawn_testrunners(n):
@@ -105,38 +107,55 @@ def read_pipe(idx, fd):
         s = fd.readline()
 
 
-def count_failures(n):
+def inspect_suite_failures(suites):
 
-    # It's helpful to have an easy global way to figure out
-    # if any of the suites failed, e.g. for integration in
-    # Jenkins. Let's write a 'failures' file counting the
-    # number of failed suites.
+    for i, suite in enumerate(suites):
+        assert 'rc' not in suite
 
-    failed = 0
-    for i in range(n):
         # If the rc file doesn't exist but the runner exited
         # nicely, then it means there was a semantic error
         # in the YAML (e.g. bad Docker image, bad ostree
         # revision, etc...).
         if not os.path.isfile("state/suite-%d/rc" % i):
-            failed += 1
+            suite['rc'] = 1
         else:
             with open("state/suite-%d/rc" % i) as f:
-                if int(f.read().strip()) != 0:
-                    failed += 1
+                suite['rc'] = int(f.read().strip())
 
+    # It's helpful to have an easy global way to figure out
+    # if any of the suites failed, e.g. for integration in
+    # Jenkins. Let's write a 'failures' file counting the
+    # number of failed suites.
     with open("state/failures", "w") as f:
-        f.write("%d" % failed)
+        f.write("%d" % count_failures(suites))
+
+
+def count_failures(suites):
+    return sum([int(suite['rc'] != 0) for suite in suites])
+
+
+def update_required_context(suites):
+
+    # only send 'required' context for branches
+    if 'github_pull_id' in os.environ:
+        return
+
+    required_suites = [suite for suite in suites if suite.get('required')]
+    failed = count_failures(required_suites)
+    total = len(required_suites)
+
+    update_gh('success' if failed == 0 else 'failure', 'required',
+              "%d/%d required testsuites passed" % (total - failed, total))
 
 
-def update_gh(state, description):
+def update_gh(state, context, description):
 
     try:
         args = {'repo': os.environ['github_repo'],
                 'commit': os.environ['github_commit'],
                 'token': os.environ['github_token'],
                 'state': state,
-                'context': 'Red Hat CI',
+                'context': context,
                 'description': description}
 
         ghupdate.send(**args)

utils/parser.py

@@ -1,99 +1,102 @@
 #!/usr/bin/env python3
 
-# This is just a trivial parser for now, but as we add more
-# functionality to the .redhat-ci.yml spec, we will want to
-# integrate pieces of the pipeline in here. E.g.
-# provisioning, prereqs, test runs, etc...
-
 import os
 import sys
 import yaml
 import shlex
-import argparse
 
 import utils.common as common
 
 from pykwalify.core import Core
 from pykwalify.errors import SchemaError
 
 
-def load_suites(filepath):
-    "Generator of testsuites parsed from the given YAML file."
+class SuiteParser:
+
+    def __init__(self, filepath):
+        self.contexts = []
+        self.met_required = False
+        self.filepath = filepath
 
-    suite = None
-    contexts = []
-    with open(filepath) as f:
-        for idx, raw_yaml in enumerate(yaml.safe_load_all(f.read())):
-            try:
-                suite = _merge(suite, raw_yaml)
-                _validate(suite, contexts)
-                yield suite
-            except SchemaError as e:
-                # if it happens on the very first document, let's just give the
-                # exact error directly
-                if idx == 0:
-                    raise e
-                msg = "failed to parse %s testsuite" % common.ordinal(idx + 1)
-                raise SchemaError(msg) from e
+    def parse(self):
+        "Generator of testsuites parsed from the given YAML file."
 
+        suite = None
+        with open(self.filepath) as f:
+            for idx, raw_yaml in enumerate(yaml.safe_load_all(f.read())):
+                try:
+                    suite = self._merge(suite, raw_yaml)
+                    self._validate(suite)
+                    yield dict(suite)
+                except SchemaError as e:
+                    # if it happens on the very first document, let's
+                    # just give the exact error directly
+                    if idx == 0:
+                        raise e
+                    raise SchemaError("failed to parse %s testsuite"
+                                      % common.ordinal(idx + 1)) from e
 
-def _merge(suite, new):
-    "Merge the next document into the current one."
+    def _merge(self, suite, new):
+        "Merge the next document into the current one."
 
-    if type(new) is not dict:
-        raise SyntaxError("top-level type should be a dict")
+        if type(new) is not dict:
+            raise SyntaxError("top-level type should be a dict")
 
-    if suite is None:
+        if suite is None:
 
-        # The 'context' key is special. It's optional on the
-        # first suite (defaulting to 'Red Hat CI'), but
-        # required on subsequent suites.
-        if 'context' not in new:
-            new['context'] = "Red Hat CI"
+            # The 'context' key is special. It's optional on the
+            # first suite (defaulting to 'Red Hat CI'), but
+            # required on subsequent suites.
+            if 'context' not in new:
+                new['context'] = "Red Hat CI"
 
-    if 'inherit' in new and type(new['inherit']) is not bool:
-        raise SyntaxError("expected 'bool' value for 'inherit' key")
+        if 'inherit' in new and type(new['inherit']) is not bool:
+            raise SyntaxError("expected 'bool' value for 'inherit' key")
 
-    # if we're not inheriting, then let's just return the new suite itself
-    if suite is None or not new.get('inherit', False):
-        return _normalize(new.copy())
+        # if we're not inheriting, then let's just return the new suite itself
+        if suite is None or not new.get('inherit', False):
+            return self._normalize(new.copy())
 
-    assert type(suite) is dict
+        assert type(suite) is dict
 
-    # if the suite specifies an envtype, then make sure we
-    # don't inherit the envtype of the old one
-    envtypes = ['container', 'host', 'cluster']
-    if any([i in new for i in envtypes]):
-        for i in envtypes:
-            if i in suite:
-                del suite[i]
+        # if the suite specifies an envtype, then make sure we
+        # don't inherit the envtype of the old one
+        envtypes = ['container', 'host', 'cluster']
+        if any([i in new for i in envtypes]):
+            for i in envtypes:
+                if i in suite:
+                    del suite[i]
 
-    # we always expect a new context key
-    del suite['context']
+        # we always expect a new context key
+        del suite['context']
 
-    suite.update(new)
+        suite.update(new)
 
-    return _normalize(suite)
+        return self._normalize(suite)
 
+    def _normalize(self, suite):
+        for k, v in list(suite.items()):
+            if k == 'inherit' or v is None:
+                del suite[k]
+        return suite
 
-def _normalize(suite):
-    for k, v in list(suite.items()):
-        if k == 'inherit' or v is None:
-            del suite[k]
-    return suite
+    def _validate(self, suite):
 
+        schema = os.path.join(sys.path[0], "utils/schema.yml")
+        ext = os.path.join(sys.path[0], "utils/ext_schema.py")
+        c = Core(source_data=suite, schema_files=[schema], extensions=[ext])
+        c.validate()
 
-def _validate(suite, contexts):
+        if suite['context'] in self.contexts:
+            raise SchemaError("duplicate 'context' value detected")
 
-    schema = os.path.join(sys.path[0], "utils/schema.yml")
-    ext = os.path.join(sys.path[0], "utils/ext_schema.py")
-    c = Core(source_data=suite, schema_files=[schema], extensions=[ext])
-    c.validate()
+        self.met_required = self.met_required or suite.get('required', False)
 
-    if suite['context'] in contexts:
-        raise SchemaError("duplicate 'context' value detected")
+        if suite['context'] == "required" and self.met_required:
+            raise SchemaError('context "required" forbidden when using the '
+                              "'required' key")
 
-    contexts.append(suite['context'])
+        self.contexts.append(suite['context'])
 
 
 def write_to_file(dir, fn, s):
@@ -193,18 +196,3 @@ def flush_suite(suite, outdir):
                           v.get('build-opts', ''))
             write_to_file(outdir, "build.install_opts",
                           v.get('install-opts', ''))
-
-
-if __name__ == '__main__':
-
-    # Just dump each parsed document in indexed subdirs of
-    # output_dir. Useful for testing and validating.
-
-    argparser = argparse.ArgumentParser()
-    argparser.add_argument('--yml-file', required=True)
-    argparser.add_argument('--output-dir', required=True)
-    args = argparser.parse_args()
-
-    for idx, suite in enumerate(load_suites(args.yml_file)):
-        suite_dir = os.path.join(args.output_dir, str(idx))
-        flush_suite(suite, suite_dir)

utils/schema.yml

@@ -24,13 +24,15 @@ mapping:
           image:
             type: str
             required: true
-  context:
-    type: str
-    required: true
   branches:
     sequence:
       - type: str
         unique: true
+  context:
+    type: str
+    required: true
+  required:
+    type: bool
   extra-repos:
     type: any
     func: ext_repos

validator.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+'''
+    Simple script to validate a YAML file.
+    Usage: ./validator.py /my/github/project/.redhat-ci.yml
+'''
+
+import os
+import pprint
+import argparse
+import utils.parser as parser
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('yml_file', help="YAML file to parse and validate")
+argparser.add_argument('--output-dir', metavar="DIR",
+                       help="directory to which to flush suites if desired")
+args = argparser.parse_args()
+
+suite_parser = parser.SuiteParser(args.yml_file)
+for idx, suite in enumerate(suite_parser.parse()):
+    print("INFO: validated suite %d" % idx)
+    pprint.pprint(suite, indent=4)
+    if args.output_dir:
+        suite_dir = os.path.join(args.output_dir, str(idx))
+        parser.flush_suite(suite, suite_dir)
+        print("INFO: flushed to %s" % suite_dir)