dependencies: compute and check release dates via GitHub API. (envoyproxy#13582)

The use of last_updated was ambiguous (is it when an Envoy contributor
creates a PR, merges a commit, or when the dependency is released?).
We really are after the release date as a measure of how stale the
dependency is.

This patch introduces a tool, tools/dependency/release_dates.py, that
uses the GitHub API to compute release date. If a mismatch is detected, an
error is raised.

This patch also introduces a dependency validation CI job that gathers existing
scripts and the release_dates.py script into a single job.

Signed-off-by: Harvey Tuch <[email protected]>

Co-authored-by: Michael Payne <[email protected]>

Author: htuch

.azure-pipelines/pipelines.yml

@@ -87,6 +87,21 @@ stages:
               AZP_BRANCH: $(Build.SourceBranch)
               AZP_SHA1: $(Build.SourceVersion)
             condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
+      - job: dependencies
+        dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+        pool:
+          vmImage: "ubuntu-18.04"
+        steps:
+          - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps'
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)
+            displayName: "Verify dependency information"
+
 
   - stage: sync
     condition: and(succeeded(), eq(variables['PostSubmit'], true))

DEPENDENCY_POLICY.md

@@ -45,8 +45,11 @@ Dependency declarations must:
   version is being used.
 * Provide accurate entries for `use_category`. Please think carefully about whether there are data
   or control plane implications of the dependency.
-* Reflect the date (YYYY-MM-DD) at which they were last updated in the `last_updated` field. This
-  date is preferably the date at which the PR is created.
+* Reflect the UTC date (YYYY-MM-DD format) for the dependency release. This is when
+  the dependency was updated in its repository. For dependencies that have
+  releases, this is the date of the release. For dependencies without releases
+  or for scenarios where we temporarily need to use a commit, this date should
+  be the date of the commit in UTC.
 * CPEs are compulsory for all dependencies that are not purely build/test.
   [CPEs](https://en.wikipedia.org/wiki/Common_Platform_Enumeration) provide metadata that allow us
   to correlate with related CVEs in dashboards and other tooling, and also provide a machine

api/bazel/external_deps.bzl

@@ -17,14 +17,18 @@ DEPENDENCY_ANNOTATIONS = [
     # Envoy (see the external dependency at the given version for information).
     "implied_untracked_deps",
 
-    # When the dependency was last updated in Envoy.
-    "last_updated",
-
     # Project metadata.
     "project_desc",
     "project_name",
     "project_url",
 
+    # Reflects the UTC date (YYYY-MM-DD format) for the dependency release. This
+    # is when the dependency was updated in its repository. For dependencies
+    # that have releases, this is the date of the release. For dependencies
+    # without releases or for scenarios where we temporarily need to use a
+    # commit, this date should be the date of the commit in UTC.
+    "release_date",
+
     # List of the categories describing how the dependency is being used. This attribute is used
     # for automatic tracking of security posture of Envoy's dependencies.
     # Possible values are documented in the USE_CATEGORIES list below.
@@ -63,8 +67,7 @@ USE_CATEGORIES = [
     "devtools",
 ]
 
-# Components with these use categories are not required to specify the 'cpe'
-# and 'last_updated' annotation.
+# Components with these use categories are not required to specify the 'cpe'.
 USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"]
 
 def _fail_missing_attribute(attr, key):
@@ -106,13 +109,13 @@ def load_repository_locations(repository_locations_spec):
             if "extensions" not in location:
                 _fail_missing_attribute("extensions", key)
 
-        if "last_updated" not in location:
-            _fail_missing_attribute("last_updated", key)
-        last_updated = location["last_updated"]
+        if "release_date" not in location:
+            _fail_missing_attribute("release_date", key)
+        release_date = location["release_date"]
 
         # Starlark doesn't have regexes.
-        if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
-            fail("last_updated must match YYYY-DD-MM: " + last_updated)
+        if len(release_date) != 10 or release_date[4] != "-" or release_date[7] != "-":
+            fail("release_date must match YYYY-DD-MM: " + release_date)
 
         if "cpe" in location:
             cpe = location["cpe"]

api/bazel/repository_locations.bzl

@@ -7,7 +7,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         version = "1.0.3",
         sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
         urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz"],
-        last_updated = "2020-08-27",
+        release_date = "2020-08-27",
         use_category = ["api"],
     ),
     com_envoyproxy_protoc_gen_validate = dict(
@@ -18,7 +18,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "e368733c9fb7f8489591ffaf269170d7658cc0cd1ee322b601512b769446d3c8",
         strip_prefix = "protoc-gen-validate-{version}",
         urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/{version}.tar.gz"],
-        last_updated = "2020-06-09",
+        release_date = "2020-06-08",
         use_category = ["api"],
     ),
     com_github_cncf_udpa = dict(
@@ -29,7 +29,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "83a7dcc316d741031f34c0409021432b74a39c4811845a177133f02f948fe2d8",
         strip_prefix = "udpa-{version}",
         urls = ["https://github.com/cncf/udpa/archive/v{version}.tar.gz"],
-        last_updated = "2020-09-23",
+        release_date = "2020-06-29",
         use_category = ["api"],
     ),
     com_github_openzipkin_zipkinapi = dict(
@@ -40,7 +40,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b",
         strip_prefix = "zipkin-api-{version}",
         urls = ["https://github.com/openzipkin/zipkin-api/archive/{version}.tar.gz"],
-        last_updated = "2020-09-23",
+        release_date = "2019-08-23",
         use_category = ["api"],
     ),
     com_google_googleapis = dict(
@@ -52,7 +52,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d1405",
         strip_prefix = "googleapis-{version}",
         urls = ["https://github.com/googleapis/googleapis/archive/{version}.tar.gz"],
-        last_updated = "2019-12-02",
+        release_date = "2019-12-02",
         use_category = ["api"],
     ),
     opencensus_proto = dict(
@@ -63,7 +63,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0",
         strip_prefix = "opencensus-proto-{version}/src",
         urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v{version}.tar.gz"],
-        last_updated = "2020-06-20",
+        release_date = "2020-07-21",
         use_category = ["api"],
     ),
     prometheus_metrics_model = dict(
@@ -74,7 +74,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "6748b42f6879ad4d045c71019d2512c94be3dd86f60965e9e31e44a3f464323e",
         strip_prefix = "client_model-{version}",
         urls = ["https://github.com/prometheus/client_model/archive/{version}.tar.gz"],
-        last_updated = "2020-06-23",
+        release_date = "2020-06-23",
         use_category = ["api"],
     ),
     rules_proto = dict(
@@ -85,7 +85,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "aa1ee19226f707d44bee44c720915199c20c84a23318bb0597ed4e5c873ccbd5",
         strip_prefix = "rules_proto-{version}",
         urls = ["https://github.com/bazelbuild/rules_proto/archive/{version}.tar.gz"],
-        last_updated = "2020-08-17",
+        release_date = "2020-08-17",
         use_category = ["api"],
     ),
 )

bazel/repositories_extra.bzl

@@ -14,7 +14,7 @@ def _python_deps():
         # project_name = "PyYAML",
         # project_url = "https://github.com/yaml/pyyaml",
         # version = "5.3.1",
-        # last_update = "2020-03-18"
+        # release_date = "2020-03-18"
         # use_category = ["devtools"],
         # cpe = "cpe:2.3:a:pyyaml:pyyaml:*",
     )
@@ -26,14 +26,14 @@ def _python_deps():
         # project_name = "Jinja",
         # project_url = "http://palletsprojects.com/p/jinja",
         # version = "2.11.2",
-        # last_update = "2020-04-13"
+        # release_date = "2020-04-13"
         # use_category = ["test"],
         # cpe = "cpe:2.3:a:palletsprojects:jinja:*",
 
         # project_name = "MarkupSafe",
         # project_url = "https://markupsafe.palletsprojects.com/en/1.1.x/",
         # version = "1.1.1",
-        # last_update = "2019-02-23"
+        # release_date = "2019-02-23"
         # use_category = ["test"],
     )
     pip3_import(
@@ -44,14 +44,14 @@ def _python_deps():
         # project_name = "Jinja",
         # project_url = "http://palletsprojects.com/p/jinja",
         # version = "2.11.2",
-        # last_update = "2020-04-13"
+        # release_date = "2020-04-13"
         # use_category = ["test"],
         # cpe = "cpe:2.3:a:palletsprojects:jinja:*",
 
         # project_name = "MarkupSafe",
         # project_url = "https://markupsafe.palletsprojects.com/en/1.1.x/",
         # version = "1.1.1",
-        # last_update = "2019-02-23"
+        # release_date = "2019-02-23"
         # use_category = ["test"],
     )
     pip3_import(
@@ -62,7 +62,7 @@ def _python_deps():
         # project_name = "Clang",
         # project_url = "https://clang.llvm.org/",
         # version = "10.0.1",
-        # last_update = "2020-07-21"
+        # release_date = "2020-07-21"
         # use_category = ["devtools"],
         # cpe = "cpe:2.3:a:llvm:clang:*",
     )
@@ -74,7 +74,7 @@ def _python_deps():
         # project_name = "PyYAML",
         # project_url = "https://github.com/yaml/pyyaml",
         # version = "5.3.1",
-        # last_update = "2020-03-18"
+        # release_date = "2020-03-18"
         # use_category = ["docs"],
         # cpe = "cpe:2.3:a:pyyaml:pyyaml:*",
     )
@@ -86,14 +86,14 @@ def _python_deps():
         # project_name = "Apache Thrift",
         # project_url = "http://thrift.apache.org/",
         # version = "0.11.0",
-        # last_update = "2017-12-07"
+        # release_date = "2017-12-07"
         # use_category = ["test"],
         # cpe = "cpe:2.3:a:apache:thrift:*",
 
         # project_name = "Six: Python 2 and 3 Compatibility Library",
         # project_url = "https://six.readthedocs.io/",
         # version = "1.15.0",
-        # last_update = "2020-05-21"
+        # release_date = "2020-05-21"
         # use_category = ["test"],
     )