feat: add git sha and remotes info to product metadata by default

 - this is for build traceability and provenance

Co-authored-by: Nick Rohn <[email protected]>

Author: notrepo05

requirements.txt

@@ -3,7 +3,8 @@ click>=6.2
 Jinja2>=3.1
 PyYAML>=3.1
 docker-py>=1.6.0
-requests<2.2.29
+requests<=2.5.3
 requests-toolbelt
 mock>=2.0.0
 pexpect>=4.2.1
+GitPython>=3.1.32

tile_generator/config.py

@@ -1,5 +1,8 @@
 #!/usr/bin/env python
+import subprocess
 
+import git
+from git import Repo
 # tile-generator
 #
 # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved.
@@ -61,6 +64,12 @@
 # Normalize Jobs - Ensure that job type, template, and properties are set for
 # every job
 
+def _find_git_root():
+	try:
+		git_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
+		return git_root
+	except subprocess.CalledProcessError as e:
+		return e
 
 # Inspired by https://gist.github.com/angstwad/bf22d1822c38a92ec0a9
 def merge_dict(dct, merge_dct):
@@ -172,6 +181,13 @@ def _validate_base_config(self):
 			'description': {'type': 'string', 'required': True},
 			'icon_file': {'type': 'string', 'required': True, 'coerce': _base64_img},
 			'metadata_version': {'type': 'number', 'default': 1.8},
+			'git_remotes': {'type': 'list', 'required': False, 'schema': {
+				'type': 'dict', 'schema': {
+					'name': {'type': 'string', 'required': True},
+					'urls': {'type': 'list', 'required': True},
+				}
+			}},
+			'git_sha': {'type': 'string', 'required': False},
 			'stemcell_criteria': {'type': 'dict', 'default': self.default_stemcell(), 'schema': {
 				'os': {'type': 'string'}, 'version': {'type': 'string'}}},
 			'release_overides': {'type': 'dict'},
@@ -455,6 +471,24 @@ def upgrade(self):
 				print('ERROR - configurable_persistence has been deprecated, use auto_services instead', file=sys.stderr)
 				sys.exit(1)
 
+
+	def set_git_remotes(self, git_remotes=True):
+		if not git_remotes:
+			return
+		git_root = _find_git_root()
+		repo = git.Repo(git_root)
+
+		remotes = [{"name": remote.name, "urls": [str(url) for url in remote.urls]} for remote in repo.remotes]
+		print("setting remotes {}".format(remotes))
+		self['git_remotes'] = remotes
+
+	def set_git_sha(self, git_sha=True):
+		if not git_sha:
+			return
+		git_root = _find_git_root()
+		repo = git.Repo(git_root)
+		self['git_sha'] = repo.head.commit.hexsha
+
 	def set_version(self, version):
 		if version is None:
 			version = 'patch'

tile_generator/config_unittest.py

@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import git
 import mock
 import json
 import os
@@ -29,6 +29,16 @@
 from .config import Config, merge_dict
 from .tile_metadata import TileMetadata
 from . import template
+from types import SimpleNamespace
+
+
+class MockRepo:
+    argument = []
+
+    def __init__(self, x):
+        self.argument.append(x)
+        self.remotes = [SimpleNamespace(urls=["[email protected]:org/repo.git"], name="origin")]
+        self.head = SimpleNamespace(commit=SimpleNamespace(hexsha="some-sha"))
 
 @contextmanager
 def capture_output():
@@ -61,28 +71,37 @@ def tearDown(self):
     self.pre_start_file.close()
 
 
-@mock.patch('tile_generator.config.Config.read_history')
+def side_effects(*args, **kwargs):
+  side_effects.captured_args = args
+  side_effects.captured_kwargs = kwargs
+
+@mock.patch('git.Repo', MockRepo)
+@mock.patch('tile_generator.config._find_git_root')
 @mock.patch('tile_generator.config._base64_img')
+@mock.patch('tile_generator.config.Config.read_history')
 class TestUltimateForm(BaseTest):
-  def test_diff_final_config_obj(self, mock_read_history, mock_base64_img):
+  def test_diff_final_config_obj(self, mock_read_history, mock_base64_img, mock__find_git_root):
     test_path = os.path.dirname(__file__)
     cfg_file_path = os.path.join(test_path, '../sample/tile.yml')
+    mock__find_git_root.return_value = "/fake/git/root"
 
     with mock.patch('tile_generator.config.CONFIG_FILE', cfg_file_path):
       cfg = self.config.read()
     cfg.set_version(None)
+    cfg.set_git_remotes(True)
     cfg.set_verbose(False)
     cfg.set_sha1(False)
     cfg.set_cache(None)
-    
+    cfg.set_git_sha(True)
+
     with open(test_path + '/test_config_generated_output.json', 'w') as f:
       generated_json = json.dumps(cfg, sort_keys=True, indent=2)
       f.write(generated_json)
       generated_output = json.loads(generated_json)
 
     with open(test_path + '/test_config_expected_output.json', 'r') as f:
       expected_output = json.load(f)
-    
+
     # Change the paths to files to be consistent
     self.fix_path(generated_output)
     self.fix_path(expected_output)
@@ -97,10 +116,14 @@ def test_diff_final_config_obj(self, mock_read_history, mock_base64_img):
     expected = json.dumps(expected_output, indent=2).split('\n')
     generated = json.dumps(generated_output, indent=2).split('\n')
 
+    self.assertGreater(len(MockRepo.argument), 0)
+    self.assertEqual(MockRepo.argument[-1], '/fake/git/root')
+
     self.assertEqual(len(expected), len(generated))
     for line in expected:
       self.assertIn(line, generated)
 
+
   def fix_path(self, obj):
     for release in obj['releases'].values():
       for package in release.get('packages', []):
@@ -169,7 +192,7 @@ def deep_comparer(self, expected, given, path):
         'Expected to have the value:\n%s\nHowever, instead got:\n%s' % (expected, given)))
 
   # mock_read_history, mock_base64_img are not used in this method
-  def test_metadata_diff(self, mock_read_history, mock_base64_img):
+  def test_metadata_diff(self, mock_read_history, mock_base64_img, mock__find_git_root):
     test_path = os.path.dirname(__file__)
     cfg_file_path = os.path.join(test_path, '../sample/tile.yml')
 

tile_generator/templates/tile/metadata.yml

@@ -52,3 +52,11 @@ update:
 provides_product_versions:
 - name: {{ tile_metadata.base.name }}
   version: '{{ tile_metadata.base.product_version }}'
+
+{% if tile_metadata.git_sha %}
+git_sha: {{ tile_metadata.git_sha }}
+{% endif %}
+{% if tile_metadata.git_remotes %}
+git_remotes:
+{{ tile_metadata.git_remotes | yaml }}
+{% endif %}

tile_generator/test_config_expected_output.json

@@ -4256,6 +4256,13 @@
     }
   ],
   "sha1": false,
+  "git_remotes": [
+    {
+      "name":  "origin",
+      "urls": ["[email protected]:org/repo.git"]
+    }
+  ],
+  "git_sha": "some-sha",
   "space": "test-tile-space",
   "standalone": false,
   "stemcell_criteria": {

tile_generator/tile.py

@@ -54,18 +54,23 @@ def init_cmd(name):
 @click.option('--verbose', is_flag=True)
 @click.option('--sha1', is_flag=True)
 @click.option('--cache', type=str, default=None)
-def build_cmd(version, verbose, sha1, cache):
+@click.option('--git-info', type=bool, default=True)
+def build_cmd(version, verbose, sha1, cache, git_info):
 	cfg = Config().read()
 
 	cfg.set_version(version)
 	cfg.set_verbose(verbose)
-	cfg.set_sha1(sha1)
 	cfg.set_cache(cache)
+	cfg.set_git_remotes(git_info)
+	cfg.set_git_sha(git_info)
+	cfg.set_sha1(sha1)
 	print('name:', cfg.get('name', '<unspecified>'))
 	print('label:', cfg.get('label', '<unspecified>'))
 	print('description:', cfg.get('description', '<unspecified>'))
 	print('version:', cfg.get('version', '<unspecified>'))
 	print('sha1:', cfg.get('sha1', '<unspecified>'))
+	print('git_remotes:', cfg.get('git_remotes', '<unspecified>'))
+	print('git_sha:', cfg.get('git_sha', '<unspecified>'))
 	stemcell = cfg.get('stemcell_criteria', {})
 	print('stemcell:', stemcell.get('os', '<unspecified>'), stemcell.get('version', '<unspecified>'))
 	print()

tile_generator/tile_metadata.py

@@ -18,6 +18,7 @@ def __init__(self, config):
     def build(self):
         self._build_base()
         self._build_stemcell_criteria()
+        self._build_git_info()
         self._build_property_blueprints()
         self._build_form_types()
         self._build_job_types()
@@ -46,6 +47,10 @@ def _build_base(self):
 
         self.tile_metadata['base'] = base
 
+    def _build_git_info(self):
+        self.tile_metadata['git_remotes'] = self.config.get('git_remotes')
+        self.tile_metadata['git_sha'] = self.config.get('git_sha')
+
     def _build_stemcell_criteria(self):
         stemcell_criteria = self.config['stemcell_criteria']
         # Note: tile.py uses self.config['stemcell_criteria']

tile_generator/tile_metadata_unittest.py

@@ -145,5 +145,19 @@ def test_non_standalone_tile_includes_default_cf_blueprints(self):
 
         self.assertEqual(metadata.tile_metadata['property_blueprints'], expected_blueprints)
 
+    def test_git_info(self):
+            config = Config(
+                name='validname',
+                git_sha='some-sha',
+                git_remotes=[{'name': 'name', 'sha1': 'abc'}]
+            )
+
+            metadata = TileMetadata(config)
+            metadata._build_git_info()
+
+            self.assertEqual(metadata.tile_metadata['git_sha'], 'some-sha')
+            self.assertEqual(metadata.tile_metadata['git_remotes'], [{'name': 'name', 'sha1': 'abc'}])
+
+
 if __name__ == '__main__':
     unittest.main()