From b64fd5d25fb341491074dba881872548a1990ab8 Mon Sep 17 00:00:00 2001 From: Patrick McCarthy Date: Wed, 4 Sep 2019 20:23:16 +0100 Subject: [PATCH] Addition of opinionated Terrafile support --- README.md | 62 ++++++++++++++++++++++- setup.py | 2 +- terrafile/__init__.py | 113 ++++++++++++++++++++++++++++++++++++------ terrafile/__main__.py | 19 +++++-- 4 files changed, 176 insertions(+), 20 deletions(-) mode change 100644 => 100755 terrafile/__init__.py mode change 100644 => 100755 terrafile/__main__.py diff --git a/README.md b/README.md index 6f66203..a298664 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,14 @@ pterrafile [path] If `path` is provided, it must be the path to a `Terrafile` file, or a directory containing one. If not provided, it looks for the file in the current working directory. -## Examples +```shell +pterrafile --terrafile +``` + +Same behavour as pterrafile [path] + + +## Examples terrafile usage ```yaml # Terraform Registry module @@ -42,3 +49,56 @@ terraform-aws-lambda: terraform-aws-lambda: source: "../../modules/terraform-aws-lambda" ``` + +## Opinionated Terrafile + +```shell +pterrafile --terrafile --optimizedownloads True +``` + +if --optimizedownloads is set to True then this indicates the usage of an opinionated Terrafile. The module names are used as the key in the terrafile and pterrafile will auto-detect the module names within your terrafiles and download only the matching names in your Terrafile. This allows you to utilize a single Terrafile at the top level of an environment. This is useful when you want to easily track the versions of all your modules in a single Terrafile and allows you to call out to this central Terrafile from sub folders, and only download the specific modules you require for terraform apply. + +## Example using opinionated Terrafile + +```shell +cd dev/apps +pterrafile --terrafile ../Terrafile --optimizedownloads True +``` + +```yaml +├── dev +│ ├── Terrafile +│ ├── apps +│ │ └── main.tf +│ │ └── modules +│ ├── ec2 +│ │ └── main.tf +│ │ └── modules +│ ├── security +│ │ └── main.tf +│ │ └── modules +│ │── vpc +│ │ └── main.tf +│ │ └── modules +│ +``` +You run the pterrafile command from /dev/apps/ folder and example.git repo would be cloned into current working directory relative path modules/example thus ending up in dev/apps/modules/example +``` + +#main.tf located in apps +module "example" { +source = "modules/example/subfolder1" +} + +#Terrafile located in DEV +example: +source: "https://github.com/joeblogs/example.git" +version: "master" +``` + +## Local installation, useful for Testing (python 3) +```shell +git clone +cd +make clean; make install; pip install . +``` diff --git a/setup.py b/setup.py index 0c63c9f..7e0dea4 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name='terrafile', - version='0.3.3', + version='0.3.4', description='Manages external Terraform modules.', author='Raymond Butcher', author_email='ray.butcher@claranet.uk', diff --git a/terrafile/__init__.py b/terrafile/__init__.py old mode 100644 new mode 100755 index ece4d90..23ecbda --- a/terrafile/__init__.py +++ b/terrafile/__init__.py @@ -6,7 +6,6 @@ import sys import yaml - REGISTRY_BASE_URL = 'https://registry.terraform.io/v1/modules' GITHUB_DOWNLOAD_URL_RE = re.compile('https://[^/]+/repos/([^/]+)/([^/]+)/tarball/([^/]+)/.*') @@ -58,11 +57,11 @@ def get_terrafile_path(path): def read_terrafile(path): try: with open(path) as open_file: - terrafile = yaml.load(open_file) + terrafile = yaml.safe_load(open_file) if not terrafile: raise ValueError('{} is empty'.format(path)) except IOError as error: - sys.stderr.write('Error loading Terrafile: {}\n'.format(error.strerror)) + sys.stderr.write('Error loading Terrafile: path {}\n'.format(error.strerror, path)) sys.exit(1) except ValueError as error: sys.stderr.write('Error loading Terrafile: {}\n'.format(error)) @@ -90,16 +89,110 @@ def is_valid_registry_source(source): return False -def update_modules(path): +def find_used_modules(module_path): + regex = re.compile(r'"(\w+)"') + modules = [] + sources = [] + moduledict = {} + allResults = [] + exclude = "modules" + for root, dirs, files in os.walk(module_path): + for file in files: + if file.endswith('.tf'): + allResults.append(os.path.join(root, file)) + filteredResults = [filtered for filtered in allResults if not exclude in filtered ] + for file in filteredResults: + try: + modules += [re.findall('.*module\s*\"(.*)\".*',line) + for line in open(file)] + sources += [re.findall('.*source.*=.*\"(.*)\".*',line) + for line in open(file)] + except IOError as error: + sys.stderr.write('Error loading tf: {}\n'.format(error.strerror)) + sys.exit(1) + except ValueError as error: + sys.stderr.write('Error reading tf: {}\n'.format(error)) + sys.exit(1) + + #Flatten out the lists + modules = [item for sublist in modules for item in sublist] + sources = [item for sublist in sources for item in sublist] + #merge lists into dict data structure + moduledict = dict(zip(modules,sources)) + return moduledict + + +def get_repo_name_from_url(url): + last_suffix_index = url.rfind(".git") + last_slash_index = url.rfind("/",0,last_suffix_index) + if last_suffix_index < 0: + last_suffix_index = len(url) + + if last_slash_index < 0 or last_suffix_index <= last_slash_index: + raise Exception("Badly formatted url {}".format(url)) + + return url[last_slash_index + 1:last_suffix_index] + + +def get_clone_target(repository_details, module_source, name): + if 'module_path' in repository_details.keys(): + target = repository_details['module_path'] + else: + last_suffix_index = module_source.rfind(name) + target = module_source[0:last_suffix_index] + name + + return target + + +def clone_remote_git( source, target, module_path, name, version): + # add token to tthe source url if exists + if 'GITHUB_TOKEN' in os.environ: + source = self._add_github_token(source, os.getenv('GITHUB_TOKEN')) + # Delete the old directory and clone it from scratch. + print('Fetching {}/{}'.format(os.path.basename(os.path.abspath(module_path)), name)) + shutil.rmtree(target, ignore_errors=True) + output, returncode = run('git', 'clone', '--branch={}'.format(version), source, target) + if returncode != 0: + sys.stderr.write(bytes.decode(output)) + sys.exit(returncode) + + +def remove_dups(dct): + reversed_dct = {} + for key, val in dct.items(): + new_key = tuple(val["source"]) + tuple(val["version"]) + (tuple(val["module_path"]) if "module_path" in val else (None,) ) + reversed_dct[new_key] = key + result_dct = {} + for key, val in reversed_dct.items(): + result_dct[val] = dct[val] + return result_dct + + +def filter_modules(terrafile,found_modules): + for key, val in terrafile.copy().items(): + if key not in found_modules.keys(): + del terrafile[key] + + return remove_dups(terrafile) + + +def update_modules(path, optimize_downloads): terrafile_path = get_terrafile_path(path) module_path = os.path.dirname(terrafile_path) module_path_name = os.path.basename(os.path.abspath(module_path)) terrafile = read_terrafile(terrafile_path) + if optimize_downloads: + found_modules = find_used_modules(os.getcwd()) + terrafile = filter_modules(terrafile, found_modules) + for name, repository_details in sorted(terrafile.items()): target = os.path.join(module_path, name) source = repository_details['source'] + if optimize_downloads: + repo_name = get_repo_name_from_url(repository_details['source']) + target = get_clone_target(repository_details,found_modules[name],repo_name) # Support modules on the local filesystem. if source.startswith('./') or source.startswith('../') or source.startswith('/'): @@ -124,13 +217,5 @@ def update_modules(path): print('Fetched {}/{}'.format(module_path_name, name)) continue - # add token to tthe source url if exists - if 'GITHUB_TOKEN' in os.environ: - source = add_github_token(source, os.getenv('GITHUB_TOKEN')) - # Delete the old directory and clone it from scratch. - print('Fetching {}/{}'.format(module_path_name, name)) - shutil.rmtree(target, ignore_errors=True) - output, returncode = run('git', 'clone', '--branch={}'.format(version), source, target) - if returncode != 0: - sys.stderr.write(bytes.decode(output)) - sys.exit(returncode) + #standard clone of remote git repo + clone_remote_git(source, target, module_path, name, version) diff --git a/terrafile/__main__.py b/terrafile/__main__.py old mode 100644 new mode 100755 index 42826ed..331ccd6 --- a/terrafile/__main__.py +++ b/terrafile/__main__.py @@ -1,11 +1,22 @@ import os import sys +import argparse from terrafile import update_modules -if len(sys.argv) > 1: - path = sys.argv[1] +optimizedownloads = False +if len(sys.argv) < 3: + if len(sys.argv) < 2: + path = os.getcwd() + else: + path = sys.argv[1] else: - path = os.getcwd() + parser = argparse.ArgumentParser() + parser.add_argument('--terrafile', required=False, default=os.getcwd(), dest='path', help='location of Terrafile') + parser.add_argument('--optimizedownloads', required=False, default=False, dest='optimizedownloads', help='Switch on opinionated distinct module downloads') + args = parser.parse_args() + path = args.path + optimizedownloads = args.optimizedownloads -update_modules(path) + +update_modules(path, optimizedownloads)