Addition of opinionated Terrafile support

Author: Patrick McCarthy

README.md

@@ -20,7 +20,14 @@ pterrafile [path]
 
 If `path` is provided, it must be the path to a `Terrafile` file, or a directory containing one. If not provided, it looks for the file in the current working directory.
 
-## Examples
+```shell
+pterrafile --terrafile <path>
+```
+
+Same behavour as pterrafile [path]
+
+
+## Examples terrafile usage
 
 ```yaml
 # Terraform Registry module
@@ -42,3 +49,56 @@ terraform-aws-lambda:
 terraform-aws-lambda:
   source: "../../modules/terraform-aws-lambda"
 ```
+
+## Opinionated Terrafile
+
+```shell
+pterrafile --terrafile <path> --optimizedownloads True
+```
+
+if --optimizedownloads is set to True then this indicates the usage of an opinionated Terrafile. The module names are used as the key in the terrafile and pterrafile will auto-detect the module names within your terrafiles and download only the matching names in your Terrafile. This allows you to utilize a single Terrafile at the top level of an environment. This is useful when you want to easily track the versions of all your modules in a single Terrafile and allows you to call out to this central Terrafile from sub folders, and only download the specific modules you require for terraform apply.
+
+## Example using opinionated Terrafile
+
+```shell
+cd dev/apps
+pterrafile --terrafile ../Terrafile --optimizedownloads True
+```
+
+```yaml 
+├── dev
+│   ├── Terrafile
+│   ├── apps
+│   │   └── main.tf
+│   │   └── modules
+│   ├── ec2
+│   │   └── main.tf
+│   │   └── modules
+│   ├── security
+│   │   └── main.tf
+│   │   └── modules
+│   │── vpc
+│   │   └── main.tf
+│   │   └── modules
+│
+```
+You run the pterrafile command from /dev/apps/ folder and example.git repo would be cloned into current working directory relative path modules/example thus ending up in dev/apps/modules/example
+```
+
+#main.tf located in apps
+module "example" {
+source = "modules/example/subfolder1"
+}
+
+#Terrafile located in DEV
+example:
+source: "https://github.com/joeblogs/example.git"
+version: "master"
+```
+ 
+## Local installation, useful for Testing (python 3)
+```shell
+git clone <pterrafile repo>
+cd <pterrafile repo>
+make clean; make install; pip install .
+```

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name='terrafile',
-    version='0.3.3',
+    version='0.3.4',
     description='Manages external Terraform modules.',
     author='Raymond Butcher',
     author_email='[email protected]',

terrafile/__init__.py

@@ -6,7 +6,6 @@
 import sys
 import yaml
 
-
 REGISTRY_BASE_URL = 'https://registry.terraform.io/v1/modules'
 GITHUB_DOWNLOAD_URL_RE = re.compile('https://[^/]+/repos/([^/]+)/([^/]+)/tarball/([^/]+)/.*')
 
@@ -58,11 +57,11 @@ def get_terrafile_path(path):
 def read_terrafile(path):
     try:
         with open(path) as open_file:
-            terrafile = yaml.load(open_file)
+            terrafile = yaml.safe_load(open_file)
         if not terrafile:
             raise ValueError('{} is empty'.format(path))
     except IOError as error:
-        sys.stderr.write('Error loading Terrafile: {}\n'.format(error.strerror))
+        sys.stderr.write('Error loading Terrafile: path {}\n'.format(error.strerror, path))
         sys.exit(1)
     except ValueError as error:
         sys.stderr.write('Error loading Terrafile: {}\n'.format(error))
@@ -90,16 +89,108 @@ def is_valid_registry_source(source):
         return False
 
 
-def update_modules(path):
+def find_used_modules(module_path):
+    regex = re.compile(r'"(\w+)"')
+    modules = []
+    sources = []
+    moduledict = {}
+    allResults = []
+    for root, dirs, files in os.walk(module_path):
+      for file in files:
+        if file.endswith('.tf'):
+          allResults.append(os.path.join(root, file))
+    for file in allResults:
+        try:
+            modules += [re.findall('.*module\s*\"(.*)\".*',line)
+                for line in open(file)]
+            sources += [re.findall('.*source.*=.*\"(.*)\".*',line)
+                for line in open(file)]
+        except IOError as error:
+            sys.stderr.write('Error loading tf: {}\n'.format(error.strerror))
+            sys.exit(1)
+        except ValueError as error:
+            sys.stderr.write('Error reading tf: {}\n'.format(error))
+            sys.exit(1)
+
+    #Flatten out the lists
+    modules = [item for sublist in modules for item in sublist]
+    sources = [item for sublist in sources for item in sublist]
+    #merge lists into dict data structure
+    moduledict = dict(zip(modules,sources))
+    return moduledict
+
+
+def get_repo_name_from_url(url):
+    last_suffix_index = url.rfind(".git")
+    last_slash_index = url.rfind("/",0,last_suffix_index)
+    if last_suffix_index < 0:
+        last_suffix_index = len(url)
+
+    if last_slash_index < 0 or last_suffix_index <= last_slash_index:
+        raise Exception("Badly formatted url {}".format(url))
+
+    return url[last_slash_index + 1:last_suffix_index]
+
+
+def get_clone_target(repository_details, module_source, name):
+    if 'module_path' in repository_details.keys():
+        target = repository_details['module_path']
+    else:
+        last_suffix_index = module_source.rfind(name)
+        target =  module_source[0:last_suffix_index] + name
+
+    return target
+
+
+def clone_remote_git( source, target, module_path, name, version):
+    # add token to tthe source url if exists
+    if 'GITHUB_TOKEN' in os.environ:
+       source = self._add_github_token(source, os.getenv('GITHUB_TOKEN'))
+    # Delete the old directory and clone it from scratch.
+    print('Fetching {}/{}'.format(os.path.basename(os.path.abspath(module_path)), name))
+    shutil.rmtree(target, ignore_errors=True)
+    output, returncode = run('git', 'clone', '--branch={}'.format(version), source, target)
+    if returncode != 0:
+       sys.stderr.write(bytes.decode(output))
+       sys.exit(returncode)
+
+
+def remove_dups(dct):
+    reversed_dct = {}
+    for key, val in dct.items():
+        new_key = tuple(val["source"]) + tuple(val["version"]) + (tuple(val["module_path"]) if "module_path" in val else (None,) )
+        reversed_dct[new_key] = key
+    result_dct = {}
+    for key, val in reversed_dct.items():
+        result_dct[val] = dct[val]
+    return result_dct
+
+
+def filter_modules(terrafile,found_modules):
+    for key, val in terrafile.copy().items():
+        if key not in found_modules.keys():
+            del terrafile[key]
+
+    return remove_dups(terrafile)
+
+
+def update_modules(path, optimize_downloads):
     terrafile_path = get_terrafile_path(path)
     module_path = os.path.dirname(terrafile_path)
     module_path_name = os.path.basename(os.path.abspath(module_path))
 
     terrafile = read_terrafile(terrafile_path)
+    if optimize_downloads:                                 
+       found_modules = find_used_modules(os.getcwd())
+       terrafile = filter_modules(terrafile, found_modules)
+
 
     for name, repository_details in sorted(terrafile.items()):
         target = os.path.join(module_path, name)
         source = repository_details['source']
+        if optimize_downloads:
+            repo_name = get_repo_name_from_url(repository_details['source'])
+            target = get_clone_target(repository_details,found_modules[name],repo_name)
 
         # Support modules on the local filesystem.
         if source.startswith('./') or source.startswith('../') or source.startswith('/'):
@@ -124,13 +215,5 @@ def update_modules(path):
             print('Fetched {}/{}'.format(module_path_name, name))
             continue
 
-        # add token to tthe source url if exists
-        if 'GITHUB_TOKEN' in os.environ:
-            source = add_github_token(source, os.getenv('GITHUB_TOKEN'))
-        # Delete the old directory and clone it from scratch.
-        print('Fetching {}/{}'.format(module_path_name, name))
-        shutil.rmtree(target, ignore_errors=True)
-        output, returncode = run('git', 'clone', '--branch={}'.format(version), source, target)
-        if returncode != 0:
-            sys.stderr.write(bytes.decode(output))
-            sys.exit(returncode)
+        #standard clone of remote git repo
+        clone_remote_git(source, target, module_path, name, version)

terrafile/__main__.py

@@ -1,11 +1,22 @@
 import os
 import sys
+import argparse
 
 from terrafile import update_modules
 
-if len(sys.argv) > 1:
-    path = sys.argv[1]
+optimizedownloads = False
+if len(sys.argv) < 3:
+    if len(sys.argv) < 2:
+        path = os.getcwd()
+    else:
+        path = sys.argv[1]
 else:
-    path = os.getcwd()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--terrafile', required=False, default=os.getcwd(), dest='path', help='location of Terrafile')
+    parser.add_argument('--optimizedownloads', required=False, default=False, dest='optimizedownloads', help='Switch on opinionated distinct module downloads')
+    args = parser.parse_args()
+    path = args.path
+    optimizedownloads = args.optimizedownloads
 
-update_modules(path)
+
+update_modules(path, optimizedownloads)