Skip to content

Commit 867f882

Browse files
author
Patrick McCarthy
committed
Addition of opinionated Terrafile support
1 parent 8c50211 commit 867f882

File tree

4 files changed

+174
-20
lines changed

4 files changed

+174
-20
lines changed

README.md

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,14 @@ pterrafile [path]
2020

2121
If `path` is provided, it must be the path to a `Terrafile` file, or a directory containing one. If not provided, it looks for the file in the current working directory.
2222

23-
## Examples
23+
```shell
24+
pterrafile --terrafile <path>
25+
```
26+
27+
Same behavour as pterrafile [path]
28+
29+
30+
## Examples terrafile usage
2431

2532
```yaml
2633
# Terraform Registry module
@@ -42,3 +49,56 @@ terraform-aws-lambda:
4249
terraform-aws-lambda:
4350
source: "../../modules/terraform-aws-lambda"
4451
```
52+
53+
## Opinionated Terrafile
54+
55+
```shell
56+
pterrafile --terrafile <path> --optimizedownloads True
57+
```
58+
59+
if --optimizedownloads is set to True then this indicates the usage of an opinionated Terrafile. The module names are used as the key in the terrafile and pterrafile will auto-detect the module names within your terrafiles and download only the matching names in your Terrafile. This allows you to utilize a single Terrafile at the top level of an environment. This is useful when you want to easily track the versions of all your modules in a single Terrafile and allows you to call out to this central Terrafile from sub folders, and only download the specific modules you require for terraform apply.
60+
61+
## Example using opinionated Terrafile
62+
63+
```shell
64+
cd dev/apps
65+
pterrafile --terrafile ../Terrafile --optimizedownloads True
66+
```
67+
68+
```yaml
69+
├── dev
70+
│ ├── Terrafile
71+
│ ├── apps
72+
│ │ └── main.tf
73+
│ │ └── modules
74+
│ ├── ec2
75+
│ │ └── main.tf
76+
│ │ └── modules
77+
│ ├── security
78+
│ │ └── main.tf
79+
│ │ └── modules
80+
│ │── vpc
81+
│ │ └── main.tf
82+
│ │ └── modules
83+
84+
```
85+
You run the pterrafile command from /dev/apps/ folder and example.git repo would be cloned into current working directory relative path modules/example thus ending up in dev/apps/modules/example
86+
```
87+
88+
#main.tf located in apps
89+
module "example" {
90+
source = "modules/example/subfolder1"
91+
}
92+
93+
#Terrafile located in DEV
94+
example:
95+
source: "https://github.com/joeblogs/example.git"
96+
version: "master"
97+
```
98+
99+
## Local installation, useful for Testing (python 3)
100+
```shell
101+
git clone <pterrafile repo>
102+
cd <pterrafile repo>
103+
make clean; make install; pip install .
104+
```

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
setup(
66
name='terrafile',
7-
version='0.3.3',
7+
version='0.3.4',
88
description='Manages external Terraform modules.',
99
author='Raymond Butcher',
1010
author_email='[email protected]',

terrafile/__init__.py

100644100755
Lines changed: 97 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import sys
77
import yaml
88

9-
109
REGISTRY_BASE_URL = 'https://registry.terraform.io/v1/modules'
1110
GITHUB_DOWNLOAD_URL_RE = re.compile('https://[^/]+/repos/([^/]+)/([^/]+)/tarball/([^/]+)/.*')
1211

@@ -58,11 +57,11 @@ def get_terrafile_path(path):
5857
def read_terrafile(path):
5958
try:
6059
with open(path) as open_file:
61-
terrafile = yaml.load(open_file)
60+
terrafile = yaml.safe_load(open_file)
6261
if not terrafile:
6362
raise ValueError('{} is empty'.format(path))
6463
except IOError as error:
65-
sys.stderr.write('Error loading Terrafile: {}\n'.format(error.strerror))
64+
sys.stderr.write('Error loading Terrafile: path {}\n'.format(error.strerror, path))
6665
sys.exit(1)
6766
except ValueError as error:
6867
sys.stderr.write('Error loading Terrafile: {}\n'.format(error))
@@ -90,16 +89,108 @@ def is_valid_registry_source(source):
9089
return False
9190

9291

93-
def update_modules(path):
92+
def find_used_modules(module_path):
93+
regex = re.compile(r'"(\w+)"')
94+
modules = []
95+
sources = []
96+
moduledict = {}
97+
allResults = []
98+
for root, dirs, files in os.walk(module_path):
99+
for file in files:
100+
if file.endswith('.tf'):
101+
allResults.append(os.path.join(root, file))
102+
for file in allResults:
103+
try:
104+
modules += [re.findall('.*module\s*\"(.*)\".*',line)
105+
for line in open(file)]
106+
sources += [re.findall('.*source.*=.*\"(.*)\".*',line)
107+
for line in open(file)]
108+
except IOError as error:
109+
sys.stderr.write('Error loading tf: {}\n'.format(error.strerror))
110+
sys.exit(1)
111+
except ValueError as error:
112+
sys.stderr.write('Error reading tf: {}\n'.format(error))
113+
sys.exit(1)
114+
115+
#Flatten out the lists
116+
modules = [item for sublist in modules for item in sublist]
117+
sources = [item for sublist in sources for item in sublist]
118+
#merge lists into dict data structure
119+
moduledict = dict(zip(modules,sources))
120+
return moduledict
121+
122+
123+
def get_repo_name_from_url(url):
124+
last_suffix_index = url.rfind(".git")
125+
last_slash_index = url.rfind("/",0,last_suffix_index)
126+
if last_suffix_index < 0:
127+
last_suffix_index = len(url)
128+
129+
if last_slash_index < 0 or last_suffix_index <= last_slash_index:
130+
raise Exception("Badly formatted url {}".format(url))
131+
132+
return url[last_slash_index + 1:last_suffix_index]
133+
134+
135+
def get_clone_target(repository_details, module_source, name):
136+
if 'module_path' in repository_details.keys():
137+
target = repository_details['module_path']
138+
else:
139+
last_suffix_index = module_source.rfind(name)
140+
target = module_source[0:last_suffix_index] + name
141+
142+
return target
143+
144+
145+
def clone_remote_git( source, target, module_path, name, version):
146+
# add token to tthe source url if exists
147+
if 'GITHUB_TOKEN' in os.environ:
148+
source = self._add_github_token(source, os.getenv('GITHUB_TOKEN'))
149+
# Delete the old directory and clone it from scratch.
150+
print('Fetching {}/{}'.format(os.path.basename(os.path.abspath(module_path)), name))
151+
shutil.rmtree(target, ignore_errors=True)
152+
output, returncode = run('git', 'clone', '--branch={}'.format(version), source, target)
153+
if returncode != 0:
154+
sys.stderr.write(bytes.decode(output))
155+
sys.exit(returncode)
156+
157+
158+
def remove_dups(dct):
159+
reversed_dct = {}
160+
for key, val in dct.items():
161+
new_key = tuple(val["source"]) + tuple(val["version"]) + (tuple(val["module_path"]) if "module_path" in val else (None,) )
162+
reversed_dct[new_key] = key
163+
result_dct = {}
164+
for key, val in reversed_dct.items():
165+
result_dct[val] = dct[val]
166+
return result_dct
167+
168+
169+
def filter_modules(terrafile,found_modules):
170+
for key, val in terrafile.copy().items():
171+
if key not in found_modules.keys():
172+
del terrafile[key]
173+
174+
return remove_dups(terrafile)
175+
176+
177+
def update_modules(path, optimize_downloads):
94178
terrafile_path = get_terrafile_path(path)
95179
module_path = os.path.dirname(terrafile_path)
96180
module_path_name = os.path.basename(os.path.abspath(module_path))
97181

98182
terrafile = read_terrafile(terrafile_path)
183+
if optimize_downloads:
184+
found_modules = find_used_modules(os.getcwd())
185+
terrafile = filter_modules(terrafile, found_modules)
186+
99187

100188
for name, repository_details in sorted(terrafile.items()):
101189
target = os.path.join(module_path, name)
102190
source = repository_details['source']
191+
if optimize_downloads:
192+
repo_name = get_repo_name_from_url(repository_details['source'])
193+
target = get_clone_target(repository_details,found_modules[name],repo_name)
103194

104195
# Support modules on the local filesystem.
105196
if source.startswith('./') or source.startswith('../') or source.startswith('/'):
@@ -124,13 +215,5 @@ def update_modules(path):
124215
print('Fetched {}/{}'.format(module_path_name, name))
125216
continue
126217

127-
# add token to tthe source url if exists
128-
if 'GITHUB_TOKEN' in os.environ:
129-
source = add_github_token(source, os.getenv('GITHUB_TOKEN'))
130-
# Delete the old directory and clone it from scratch.
131-
print('Fetching {}/{}'.format(module_path_name, name))
132-
shutil.rmtree(target, ignore_errors=True)
133-
output, returncode = run('git', 'clone', '--branch={}'.format(version), source, target)
134-
if returncode != 0:
135-
sys.stderr.write(bytes.decode(output))
136-
sys.exit(returncode)
218+
#standard clone of remote git repo
219+
clone_remote_git(source, target, module_path, name, version)

terrafile/__main__.py

100644100755
Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,22 @@
11
import os
22
import sys
3+
import argparse
34

45
from terrafile import update_modules
56

6-
if len(sys.argv) > 1:
7-
path = sys.argv[1]
7+
optimizedownloads = False
8+
if len(sys.argv) < 3:
9+
if len(sys.argv) < 2:
10+
path = os.getcwd()
11+
else:
12+
path = sys.argv[1]
813
else:
9-
path = os.getcwd()
14+
parser = argparse.ArgumentParser()
15+
parser.add_argument('--terrafile', required=False, default=os.getcwd(), dest='path', help='location of Terrafile')
16+
parser.add_argument('--optimizedownloads', required=False, default=False, dest='optimizedownloads', help='Switch on opinionated distinct module downloads')
17+
args = parser.parse_args()
18+
path = args.path
19+
optimizedownloads = args.optimizedownloads
1020

11-
update_modules(path)
21+
22+
update_modules(path, optimizedownloads)

0 commit comments

Comments
 (0)