From 6644cdde5540ab1c691d258a5baabbfc2fb8c6a1 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 18 Apr 2024 15:06:11 -0400 Subject: [PATCH 1/4] code review --- src/osbuild-manifests/aws_tag.py | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/osbuild-manifests/aws_tag.py diff --git a/src/osbuild-manifests/aws_tag.py b/src/osbuild-manifests/aws_tag.py new file mode 100644 index 0000000000..06c1e33c1d --- /dev/null +++ b/src/osbuild-manifests/aws_tag.py @@ -0,0 +1,59 @@ +import os +import subprocess +import json +import argparse + + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--stream', dest='stream', type=str, help='Fedora stream', required=True) + parser.add_argument('--arch', dest='arch', type=str, help='Architecture', default='x86_64') + args = parser.parse_args() + + builds = getBuildsForStream(args.stream, args.arch) + for build in builds: + print("The build is "+build) + buildFetch(args.stream, build, args.arch) + meta = open('builds/'+build+'/'+args.arch+'/meta.json') + data = json.load(meta) + + # Delete this when actually running. Just here while I make this script + # data ={"amis":[{ + # "name": "us-east-1", + # "hvm": "ami-0016d5df3041499f9", + # "snapshot": "snap-0c1ca4850fcd5e573" + # }]} + amis = data['amis'] + for ami in amis: + UpdateTagCmd = 'aws ec2 create-tags --resource ' + ami['hvm'] + ' --tags '+ 'Key="FedoraUser",Value="coreos"' + ' --region=' + ami['name'] + try: + subprocess.call(['/bin/bash', '-i', '-c', UpdateTagCmd]) + except subprocess.CalledProcessError as e: + return(e.output) + return + +def getBuildsForStream(stream, arch): + buildFetch = 'cosa buildfetch --stream='+ stream + ' --arch='+ arch + try: + subprocess.call(['/bin/bash', '-i', '-c', buildFetch]) + except subprocess.CalledProcessError as e: + return(e.output) + + f = open('builds/builds.json') + data = json.load(f) + builds = [] + + for i in data['builds']: + builds.append(i['id']) + return builds + +def buildFetch(stream, build, arch): + buildFetchCmd = 'cosa buildfetch --stream='+ stream + '--build=' + build + '--arch=' + arch + try: + subprocess.call(['/bin/bash', '-i', '-c', buildFetchCmd]) + except subprocess.CalledProcessError as e: + return(e.output) + +if __name__ == '__main__': + main() From 5eca0e497c569c31cb956f190e037b047de1fd32 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 19 Apr 2024 16:28:15 -0400 Subject: [PATCH 2/4] fix --- src/osbuild-manifests/aws_tag.py | 80 ++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/src/osbuild-manifests/aws_tag.py b/src/osbuild-manifests/aws_tag.py index 06c1e33c1d..9fa139811e 100644 --- a/src/osbuild-manifests/aws_tag.py +++ b/src/osbuild-manifests/aws_tag.py @@ -8,33 +8,57 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--stream', dest='stream', type=str, help='Fedora stream', required=True) - parser.add_argument('--arch', dest='arch', type=str, help='Architecture', default='x86_64') args = parser.parse_args() - builds = getBuildsForStream(args.stream, args.arch) + builds = getBuildsForStream(args.stream) for build in builds: - print("The build is "+build) - buildFetch(args.stream, build, args.arch) - meta = open('builds/'+build+'/'+args.arch+'/meta.json') - data = json.load(meta) - - # Delete this when actually running. Just here while I make this script - # data ={"amis":[{ - # "name": "us-east-1", - # "hvm": "ami-0016d5df3041499f9", - # "snapshot": "snap-0c1ca4850fcd5e573" - # }]} - amis = data['amis'] - for ami in amis: - UpdateTagCmd = 'aws ec2 create-tags --resource ' + ami['hvm'] + ' --tags '+ 'Key="FedoraUser",Value="coreos"' + ' --region=' + ami['name'] - try: - subprocess.call(['/bin/bash', '-i', '-c', UpdateTagCmd]) - except subprocess.CalledProcessError as e: - return(e.output) - return - -def getBuildsForStream(stream, arch): - buildFetch = 'cosa buildfetch --stream='+ stream + ' --arch='+ arch + build_id=build['id'] + arches=build['arches'] + for arch in arches: + print(f"The build is {build_id}") + buildFetch(args.stream, build_id, arch) + meta = open('builds/'+build_id+'/'+arch+'/meta.json') + data = json.load(meta) + + # Delete this when actually running. Just here while I make this script + data ={"amis":[{ + "name": "us-east-1", + "hvm": "ami-0016d5df3041499f9", + "snapshot": "snap-0c1ca4850fcd5e573" + }]} + amis = data['amis'] + for ami in amis: + checkAndAddTag(ami["hvm"], ami["name"]) + checkAndAddTag(ami["snapshot"], ami["name"]) + return + +def checkAndAddTag(resourceId, region): + tagExists = checkTag(resourceId) + if tagExists: + print(f"{resourceId} already tagged with FedoraUser=coreos tag") + else: + addTag(resourceId, region) + print(f"FedoraUser=coreos tag successfully added to {resourceId}") + +def checkTag(resourceId): + checkTagCmd = f'aws ec2 describe-tags --filters Name=resource-id,Values={resourceId} Name=value,Values=coreos' + try: + tagCheck=subprocess.run([checkTagCmd], shell=True, capture_output=True, text=True) + if "FedoraUser" and "coreos" in tagCheck.stdout: + return True + return False + except subprocess.CalledProcessError as e: + return(e.output) + +def addTag(resourceId, region): + UpdateTagCmd = f'aws ec2 create-tags --resource {resourceId} --tags Key="FedoraUser",Value="coreos" --region {region}' + try: + subprocess.run([UpdateTagCmd], shell=True) + except subprocess.CalledProcessError as e: + return(e.output) + +def getBuildsForStream(stream): + buildFetch = 'cosa buildfetch --stream='+ stream + ' --arch=all' try: subprocess.call(['/bin/bash', '-i', '-c', buildFetch]) except subprocess.CalledProcessError as e: @@ -42,14 +66,10 @@ def getBuildsForStream(stream, arch): f = open('builds/builds.json') data = json.load(f) - builds = [] - - for i in data['builds']: - builds.append(i['id']) - return builds + return data['builds'] def buildFetch(stream, build, arch): - buildFetchCmd = 'cosa buildfetch --stream='+ stream + '--build=' + build + '--arch=' + arch + buildFetchCmd = 'cosa buildfetch --stream='+ stream + ' --build=' + build + ' --arch=' + arch try: subprocess.call(['/bin/bash', '-i', '-c', buildFetchCmd]) except subprocess.CalledProcessError as e: From 179bfd63e2db41829be44757afce57a459f73063 Mon Sep 17 00:00:00 2001 From: gursewak1997 Date: Tue, 23 Apr 2024 11:30:13 -0700 Subject: [PATCH 3/4] Updated changes --- src/osbuild-manifests/aws_tag.py | 106 +++++++++++++++---------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/osbuild-manifests/aws_tag.py b/src/osbuild-manifests/aws_tag.py index 9fa139811e..1eab609f23 100644 --- a/src/osbuild-manifests/aws_tag.py +++ b/src/osbuild-manifests/aws_tag.py @@ -1,79 +1,79 @@ -import os import subprocess import json import argparse - - - + def main(): parser = argparse.ArgumentParser() parser.add_argument('--stream', dest='stream', type=str, help='Fedora stream', required=True) + parser.add_argument('--dry-run', dest='dry_run', help='Check if the resources have tags but not add them', action='store_true') args = parser.parse_args() - + builds = getBuildsForStream(args.stream) for build in builds: build_id=build['id'] arches=build['arches'] for arch in arches: - print(f"The build is {build_id}") + print(f"Parsing AMIs for {build_id} for {arch}") buildFetch(args.stream, build_id, arch) - meta = open('builds/'+build_id+'/'+arch+'/meta.json') + meta = open(f'builds/{build_id}/{arch}/meta.json') data = json.load(meta) - + + if 'amis' in data.keys(): + amis = data['amis'] + else: + print(f"{build_id} does not have any AMIs for {arch} in meta.json") + continue # Delete this when actually running. Just here while I make this script - data ={"amis":[{ - "name": "us-east-1", - "hvm": "ami-0016d5df3041499f9", - "snapshot": "snap-0c1ca4850fcd5e573" - }]} - amis = data['amis'] + # data ={"amis":[{ + # "name": "us-east-1", + # "hvm": "ami-0016d5df3041499f9", + # "snapshot": "snap-0c1ca4850fcd5e573" + # }]} + # amis = data['amis'] + for ami in amis: - checkAndAddTag(ami["hvm"], ami["name"]) - checkAndAddTag(ami["snapshot"], ami["name"]) - return + region = ami["name"] + checkAndAddTag(ami["hvm"], region, args.dry_run) + checkAndAddTag(ami["snapshot"], region, args.dry_run) + return + +def checkAndAddTag(resourceId, region, dry_run): + describeTagsCmd = f'aws ec2 describe-tags --filters Name=resource-id,Values={resourceId} --region {region} --output=json' + tagCheck=subprocess.run([describeTagsCmd], shell=True, capture_output=True, text=True) + if tagCheck.stdout == None or tagCheck.stdout == '': + print(f"No tags detected for {resourceId}; assuming it doesn't exist") + return + tagCheck=json.loads(tagCheck.stdout) -def checkAndAddTag(resourceId, region): - tagExists = checkTag(resourceId) - if tagExists: - print(f"{resourceId} already tagged with FedoraUser=coreos tag") + if any((tag['Key'] == 'FedoraGroup' and tag['Value'] == 'coreos') for tag in tagCheck['Tags']): + print(f"{resourceId} already tagged with FedoraGroup=coreos tag") + return else: - addTag(resourceId, region) - print(f"FedoraUser=coreos tag successfully added to {resourceId}") - -def checkTag(resourceId): - checkTagCmd = f'aws ec2 describe-tags --filters Name=resource-id,Values={resourceId} Name=value,Values=coreos' - try: - tagCheck=subprocess.run([checkTagCmd], shell=True, capture_output=True, text=True) - if "FedoraUser" and "coreos" in tagCheck.stdout: - return True - return False - except subprocess.CalledProcessError as e: - return(e.output) + if dry_run: + print(f"Would add tag 'FedoraGroup=coreos' to {resourceId} in region {region}") + return + else: + addTag(resourceId, region, dry_run) + +def addTag(resourceId, region, dry_run): + if dry_run: + print(f"Would add tag 'FedoraGroup=coreos' to {resourceId} in region {region}") + else: + UpdateTagCmd = f'aws ec2 create-tags --resource {resourceId} --tags Key="FedoraGroup",Value="coreos" --region {region}' + subprocess.run([UpdateTagCmd], shell=True) + print(f"'FedoraGroup=coreos' tag successfully added to {resourceId}") -def addTag(resourceId, region): - UpdateTagCmd = f'aws ec2 create-tags --resource {resourceId} --tags Key="FedoraUser",Value="coreos" --region {region}' - try: - subprocess.run([UpdateTagCmd], shell=True) - except subprocess.CalledProcessError as e: - return(e.output) - def getBuildsForStream(stream): - buildFetch = 'cosa buildfetch --stream='+ stream + ' --arch=all' - try: - subprocess.call(['/bin/bash', '-i', '-c', buildFetch]) - except subprocess.CalledProcessError as e: - return(e.output) - - f = open('builds/builds.json') + buildFetchCmd = 'cosa buildfetch --stream='+ stream + ' --arch=all' + subprocess.check_output(['/bin/bash', '-i', '-c', buildFetchCmd]) + + f = open(f'builds/builds.json') data = json.load(f) return data['builds'] - + def buildFetch(stream, build, arch): buildFetchCmd = 'cosa buildfetch --stream='+ stream + ' --build=' + build + ' --arch=' + arch - try: - subprocess.call(['/bin/bash', '-i', '-c', buildFetchCmd]) - except subprocess.CalledProcessError as e: - return(e.output) - + subprocess.check_output(['/bin/bash', '-i', '-c', buildFetchCmd]) + if __name__ == '__main__': main() From 4f37b1f29be2df7dc4adbc7af5722802c3f4e7de Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Sun, 28 Apr 2024 10:45:32 -0400 Subject: [PATCH 4/4] fix --- src/osbuild-manifests/aws_tag.py | 59 ++++++++++++++------------------ 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/src/osbuild-manifests/aws_tag.py b/src/osbuild-manifests/aws_tag.py index 1eab609f23..98eafd5925 100644 --- a/src/osbuild-manifests/aws_tag.py +++ b/src/osbuild-manifests/aws_tag.py @@ -1,13 +1,18 @@ +#!/usr/bin/python3 + +# Script to go through all builds and add the FedoraGroup=coreos +# tag to all AMIs and snapshots that we know about. + import subprocess import json import argparse - + def main(): parser = argparse.ArgumentParser() parser.add_argument('--stream', dest='stream', type=str, help='Fedora stream', required=True) parser.add_argument('--dry-run', dest='dry_run', help='Check if the resources have tags but not add them', action='store_true') args = parser.parse_args() - + builds = getBuildsForStream(args.stream) for build in builds: build_id=build['id'] @@ -17,63 +22,49 @@ def main(): buildFetch(args.stream, build_id, arch) meta = open(f'builds/{build_id}/{arch}/meta.json') data = json.load(meta) - + if 'amis' in data.keys(): amis = data['amis'] else: print(f"{build_id} does not have any AMIs for {arch} in meta.json") continue - # Delete this when actually running. Just here while I make this script - # data ={"amis":[{ - # "name": "us-east-1", - # "hvm": "ami-0016d5df3041499f9", - # "snapshot": "snap-0c1ca4850fcd5e573" - # }]} - # amis = data['amis'] - for ami in amis: region = ami["name"] checkAndAddTag(ami["hvm"], region, args.dry_run) checkAndAddTag(ami["snapshot"], region, args.dry_run) - return - + def checkAndAddTag(resourceId, region, dry_run): describeTagsCmd = f'aws ec2 describe-tags --filters Name=resource-id,Values={resourceId} --region {region} --output=json' - tagCheck=subprocess.run([describeTagsCmd], shell=True, capture_output=True, text=True) + tagCheck=subprocess.run(describeTagsCmd.split(' '), capture_output=True, text=True) if tagCheck.stdout == None or tagCheck.stdout == '': - print(f"No tags detected for {resourceId}; assuming it doesn't exist") + print(f"\tNo tags detected for {resourceId}; assuming it doesn't exist") return tagCheck=json.loads(tagCheck.stdout) if any((tag['Key'] == 'FedoraGroup' and tag['Value'] == 'coreos') for tag in tagCheck['Tags']): - print(f"{resourceId} already tagged with FedoraGroup=coreos tag") - return + print(f"\t{resourceId} in {region} already tagged with FedoraGroup=coreos tag") + else: + addTag(resourceId, region, dry_run) + +def addTag(resourceId, region, dry_run): + if dry_run: + print(f"\tWould add tag 'FedoraGroup=coreos' to {resourceId} in region {region}") else: - if dry_run: - print(f"Would add tag 'FedoraGroup=coreos' to {resourceId} in region {region}") - return - else: - addTag(resourceId, region, dry_run) - -def addTag(resourceId, region, dry_run): - if dry_run: - print(f"Would add tag 'FedoraGroup=coreos' to {resourceId} in region {region}") - else: UpdateTagCmd = f'aws ec2 create-tags --resource {resourceId} --tags Key="FedoraGroup",Value="coreos" --region {region}' - subprocess.run([UpdateTagCmd], shell=True) - print(f"'FedoraGroup=coreos' tag successfully added to {resourceId}") + subprocess.run(UpdateTagCmd.split(' ')) + print(f"\t'FedoraGroup=coreos' tag successfully added to {resourceId} in {region}") def getBuildsForStream(stream): buildFetchCmd = 'cosa buildfetch --stream='+ stream + ' --arch=all' - subprocess.check_output(['/bin/bash', '-i', '-c', buildFetchCmd]) - + subprocess.check_output(buildFetchCmd.split(' ')) + f = open(f'builds/builds.json') data = json.load(f) return data['builds'] - + def buildFetch(stream, build, arch): buildFetchCmd = 'cosa buildfetch --stream='+ stream + ' --build=' + build + ' --arch=' + arch - subprocess.check_output(['/bin/bash', '-i', '-c', buildFetchCmd]) - + subprocess.check_output(buildFetchCmd.split(' ')) + if __name__ == '__main__': main()