From 42b7be0bc9c9482d5ccfe0301216070ebe6bfee4 Mon Sep 17 00:00:00 2001 From: Mike O Date: Fri, 2 Nov 2018 12:13:09 -0400 Subject: [PATCH 1/3] Added Content-Type header to ES REST API calls. As of 6.0 Content-Type is required. --- aws_config_to_es/elastic.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/aws_config_to_es/elastic.py b/aws_config_to_es/elastic.py index 0ea8346..0a51f8b 100644 --- a/aws_config_to_es/elastic.py +++ b/aws_config_to_es/elastic.py @@ -27,6 +27,9 @@ def add( Returns the id of the newly inserted value or None if the added date is not there, then I'm adding it in """ + + headers = {'content-type': 'application/json'} + if not isinstance(json_message, dict): json_message_dict = json.loads(json_message) else: @@ -42,11 +45,11 @@ def add( response = requests.put(self.connections + "/" + index_name + "/" + doc_type + "/" + - index_id, data=json_message) + index_id, data=json_message, headers=headers) else: response = requests.post(self.connections + "/" + index_name + "/" + - doc_type, data=json_message) + doc_type, data=json_message, headers=headers) self.log.info( "response: " + str( @@ -69,6 +72,8 @@ def set_not_analyzed_template(self): searched by the value itself """ + headers = {'content-type': 'application/json'} + payload = { "template": "*", "settings": { @@ -98,4 +103,4 @@ def set_not_analyzed_template(self): } requests.put( self.connections + "/_template/configservice", - data=json.dumps(payload)) + data=json.dumps(payload), headers=headers) From 9dee0680a467480cbfab34860a57118118b7eb80 Mon Sep 17 00:00:00 2001 From: Mike O Date: Fri, 2 Nov 2018 12:14:28 -0400 Subject: [PATCH 2/3] Tested with Python 3.6 Added ability to specify a snapshotid and region without executing a snapshot delivery. This avoids throttling when testing. Changed REGIONS dict from a static definition to using the boto3 ec2 API to get a list of all regions. --- aws_config_to_es/esingest.py | 47 +++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/aws_config_to_es/esingest.py b/aws_config_to_es/esingest.py index 3dc07c7..7568876 100755 --- a/aws_config_to_es/esingest.py +++ b/aws_config_to_es/esingest.py @@ -17,11 +17,21 @@ DOWNLOADED_SNAPSHOT_FILE_NAME = "/tmp/configsnapshot" + \ str(time.time()) + ".json.gz" -REGIONS = [ - 'us-west-1', 'us-west-2', 'eu-west-1', 'us-east-1', - 'eu-central-1', 'ap-southeast-1', 'ap-northeast-1', - 'ap-southeast-2', 'ap-northeast-2', 'sa-east-1'] +my_session = boto3.session.Session() +my_region = my_session.region_name +# If no region configured just use us-east-1 because it doesn't matter +if my_region == None: my_region = 'us-east-1' + +ec2 = boto3.client('ec2', region_name=my_region) + +response = ec2.describe_regions() + +REGIONS = [] + +for region in response['Regions']: + REGIONS.append(region['RegionName']) + # REGIONSEL[region['RegionName']] = 0 def get_configuration_snapshot_file(s3conn, bucket_name, file_partial_name): @@ -109,13 +119,16 @@ def loop_through_regions(cur_region, iso_now_time, es): "Using the following bucket name to search " "for the snapshot: " + str(bucket_name)) - verbose_log.info("Creating snapshot") - snapshotid = config_service.deliver_snapshot() - verbose_log.info("Got a new snapshot with an id of " + str(snapshotid)) - if snapshotid is None: - app_log.info( - "AWS Config isn't setup or your requests are being throttled") - return + if not args.snapshotid: + verbose_log.info("Creating snapshot") + snapshotid = config_service.deliver_snapshot() + verbose_log.info("Got a new snapshot with an id of " + str(snapshotid)) + if snapshotid is None: + app_log.info( + "AWS Config isn't setup or your requests are being throttled") + return + else: + snapshotid = args.snapshotid snapshot_file_path = get_configuration_snapshot_file( s3conn, bucket_name, snapshotid) @@ -152,7 +165,7 @@ def loop_through_regions(cur_region, iso_now_time, es): " items into ElasticSearch from " + cur_region) if could_not_add > 0: app_log.warn("Couldn't add " + str(could_not_add) + - " to ElasticSearch. Maybe you have permission issues? ") + " items to ElasticSearch. Maybe you have permission issues? ") def main(args, es): @@ -187,6 +200,10 @@ def main(args, es): parser.add_argument('--verbose', '-v', action='store_true', default=False, help='If selected, the app runs in verbose mode ' '-- a lot more logs!') + parser.add_argument('--snapshotid', '-s', + help='If specified the app will read the snapshot Id ' + 'rather than perform a snapshot. A single region must ' + 'also be specified.') args = parser.parse_args() logging.basicConfig(format=' %(name)-12s: %(message)s') @@ -219,6 +236,12 @@ def main(args, es): destination = "http://" + args.destination + if args.snapshotid and not args.region: + app_log.error( + "You must specify a region when providing a snapshot id." + ) + exit() + verbose_log.info("Setting up the elasticsearch instance") main(args, elastic.ElasticSearch(connections=destination, log=verbose_log)) From b41eb2ac211f6b28eda980f3862a52556f297be4 Mon Sep 17 00:00:00 2001 From: Mike O Date: Fri, 2 Nov 2018 13:01:32 -0400 Subject: [PATCH 3/3] Updated doc with latest changes: python 3.6 support, added --snapshotid parameter --- README.md | 108 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index deb9f57..1c96623 100644 --- a/README.md +++ b/README.md @@ -1,68 +1,106 @@ Import your AWS Config Snapshots into ElasticSearch =================================================== + ### Author: Vladimir Budilov -* [LinkedIn](https://www.linkedin.com/in/vbudilov/) -* [Medium](https://medium.com/@budilov) + +- [LinkedIn](https://www.linkedin.com/in/vbudilov/) + +- [Medium](https://medium.com/@budilov) ### What problem does this app solve? -You have a lot of resources in your AWS account and want to search and visualize them. For example, you'd like to know your EC2 Avaiability Zone distribution or how many EC2 instances are uzing a particular Security Group + +You have a lot of resources in your AWS account and want to search and visualize +them. For example, you'd like to know your EC2 Avaiability Zone distribution or +how many EC2 instances are uzing a particular Security Group ### What does this app do? -It will ingest your AWS Config Snapshots into ElasticSearch for further analysis with Kibana. Please -refer to [this blog post](https://aws.amazon.com/blogs/developer/how-to-analyze-aws-config-snapshots-with-elasticsearch-and-kibana/) -for a more in-depth explanation of this solution. +It will ingest your AWS Config Snapshots into ElasticSearch for further analysis +with Kibana. Please refer to [this blog +post](https://aws.amazon.com/blogs/developer/how-to-analyze-aws-config-snapshots-with-elasticsearch-and-kibana/) +for a more in-depth explanation of this solution. ### Getting the code -``` + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ git clone --depth 1 git@github.com:awslabs/aws-config-to-elasticsearch.git -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ### The code + #### Prerequisites -* Python 2.7 -* An ELK stack, up and running -* Install the required packages. The requirements.txt file is included with this repo. -``` + +- AWS Config configured in one or more regions + +- Python 2.7 or 3.6 + +- An ELK stack, up and running + +- Install the required packages. The requirements.txt file is included with + this repo. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pip install -r ./requirements.txt -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #### The command -```bash + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash ./esingest.py -usage: esingest.py [-h] [--region REGION] --destination DESTINATION [--verbose] +usage: esingest.py [-h] [--region REGION [--snapshotid]] --destination DESTINATION [--verbose] +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``` +1. Let's say that you have your ElasticSearch node running on localhost:9200 + and you want to import only your us-east-1 snapshot, then you'd run the + following command: -1. Let's say that you have your ElasticSearch node running on localhost:9200 and you want to import only your us-east-1 snapshot, then you'd run the following command: -```bash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash ./esingest.py -d localhost:9200 -r us-east-1 -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -2. If you want to import Snapshots from all of your AWS Config-enabled regions, run the command without the '-r' parameter: -```bash +1. If you want to import Snapshots from all of your AWS Config-enabled regions, + run the command without the '-r' parameter: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash ./esingest.py -d localhost:9200 -``` -3. To run the command in verbose mode, use the -v parameter -```bash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. To run the command in verbose mode, use the -v parameter + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash ./esingest.py -v -d localhost:9200 -r us-east-1 -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. To run the command against an existing snapshot, use the -s parameter with + the -r parameter + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +./esingest.py --region us-east-1 --snapshotid 43499c03-c911-4ace-94f8-f36f38f1ff2d --verbose -d localhost:9200 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ### Cleanup -####DON'T RUN THESE COMMANDS IF YOU DON'T WANT TO LOSE EVERYTHING IN YOUR ELASTICSEARCH NODE! +\#\#\#\#DON'T RUN THESE COMMANDS IF YOU DON'T WANT TO LOSE EVERYTHING IN YOUR +ELASTICSEARCH NODE! + +\#\#\#\#\#*THIS COMMAND WILL ERASE EVERYTHING FROM YOUR ES NODE --- BE CAREFUL +BEFORE RUNNING* -#####_THIS COMMAND WILL ERASE EVERYTHING FROM YOUR ES NODE --- BE CAREFUL BEFORE RUNNING_ -```bash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash curl -XDELETE localhost:9200/_all -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to avoid losing all of your data, you can just iterate over all of your indexes and delete them that way. The below command will print out all of your indexes that contain 'aws::'. You can then run a DELETE on just these indexes. -```bash +In order to avoid losing all of your data, you can just iterate over all of your +indexes and delete them that way. The below command will print out all of your +indexes that contain 'aws::'. You can then run a DELETE on just these indexes. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash curl 'localhost:9200/_cat/indices' | awk '{print $3}' | grep "aws::" -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Also delete the template which allows for creationg of a 'raw' string value +alongside every 'analyzed' one -Also delete the template which allows for creationg of a 'raw' string value alongside every 'analyzed' one -```bash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ bash curl -XDELETE localhost:9200/_template/configservice -``` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~