diff --git a/exec.py b/exec.py new file mode 100644 index 0000000..b00243e --- /dev/null +++ b/exec.py @@ -0,0 +1,88 @@ +import argparse +from art import text2art +import random +import boto3 +import os +import glob +from src.logger import setup_logger +from src.snapper import Snapper +from src.scanner import Scanner + + +def getting_all_pem_file_names(): + """ + :return: .pem file names from the red-detector directory. + """ + file_path = os.path.realpath(__file__) # getting the script's path + file_path = file_path.split("red-detector") + files_path = file_path[0] + "red-detector" # (the pem files arent in the same directory as the script.) + + lst = (glob.glob(files_path+"/*.pem")) + index = 0 + for i in lst: + lst[index] = lst[index].replace(files_path+"/", "").replace(".pem","") + index += 1 + return lst + + +def used_key_pairs(): + keypairs = [] # list of used keyPair names + ec2 = boto3.client('ec2') + response = ec2.describe_key_pairs() + + for i in response["KeyPairs"]: + keypairs.append(i["KeyName"]) + return keypairs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--region', action='store', dest='region', type=str, + help='region name', required=False) + parser.add_argument('--instance-id', action='store', dest='instance_id', type=str, + help='EC2 instance id', required=False) + parser.add_argument('--keypair', action='store', dest='keypair', type=str, + help='existing key pair name', required=False) + parser.add_argument('--log-level', action='store', dest='log_level', type=str, + help='log level', required=False, default="INFO") + + cmd_args = parser.parse_args() + logger = setup_logger(id=cmd_args.instance_id, log_level=cmd_args.log_level) + snapper = Snapper(logger=logger) + if cmd_args.region: + snapper.region = cmd_args.region + else: + snapper.region = snapper.select_region() + + snapper.create_client() + + if cmd_args.instance_id: + try: + source_volume_id = snapper.get_instance_root_vol(instance_id=cmd_args.instance_id) + except Exception as e: + print(e, " : (probably problem with the given instance id or internet connection)") + exit(99) + else: + source_volume_id = snapper.select_ec2_instance() + + volume_id, selected_az, snapshot_id = snapper.snapshot2volume(volume_id=source_volume_id) + + if cmd_args.keypair: + scanner = Scanner(logger=logger, region=snapper.region, key_pair_name=cmd_args.keypair) + else: + used_key_pairs_list_from_aws = used_key_pairs() + used_key_pairs_list_locally = getting_all_pem_file_names() + num = 0 + key_name = "red_detector_key{number}".format(number=str(num)) + while key_name in used_key_pairs_list_from_aws or key_name in used_key_pairs_list_locally: + num += 1 + key_name = "red_detector_key{number}".format(number=str(num)) + + scanner = Scanner(logger=logger, region=snapper.region, key_pair_name=key_name) + scanner.keypair_name = scanner.create_keypair(key_name=key_name) + + ec2_instance_id, ec2_instance_public_ip, report_service_port = scanner.create_ec2(selected_az=selected_az) + scanner.attach_volume_to_ec2(ec2_instance_id=ec2_instance_id, volume_id=volume_id) + scanner.scan_and_report(ec2_instance_public_ip=ec2_instance_public_ip, + report_service_port=report_service_port, ec2_instance_id=ec2_instance_id, + snapshot_id=snapshot_id) diff --git a/main.py b/main.py index 1c3070e..b2dfe22 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,43 @@ +import subprocess import argparse +import sys +import threading from art import text2art +import datetime +import boto3 +begin_time = datetime.datetime.now() + + +class Scan(threading.Thread): + def __init__(self, instance_region, instance_id, instance_keypair, instance_log_level): + threading.Thread.__init__(self) + self.region = instance_region + self.id = instance_id + self.keypair = instance_keypair + self.log_level = instance_log_level + + def run(self): + """ + running the exec file (old main) with "one instance at a time" (in threads of course) + """ + command = "python3 exec.py --region {region} --instance-id {id} --keypair {keypair} --log-level {loglevel}". \ + format(region=self.region, id=self.id, keypair=self.keypair, loglevel=self.log_level) + command = command.split(" ") # the command should be in this format in order to get live output + with open('test.log', 'wb') as f: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE) + for c in iter(lambda: process.stdout.readline(1), b''): + # sys.stdout.write(" [ From: " + self.instance_id + " ]" + str(c)) + pass -from src.logger import setup_logger -from src.snapper import Snapper -from src.scanner import Scanner if __name__ == "__main__": + + text_art = text2art("RED DETECTOR") + print(text_art) + print(" +++ WELCOME RED-DETECTOR - CVE SCANNER USING VULS +++\n\n") + parser = argparse.ArgumentParser() parser.add_argument('--region', action='store', dest='region', type=str, help='region name', required=False) @@ -15,35 +47,81 @@ help='existing key pair name', required=False) parser.add_argument('--log-level', action='store', dest='log_level', type=str, help='log level', required=False, default="INFO") + region = "us-east-2" + source_volume_id = "id" + keypair = "" + log_level = "INFO" - text_art = text2art("RED DETECTOR") - print(text_art) - print(" +++ WELCOME RED-DETECTOR - CVE SCANNER USING VULS +++\n\n") + """ + sample inputs for instance-id: + * ami-0fb653ca2d3203ac1 + * i-008966f80522a3c34_i-0ff28ad4240aef353 + * account_scan + * regions:us-esat-1... + """ cmd_args = parser.parse_args() - logger = setup_logger(log_level=cmd_args.log_level) - snapper = Snapper(logger=logger) if cmd_args.region: - snapper.region = cmd_args.region - else: - snapper.region = snapper.select_region() - - snapper.create_client() - + region = cmd_args.region if cmd_args.instance_id: - source_volume_id = snapper.get_instance_root_vol(instance_id=cmd_args.instance_id) - else: - source_volume_id = snapper.select_ec2_instance() + source_volume_id = cmd_args.instance_id + if cmd_args.keypair: + keypair = cmd_args.keypair + if cmd_args.log_level: + log_level = cmd_args.log_level - volume_id, selected_az, snapshot_id = snapper.snapshot2volume(volume_id=source_volume_id) + lst_of_ids = [] + ec2 = boto3.resource('ec2') - scanner = Scanner(logger=logger, region=snapper.region) - if cmd_args.keypair: - scanner.keypair_name = cmd_args.keypair + if source_volume_id == "account_scan": + ec2 = boto3.resource('ec2') + for instance in ec2.instances.all(): + if str(instance.state["Code"]) == "16": # getting just the running instances + lst_of_ids.append(instance.id) + + elif "region" in source_volume_id: # input in this form: region:us-east-2 + # source_volume_id = "regions:us-east-2,us-east-1" + source_volume_id = source_volume_id.replace("regions:", "") + try: + regions = source_volume_id.split(",") + except: + regions = source_volume_id[0] # means got one region + source_volume_id = source_volume_id.split(":") + client = boto3.client('ec2') + for region in regions: + conn = boto3.resource('ec2', region_name=region) + instances = conn.instances.filter() + for instance in instances: + if instance.state["Name"] == "running": + # without the if below: scan all regions. + if region in regions: + # print(instance.id, instance.instance_type, region) + lst_of_ids.append(instance.id) + elif "ami" in source_volume_id: + # ami = "ami-0fb653ca2d3203ac1" + ami = source_volume_id + client = boto3.client('ec2') + cl = client.describe_instances() + for data in cl['Reservations']: + for i in data["Instances"]: + if i['ImageId'] == ami: + lst_of_ids.append(i['InstanceId']) else: - scanner.keypair_name = scanner.create_keypair(key_name='red_detector_key') - ec2_instance_id, ec2_instance_public_ip, report_service_port = scanner.create_ec2(selected_az=selected_az) - scanner.attach_volume_to_ec2(ec2_instance_id=ec2_instance_id, volume_id=volume_id) - scanner.scan_and_report(ec2_instance_public_ip=ec2_instance_public_ip, - report_service_port=report_service_port, ec2_instance_id=ec2_instance_id, - snapshot_id=snapshot_id) + lst_of_ids = source_volume_id.split("_") # need to provide the ids with a _ between them. + + print("Going to scan: ", lst_of_ids) + threads = [] + for instance_id in lst_of_ids: + # print(instance_id) + instance_scan = Scan(region, instance_id, keypair, log_level) + instance_scan.start() + threads.append(instance_scan) + + for x in threads: + x.join() # wait for all the threads to end. +with open("results.txt", "r") as f: + for line in f: + print(line) +with open("results.txt", "w") as f: + pass +print("Time took to execute: ", datetime.datetime.now() - begin_time) diff --git a/src/logger.py b/src/logger.py index 4068769..ff476ec 100644 --- a/src/logger.py +++ b/src/logger.py @@ -1,11 +1,14 @@ import logging -def setup_logger(log_level="INFO"): +def setup_logger(id, log_level="INFO"): logger = logging.getLogger(__name__) log_handler = logging.StreamHandler() logger.setLevel(log_level) - log_format = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + extra = {'id': id} + log_format = logging.Formatter('%(asctime)s: [%(id)s] - %(levelname)s - %(message)s', ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) + logger = logging.LoggerAdapter(logger, extra) + logger = logging.LoggerAdapter(logger, extra) return logger diff --git a/src/remote_scripts.py b/src/remote_scripts.py index 2e66908..d2ea8dc 100644 --- a/src/remote_scripts.py +++ b/src/remote_scripts.py @@ -1,4 +1,5 @@ script_a = '''#!/bin/bash -ex + exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 apt-get update @@ -7,82 +8,64 @@ mkdir -p /home/ubuntu/vuls cd /home/ubuntu/ wget https://downloads.cisofy.com/lynis/lynis-3.0.3.tar.gz -wget ftp://ftp.pangeia.com.br/pub/seg/pac/chkrootkit.tar.gz + +apt-get install chkrootkit -y + mkdir -p chkrootkit && cd chkrootkit -tar xvf /home/ubuntu/chkrootkit.tar.gz --strip-components 1 -make sense cd /home/ubuntu/vuls -docker pull vuls/go-cve-dictionary -docker pull vuls/goval-dictionary -docker pull vuls/gost -docker pull vuls/go-exploitdb -docker pull vuls/gost -docker pull vuls/vuls +sudo docker pull vuls/go-cve-dictionary +sudo docker pull vuls/goval-dictionary +sudo docker pull vuls/gost +sudo docker pull vuls/go-exploitdb +sudo docker pull vuls/gost +sudo docker pull vuls/vuls +sudo apt install python3-pip -y +pip3 install subprocess.run +pip install subprocess.run -PWD=/home/ubuntu/vuls/ -for i in `seq 2002 $(date +"%Y")`; do \ - docker run --rm -i\ - -v $PWD:/vuls \ - -v $PWD/go-cve-dictionary-log:/var/log/vuls \ - vuls/go-cve-dictionary fetchnvd -years $i; \ - done - -docker run --rm -i \ - -v $PWD:/vuls \ - -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-redhat 5 6 7 8 -docker run --rm -i \ - -v $PWD:/vuls \ - -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-debian 7 8 9 10 - -docker run --rm -i \ +cd /home/ubuntu/vuls +sudo docker run --rm -i \ -v $PWD:/vuls \ - -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-alpine 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 - -docker run --rm -i \ + -v $PWD/go-cve-dictionary-log:/var/log/vuls \ + vuls/go-cve-dictionary fetch nvd +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-ubuntu 14 16 18 19 20 + vuls/goval-dictionary fetch redhat 5 6 7 8 -docker run --rm -i \ +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-suse -opensuse 13.2 - -docker run --rm -i \ + vuls/goval-dictionary fetch alpine 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-suse -suse-enterprise-server 12 - -docker run --rm -i \ + vuls/goval-dictionary fetch ubuntu 14 16 18 19 20 +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-oracle + vuls/goval-dictionary fetch oracle -docker run --rm -i \ +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/goval-dictionary-log:/var/log/vuls \ - vuls/goval-dictionary fetch-amazon - -docker run --rm -i \ - -v $PWD:/vuls \ - -v $PWD/gost-log:/var/log/gost \ - vuls/gost fetch redhat + vuls/goval-dictionary fetch amazon -docker run --rm -i \ +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/go-exploitdb-log:/var/log/go-exploitdb \ vuls/go-exploitdb fetch exploitdb -docker run --rm -i \ +sudo docker run --rm -i \ -v $PWD:/vuls \ -v $PWD/go-msfdb-log:/var/log/go-msfdb \ vuls/go-msfdb fetch msfdb - + + +touch config_scan.toml + cat > config_scan.toml < /tmp/tmp_authorized_keys + sudo mv /tmp/tmp_authorized_keys /vol/root/.ssh/tmp_authorized_keys -sudo chown root:root /vol/root/.ssh/tmp_authorized_keys + +sudo chown root:root /vol/root/.ssh/tmp_authorized_keys + sudo chmod 600 /vol/root/.ssh/tmp_authorized_keys sudo mount -t proc none /vol/proc @@ -141,9 +130,9 @@ sudo mount -o bind /run /vol/run sudo chroot /vol /bin/mount devpts /dev/pts -t devpts - # Reporting mkdir -p /home/ubuntu/nginx/html + cat > /home/ubuntu/nginx/default.conf < EOF + sudo docker run --name docker-nginx -p {port}:80 -d -v /home/ubuntu/nginx/html:/usr/share/nginx/html -v /home/ubuntu/nginx/default.conf:/etc/nginx/conf.d/default.conf nginx # Lynis audit -sudo cp /home/ubuntu/lynis-3.0.3.tar.gz /vol/root/ -sudo su -c "chroot /vol tar xvf /root/lynis-3.0.3.tar.gz -C /root/" -sudo su -c "chroot /vol printf 'cd /root/lynis/\n./lynis audit system\n' > /vol/root/lynis/run.sh && chmod +x /vol/root/lynis/run.sh" -sudo su -c "chroot /vol /root/lynis/run.sh" | ansi2html -l > /home/ubuntu/nginx/html/lynis_report.html + +sudo su -c "chroot /vol apt install lynis -y" +sudo su -c "chroot /vol lynis audit system" | ansi2html > /home/ubuntu/nginx/html/lynis_report.html -# Chkrootkit scan +# Chkrootkit scan cd /home/ubuntu/chkrootkit # sudo ./chkrootkit -r /vol | sed -n '/INFECTED/,/Searching/p' | head -n -1 | ansi2html -l > /home/ubuntu/nginx/html/chkrootkit_report.html -sudo ./chkrootkit -r /vol | ansi2html -l > /home/ubuntu/nginx/html/chkrootkit_report.html - +# sudo ./chkrootkit -r /vol | ansi2html -l > /home/ubuntu/nginx/html/chkrootkit_report.html +sudo touch chkrootkit_script.py +sudo chmod 777 chkrootkit_script.py + +cat > chkrootkit_script.py < ~/.ssh/config < permissions.py < jsoning.py < chkrookitjson.py <") +file = file[1].splitlines() +for i in file: + i = i.split("...") + try: + i[1] = i[1].strip() + data = {i[0]: i[1]} + if i[0] != "The following suspicious files and directories were found": + with open('rootkit.json', 'a') as outfile: + outfile.write(json.dumps(data)) + except Exception as e: + # catch the suspicious files and directories: + if "/vol" in i[0] and "
" not in i[0] and "Searching" not in i[0]:
+            suspicious_files += i[0] + " , "
+
+with open('rootkit.json', 'a') as outfile:
+    outfile.write(json.dumps({"The following suspicious files and directories were found": suspicious_files}))
+EOF
+python3 chkrookitjson.py
+
+
+
+cd
+cat > lynisjson.py <")
+file1 = file[1].split("Checking for system binaries that are required by Debian Tests...")
+
+file = file1[1].splitlines()
+for i in file:
+    if "-" in i and "--" not in i:
+        i = i.replace("", "")
+        i = i.replace("", "")
+        i = i.replace("", "")
+        i = i.replace("", "")
+        i = i.replace("", "")
+        i = i.replace("", "")
+        i = i.replace("", "")
+        try:
+            i = i.split("[")
+
+            i[1] = i[1].replace("]", "")
+            i[0] = i[0].replace("-", "")
+            i[0] = i[0].strip()
+        except Exception as e:
+            pass  # things that come here are irrelevant data
+        if len(i) == 2:
+            with open('lynis.json', 'a') as outfile:
+                outfile.write(
+                    json.dumps({i[0]: i[1]}))
+        if "Running custom tests" in i[0]:
+            break
+
+
+EOF
+python3 lynisjson.py
+"""
diff --git a/src/scanner.py b/src/scanner.py
index 108831e..9c7a8fa 100644
--- a/src/scanner.py
+++ b/src/scanner.py
@@ -1,8 +1,8 @@
 import json
 import random
 import time
-
 import boto3
+import subprocess
 import paramiko
 import requests
 from botocore.exceptions import ClientError, WaiterError
@@ -12,14 +12,16 @@
 
 
 class Scanner:
-    def __init__(self, logger, region):
+    def __init__(self, logger, region, key_pair_name):
         self.logger = logger
         self.region = region
+        self.key_pair_name = key_pair_name
         self.client = boto3.client('ec2', region_name=region)
         self.ec2 = boto3.resource('ec2', region_name=region)
         self.keypair_name = None
+        self.public_ip = ""
 
-    def create_keypair(self, key_name='red_detector_key'):
+    def create_keypair(self, key_name):
         try:
             new_keypair = self.ec2.create_key_pair(KeyName=key_name)
         except ClientError as err:
@@ -30,9 +32,10 @@ def create_keypair(self, key_name='red_detector_key'):
                     return key_name
             self.logger.error(f"create key pair: {err}")
             exit(99)
-        self.logger.info(f'creating key pair: "red_detector_key"')
-        with open('red_detector_key.pem', 'w') as f:
+        self.logger.info('creating key pair: {red_detector_key}'.format(red_detector_key=self.key_pair_name))
+        with open(self.key_pair_name+'.pem', 'w') as f:  # NEED TO OPEN A LOCAL FILE FOR "OLD" KEY PAIR TOO.
             f.write(new_keypair.key_material)
+            output = subprocess.getoutput("chmod 400 "+self.key_pair_name+'.pem')
         return key_name
 
     @staticmethod
@@ -137,7 +140,7 @@ def create_ec2(self, selected_az):
                 MinCount=1,
                 MaxCount=1,
                 InstanceType='t2.large',
-                KeyName=self.keypair_name,
+                KeyName=self.key_pair_name,
                 UserData=user_data,
                 SecurityGroupIds=[
                     security_group_id,
@@ -173,6 +176,7 @@ def create_ec2(self, selected_az):
             self.logger.error(f"describe instances: {err}")
             exit(99)
         ec2_instance_public_ip = ec2_instance_describe['Reservations'][0]['Instances'][0]['PublicIpAddress']
+        self.public_ip = ec2_instance_public_ip
         self.logger.info(f"EC2 instance: {ec2_instance_id} is running ({ec2_instance_public_ip})")
         return ec2_instance_id, ec2_instance_public_ip, report_service_port
 
@@ -208,14 +212,14 @@ def attach_volume_to_ec2(self, ec2_instance_id, volume_id):
     def scan_and_report(self, ec2_instance_public_ip, report_service_port, ec2_instance_id, snapshot_id):
         ssh = paramiko.SSHClient()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        privet_key = paramiko.RSAKey.from_private_key_file("red_detector_key.pem")
+        privet_key = paramiko.RSAKey.from_private_key_file(self.key_pair_name+".pem")
         connect = 0
         while not connect:
             try:
                 ssh.connect(hostname=ec2_instance_public_ip, username='ubuntu', pkey=privet_key)
                 connect = 1
             except Exception as err:
-                self.logger.error(f"failed connecting to EC2 instance: {err}")
+                self.logger.error(f"failed connecting to EC2 instance: {err}. Trying again...")
 
         wait_4_update = True
         c = 0
@@ -249,6 +253,8 @@ def scan_and_report(self, ec2_instance_public_ip, report_service_port, ec2_insta
                 remote_scripts.script_b.format(port=report_service_port, ip_address=ec2_instance_public_ip,
                                                instance_id=ec2_instance_id,
                                                mount_point=mount))
+
+
             stdout = stdout.readlines()
             for line in stdout:
                 self.logger.debug(line)
@@ -264,7 +270,9 @@ def scan_and_report(self, ec2_instance_public_ip, report_service_port, ec2_insta
                 stdout = stdout.readlines()
                 for line in stdout:
                     self.logger.debug(line)
-                self.logger.info(f"Check the report at: http://{ec2_instance_public_ip}:{report_service_port}")
+                with open("results.txt","a") as f:
+                    f.write(f"Check the report at: http://{ec2_instance_public_ip}:{report_service_port} \n")
+                # self.logger.info(f"Check the report at: http://{ec2_instance_public_ip}:{report_service_port}")
                 wait_4_update = False
             else:
                 c += 1
@@ -274,6 +282,14 @@ def scan_and_report(self, ec2_instance_public_ip, report_service_port, ec2_insta
             self.logger.error("generating scan report failed")
             exit(99)
 
+        stdin, stdout, stderr = ssh.exec_command(
+            remote_scripts.script_d, get_pty=True)
+
+        output = subprocess.getoutput('scp -i {ec2keypair} ubuntu@{ec2ip}:/home/ubuntu/rootkit.json . -y'
+                                      .format(ec2keypair=self.key_pair_name+".pem", ec2ip=self.public_ip))
+
+        stdout = stdout.readlines()
+        # self.logger.info("after running d:", stdout)
         ssh.close()
         self.logger.info(f"cleaning up snapshot: {snapshot_id}")
         try: