diff --git a/README.md b/README.md index a1674d1..4c49f87 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,10 @@ CloudProxy exposes an API with the IPs and credentials of the provisioned proxie ### Providers supported: * [DigitalOcean](docs/digitalocean.md) * [AWS](docs/aws.md) +* [Google Cloud](docs/gcp.md) * [Hetzner](docs/hetzner.md) ### Planned: -* Google Cloud * Azure * Scaleway * Vultr @@ -138,7 +138,7 @@ You can scale up and down your proxies and remove them for each provider via the ["Proxy <{IP}> to be destroyed"] -### Restart proxy server (AWS only) +### Restart proxy server (AWS & GCP only) #### Request `DELETE /restart` diff --git a/cloudproxy/main.py b/cloudproxy/main.py index d17a51a..b43e9b5 100644 --- a/cloudproxy/main.py +++ b/cloudproxy/main.py @@ -42,42 +42,19 @@ def main(): def get_ip_list(): ip_list = [] - if settings.config["providers"]["digitalocean"]["ips"]: - for ip in settings.config["providers"]["digitalocean"]["ips"]: - if ip not in delete_queue and ip not in restart_queue: - ip_list.append( - "http://" - + settings.config["auth"]["username"] - + ":" - + settings.config["auth"]["password"] - + "@" - + ip - + ":8899" - ) - if settings.config["providers"]["aws"]["ips"]: - for ip in settings.config["providers"]["aws"]["ips"]: - if ip not in delete_queue and ip not in restart_queue: - ip_list.append( - "http://" - + settings.config["auth"]["username"] - + ":" - + settings.config["auth"]["password"] - + "@" - + ip - + ":8899" - ) - if settings.config["providers"]["hetzner"]["ips"]: - for ip in settings.config["providers"]["hetzner"]["ips"]: - if ip not in delete_queue and ip not in restart_queue: - ip_list.append( - "http://" - + settings.config["auth"]["username"] - + ":" - + settings.config["auth"]["password"] - + "@" - + ip - + ":8899" - ) + for provider in ['digitalocean', 'aws', 'gcp', 'hetzner']: + if settings.config["providers"][provider]["ips"]: + for ip in settings.config["providers"][provider]["ips"]: + if ip not in delete_queue and ip not in restart_queue: + ip_list.append( + "http://" + + settings.config["auth"]["username"] + + ":" + + settings.config["auth"]["password"] + + "@" + + ip + + ":8899" + ) return ip_list diff --git a/cloudproxy/providers/gcp/functions.py b/cloudproxy/providers/gcp/functions.py new file mode 100644 index 0000000..e6d2c79 --- /dev/null +++ b/cloudproxy/providers/gcp/functions.py @@ -0,0 +1,113 @@ +import json +import uuid + +from loguru import logger + +import googleapiclient.discovery +from google.oauth2 import service_account + +from cloudproxy.providers.config import set_auth +from cloudproxy.providers.settings import config + +gcp = config["providers"]["gcp"] +try: + credentials = service_account.Credentials.from_service_account_info( + json.loads(gcp["secrets"]["service_account_key"]) + ) + compute = googleapiclient.discovery.build('compute', 'v1', credentials=credentials) +except TypeError: + logger.error("GCP -> Invalid service account key") + + +def create_proxy(): + image_response = compute.images().getFromFamily( + project=gcp["image_project"], + family=gcp["image_family"] + ).execute() + source_disk_image = image_response['selfLink'] + + body = { + 'name': 'cloudproxy-' + str(uuid.uuid4()), + 'machineType': + f"zones/{gcp['zone']}/machineTypes/{gcp['size']}", + 'tags': { + 'items': [ + 'cloudproxy' + ] + }, + "labels": { + 'cloudproxy': 'cloudproxy' + }, + 'disks': [ + { + 'boot': True, + 'autoDelete': True, + 'initializeParams': { + 'sourceImage': source_disk_image, + } + } + ], + 'networkInterfaces': [{ + 'network': 'global/networks/default', + 'accessConfigs': [ + { + 'name': 'External NAT', + 'type': 'ONE_TO_ONE_NAT', + 'networkTier': 'STANDARD' + } + ] + }], + 'metadata': { + 'items': [{ + 'key': 'startup-script', + 'value': set_auth(config["auth"]["username"], config["auth"]["password"]) + }] + } + } + + return compute.instances().insert( + project=gcp["project"], + zone=gcp["zone"], + body=body + ).execute() + +def delete_proxy(name): + try: + return compute.instances().delete( + project=gcp["project"], + zone=gcp["zone"], + instance=name + ).execute() + except(googleapiclient.errors.HttpError): + logger.info(f"GCP --> HTTP Error when trying to delete proxy {name}. Probably has already been deleted.") + return None + +def stop_proxy(name): + try: + return compute.instances().stop( + project=gcp["project"], + zone=gcp["zone"], + instance=name + ).execute() + except(googleapiclient.errors.HttpError): + logger.info(f"GCP --> HTTP Error when trying to stop proxy {name}. Probably has already been deleted.") + return None + +def start_proxy(name): + try: + return compute.instances().start( + project=gcp["project"], + zone=gcp["zone"], + instance=name + ).execute() + except(googleapiclient.errors.HttpError): + logger.info(f"GCP --> HTTP Error when trying to start proxy {name}. Probably has already been deleted.") + return None + +def list_instances(): + result = compute.instances().list( + project=gcp["project"], + zone=gcp["zone"], + filter='labels.cloudproxy eq cloudproxy' + ).execute() + return result['items'] if 'items' in result else [] diff --git a/cloudproxy/providers/gcp/main.py b/cloudproxy/providers/gcp/main.py new file mode 100644 index 0000000..0f778ce --- /dev/null +++ b/cloudproxy/providers/gcp/main.py @@ -0,0 +1,109 @@ +import datetime +import itertools + +from loguru import logger + +from cloudproxy.check import check_alive +from cloudproxy.providers.gcp.functions import ( + list_instances, + create_proxy, + delete_proxy, + stop_proxy, + start_proxy, +) +from cloudproxy.providers.settings import delete_queue, restart_queue, config + +def gcp_deployment(min_scaling): + total_instances = len(list_instances()) + if min_scaling < total_instances: + logger.info("Overprovisioned: GCP destroying.....") + for instance in itertools.islice( + list_instances(), 0, (total_instances - min_scaling) + ): + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{instance['name']} {access_configs['natIP']}" + delete_proxy(instance['name']) + logger.info("Destroyed: GCP -> " + msg) + if min_scaling - total_instances < 1: + logger.info("Minimum GCP instances met") + else: + total_deploy = min_scaling - total_instances + logger.info("Deploying: " + str(total_deploy) + " GCP instances") + for _ in range(total_deploy): + create_proxy() + logger.info("Deployed") + return len(list_instances()) + +def gcp_check_alive(): + ip_ready = [] + for instance in list_instances(): + try: + elapsed = datetime.datetime.now( + datetime.timezone.utc + ) - datetime.datetime.strptime(instance["creationTimestamp"], '%Y-%m-%dT%H:%M:%S.%f%z') + + if config["age_limit"] > 0 and elapsed > datetime.timedelta(seconds=config["age_limit"]): + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{instance['name']} {access_configs['natIP'] if 'natIP' in access_configs else ''}" + delete_proxy(instance['name']) + logger.info("Recycling instance, reached age limit -> " + msg) + + elif instance['status'] == "TERMINATED": + logger.info("Waking up: GCP -> Instance " + instance['name']) + started = start_proxy(instance['name']) + if not started: + logger.info("Could not wake up, trying again later.") + + elif instance['status'] == "STOPPING": + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{instance['name']} {access_configs['natIP'] if 'natIP' in access_configs else ''}" + logger.info("Stopping: GCP -> " + msg) + + elif instance['status'] == "PROVISIONING" or instance['status'] == "STAGING": + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{instance['name']} {access_configs['natIP'] if 'natIP' in access_configs else ''}" + logger.info("Provisioning: GCP -> " + msg) + + # If none of the above, check if alive or not. + elif check_alive(instance['networkInterfaces'][0]['accessConfigs'][0]['natIP']): + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{instance['name']} {access_configs['natIP']}" + logger.info("Alive: GCP -> " + msg) + ip_ready.append(access_configs['natIP']) + + else: + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + msg = f"{instance['name']} {access_configs['natIP']}" + if elapsed > datetime.timedelta(minutes=10): + delete_proxy(instance['name']) + logger.info("Destroyed: took too long GCP -> " + msg) + else: + logger.info("Waiting: GCP -> " + msg) + except (TypeError, KeyError): + logger.info("Pending: GCP -> Allocating IP") + return ip_ready + +def gcp_check_delete(): + for instance in list_instances(): + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + if 'natIP' in access_configs and access_configs['natIP'] in delete_queue: + msg = f"{instance['name']}, {access_configs['natIP']}" + delete_proxy(instance['name']) + logger.info("Destroyed: not wanted -> " + msg) + delete_queue.remove(access_configs['natIP']) + +def gcp_check_stop(): + for instance in list_instances(): + access_configs = instance['networkInterfaces'][0]['accessConfigs'][0] + if 'natIP' in access_configs and access_configs['natIP'] in restart_queue: + msg = f"{instance['name']}, {access_configs['natIP']}" + stop_proxy(instance['name']) + logger.info("Stopped: getting new IP -> " + msg) + restart_queue.remove(access_configs['natIP']) + +def gcp_start(): + gcp_check_delete() + gcp_check_stop() + gcp_deployment(config["providers"]["gcp"]["scaling"]["min_scaling"]) + ip_ready = gcp_check_alive() + return ip_ready \ No newline at end of file diff --git a/cloudproxy/providers/manager.py b/cloudproxy/providers/manager.py index 9ed41f0..855c99d 100644 --- a/cloudproxy/providers/manager.py +++ b/cloudproxy/providers/manager.py @@ -2,6 +2,7 @@ from loguru import logger from cloudproxy.providers import settings from cloudproxy.providers.aws.main import aws_start +from cloudproxy.providers.gcp.main import gcp_start from cloudproxy.providers.digitalocean.main import do_start from cloudproxy.providers.hetzner.main import hetzner_start @@ -11,12 +12,15 @@ def do_manager(): settings.config["providers"]["digitalocean"]["ips"] = [ip for ip in ip_list] return ip_list - def aws_manager(): ip_list = aws_start() settings.config["providers"]["aws"]["ips"] = [ip for ip in ip_list] return ip_list +def gcp_manager(): + ip_list = gcp_start() + settings.config["providers"]["gcp"]["ips"] = [ip for ip in ip_list] + return ip_list def hetzner_manager(): ip_list = hetzner_start() @@ -35,7 +39,11 @@ def init_schedule(): sched.add_job(aws_manager, "interval", seconds=20) else: logger.info("AWS not enabled") + if settings.config["providers"]["gcp"]["enabled"] == 'True': + sched.add_job(gcp_manager, "interval", seconds=20) + else: + logger.info("GCP not enabled") if settings.config["providers"]["hetzner"]["enabled"] == 'True': sched.add_job(hetzner_manager, "interval", seconds=20) else: - logger.info("Hetzner not enabled") \ No newline at end of file + logger.info("Hetzner not enabled") diff --git a/cloudproxy/providers/settings.py b/cloudproxy/providers/settings.py index 7900ab2..21e03b2 100644 --- a/cloudproxy/providers/settings.py +++ b/cloudproxy/providers/settings.py @@ -23,6 +23,17 @@ "secrets": {"access_key_id": "", "secret_access_key": ""}, "spot": False, }, + "gcp": { + "enabled": False, + "project": "", + "ips": [], + "scaling": {"min_scaling": 0, "max_scaling": 0}, + "size": "", + "zone": "", + "image_project": "", + "image_family": "", + "secrets": {"service_account_key": ""}, + }, "hetzner": { "enabled": False, "ips": [], @@ -82,9 +93,25 @@ config["providers"]["aws"]["size"] = os.environ.get("AWS_SIZE", "t2.micro") config["providers"]["aws"]["region"] = os.environ.get("AWS_REGION", "eu-west-2") config["providers"]["aws"]["spot"] = os.environ.get("AWS_SPOT", False) +config["providers"]["aws"]["ami"] = os.environ.get("AWS_AMI", "ami-096cb92bb3580c759") +# Set GCP Config +config["providers"]["gcp"]["enabled"] = os.environ.get("GCP_ENABLED", False) +config["providers"]["gcp"]["project"] = os.environ.get("GCP_PROJECT") +config["providers"]["gcp"]["secrets"]["service_account_key"] = os.environ.get( + "GCP_SERVICE_ACCOUNT_KEY" +) -config["providers"]["aws"]["ami"] = os.environ.get("AWS_AMI", "ami-096cb92bb3580c759") +config["providers"]["gcp"]["scaling"]["min_scaling"] = int( + os.environ.get("GCP_MIN_SCALING", 2) +) +config["providers"]["gcp"]["scaling"]["max_scaling"] = int( + os.environ.get("GCP_MAX_SCALING", 2) +) +config["providers"]["gcp"]["size"] = os.environ.get("GCP_SIZE", "f1-micro") +config["providers"]["gcp"]["zone"] = os.environ.get("GCP_REGION", "us-central1-a") +config["providers"]["gcp"]["image_project"] = os.environ.get("GCP_IMAGE_PROJECT", "ubuntu-os-cloud") +config["providers"]["gcp"]["image_family"] = os.environ.get("GCP_IMAGE_FAMILY", "ubuntu-minimal-2004-lts") # Set Hetzner config config["providers"]["hetzner"]["enabled"] = os.environ.get( diff --git a/docs/gcp.md b/docs/gcp.md new file mode 100644 index 0000000..ed8c614 --- /dev/null +++ b/docs/gcp.md @@ -0,0 +1,59 @@ +# Google Cloud Configuration + +To use Google Cloud as a provider, you’ll first need to create a Service Account. + +## Steps + +1. Login to your GCP console. Create a new Project if needed. +2. Go to Identity and Access Management (IAM & Admin). +3. On the left-hand panel, click 'Service Accounts'. +3. Click 'Create Service Account'. +4. Choose a service account name e.g. 'cloudproxy' and select Create and Continue. +5. Choose Compute Engine / Compute Admin Role, then click Continue. +6. Click Done. +7. The service account will appear in the list as service-account-name@project-id.iam.gserviceaccount.com. + +Now you have your Service Account created, but you still need to create a key for this Service Account. + +1. Click the newly created service account. +2. Click 'Keys'. +3. Choose Add Key / Create New Key. +4. Choose JSON as Key type. +5. Click Create. +6. Save the key to your local storage. + +Last, you need to create a firewall rule for the default network. + +1. On the left-hand panel, select VPC network / Firewall. +2. Click 'Create Firewall Rule'. +3. Name --> cloudproxy +4. Target tags --> cloudproxy +5. Source IP ranges --> 0.0.0.0/0 +6. Specified protocols and ports --> tcp --> 8899 +7. Leave all other options intact and click 'Create'. + +You can now use GCP as a proxy provider, below details of the environment variables. + +## Configuration options +### Environment variables: +#### Required: +``GCP_ENABLED`` - to enable GCP as a provider, set as True. Default value: False + +``GCP_PROJECT`` - GCP project ID where to create proxy instances. + +``GCP_SERVICE_ACCOUNT_KEY`` - the service account key to allow CloudProxy access to your account. Please note this is not the path to the key, but the key itself. + +The easiest method to set the ``GCP_SERVICE_ACCOUNT_KEY`` is to use a shell variable. For example, ``GCP_KEY=$(cat /path/to/service_account.json)`` and then use the new variable ``$GCP_KEY``. + +#### Optional: +``GCP_MIN_SCALING`` - this is the minimal proxies you require to be provisioned. Default value: 2 + +``GCP_MAX_SCALING`` - this is currently unused, however will be when autoscaling is implemented. We recommend you set this as the same as the minimum scaling to avoid future issues for now. Default value: 2 + +``GCP_SIZE`` - this sets the instance size, we recommend the smallest instance as the volume even a small instance can handle is high. Default value: f1-micro + +``GCP_ZONE`` - this sets the region & zone where the instance is deployed. Some websites may redirect to the language of the country your IP is from. Default value: us-central1-a + +``GCP_IMAGE_PROJECT`` - this sets the project of the image family the instance is deployed with. The default image family project is ubuntu-os-cloud. Default value: ubuntu-os-cloud + +``GCP_IMAGE_FAMILY`` - this sets the image family the instance is deployed with. The default image family is Ubuntu 20.04 LTS Minimal. Default value: ubuntu-minimal-2004-lts diff --git a/requirements.txt b/requirements.txt index 38d2327..b91bc31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ boto3==1.17.55 urllib3==1.26.5 aiofiles==0.6.0 botocore~=1.20.84 -hcloud==1.12.0 \ No newline at end of file +hcloud==1.12.0 +google-api-python-client==2.11.0 \ No newline at end of file