Feat/gen2 attempt2 (#145)

* try with all the docs available

* update url parameter

* Add more perms

* try adding instance connection name

* Revert "try adding instance connection name"

This reverts commit 740ac55.

* add vpc connector?

* names

* try common vpc connector

* change cidr block

* import instance

* stack names for vpc connector

* too many characters

* add ip address connection string

* try again

* oops

* correct the connection string

* missing psycopg2-binary?

* cloudfunctions gen 2 updated???

Author: jonaraphael

cerulean_cloud/cloud_function_asa/requirements.txt

@@ -19,4 +19,5 @@ pandas
 requests
 pyproj
 google-auth
-pydantic<2.0
+pydantic<2.0
+psycopg2-binary

stack/__main__.py

@@ -23,9 +23,7 @@
 pulumi.export("database_url", database.sql_instance_url)
 pulumi.export("database_instance_name", database.instance.connection_name)
 pulumi.export("database_url_alembic", database.sql_instance_url_alembic)
-pulumi.export(
-    "scene_relevancy_url", cloud_function_scene_relevancy.fxn.https_trigger_url
-)
-pulumi.export("historical_run_url", cloud_function_historical_run.fxn.https_trigger_url)
-pulumi.export("asa_url", cloud_function_asa.fxn.https_trigger_url)
+pulumi.export("scene_relevancy_url", cloud_function_scene_relevancy.fxn.url)
+pulumi.export("historical_run_url", cloud_function_historical_run.fxn.url)
+pulumi.export("asa_url", cloud_function_asa.fxn.url)
 pulumi.export("sns_topic_subscription", sns_subscription.sentinel1_sqs_target.arn)

stack/cloud_function_asa.py

@@ -6,8 +6,16 @@
 import database
 import git
 import pulumi
-from pulumi_gcp import cloudfunctions, cloudtasks, projects, serviceaccount, storage
+from pulumi_gcp import (
+    cloudfunctionsv2,
+    cloudrun,
+    cloudtasks,
+    projects,
+    serviceaccount,
+    storage,
+)
 from utils import construct_name, pulumi_create_zip
+from vpc_connector import vpc_connector
 
 stack = pulumi.get_stack()
 # We will store the source code to the Cloud Function in a Google Cloud Storage bucket.
@@ -57,7 +65,7 @@
 
 function_name = construct_name("cf-asa")
 config_values = {
-    "DB_URL": database.sql_instance_url_with_asyncpg,
+    "DB_URL": database.sql_instance_url_with_ip_asyncpg,
     "GIT_HASH": git_sha,
     "GIT_TAG": git_tag,
 }
@@ -72,7 +80,7 @@
 archive = package.apply(lambda x: pulumi.FileAsset(x))
 
 # Create the single Cloud Storage object, which contains all of the function's
-# source code. ("main.py" and "requirements.txt".)
+# source code.
 source_archive_object = storage.BucketObject(
     construct_name("source-cf-asa"),
     name=f"handler.py-asa-{time.time():f}",
@@ -96,55 +104,70 @@
     ),
 )
 
-gfw_credentials = cloudfunctions.FunctionSecretEnvironmentVariableArgs(
-    key="GOOGLE_APPLICATION_CREDENTIALS",
-    secret=pulumi.Config("ais").require("credentials"),
-    version="latest",
-    project_id=pulumi.Config("gcp").require("project"),
-)
+# Define secret environment variables
+gfw_credentials = {
+    "key": "GOOGLE_APPLICATION_CREDENTIALS",
+    "secret": pulumi.Config("ais").require("credentials"),
+    "version": "latest",
+    "project_id": pulumi.Config("gcp").require("project"),
+}
+infra_api_key = {
+    "key": "INFRA_API_TOKEN",
+    "secret": pulumi.Config("cerulean-cloud").require("infra_keyname"),
+    "version": "latest",
+    "project_id": pulumi.Config("gcp").require("project"),
+}
+api_key = {
+    "key": "API_KEY",
+    "secret": pulumi.Config("cerulean-cloud").require("keyname"),
+    "version": "latest",
+    "project_id": pulumi.Config("gcp").require("project"),
+}
 
-infra_api_key = cloudfunctions.FunctionSecretEnvironmentVariableArgs(
-    key="INFRA_API_TOKEN",
-    secret=pulumi.Config("cerulean-cloud").require("infra_keyname"),
-    version="latest",
-    project_id=pulumi.Config("gcp").require("project"),
-)
 
-api_key = cloudfunctions.FunctionSecretEnvironmentVariableArgs(
-    key="API_KEY",
-    secret=pulumi.Config("cerulean-cloud").require("keyname"),
-    version="latest",
-    project_id=pulumi.Config("gcp").require("project"),
-)
-
-fxn = cloudfunctions.Function(
+# Create the Cloud Function (Gen2)
+fxn = cloudfunctionsv2.Function(
     function_name,
     name=function_name,
-    entry_point="main",
-    environment_variables=config_values,
-    region=pulumi.Config("gcp").require("region"),
-    runtime="python39",
-    source_archive_bucket=bucket.name,
-    source_archive_object=source_archive_object.name,
-    trigger_http=True,
-    service_account_email=cloud_function_service_account.email,
-    available_memory_mb=4096,
-    timeout=540,
-    secret_environment_variables=[
-        gfw_credentials,
-        infra_api_key,
-        api_key,
-    ],
+    location=pulumi.Config("gcp").require("region"),
+    description="Cloud Function for ASA",
+    build_config={
+        "runtime": "python39",
+        "entry_point": "main",
+        "source": {
+            "storage_source": {
+                "bucket": bucket.name,
+                "object": source_archive_object.name,
+            },
+        },
+    },
+    service_config={
+        "environment_variables": config_values,
+        "available_memory": "4096M",
+        "timeout_seconds": 540,
+        "service_account_email": cloud_function_service_account.email,
+        "secret_environment_variables": [gfw_credentials, infra_api_key, api_key],
+        "vpc_connector": vpc_connector.id,
+    },
     opts=pulumi.ResourceOptions(
         depends_on=[cloud_function_service_account_iam],
     ),
 )
 
-invoker = cloudfunctions.FunctionIamMember(
+invoker = cloudfunctionsv2.FunctionIamMember(
     construct_name("cf-asa-invoker"),
     project=fxn.project,
-    region=fxn.region,
+    location=fxn.location,
     cloud_function=fxn.name,
     role="roles/cloudfunctions.invoker",
     member="allUsers",
 )
+
+cloud_run_invoker = cloudrun.IamMember(
+    "cf-asa-run-invoker",
+    project=fxn.project,
+    location=fxn.location,
+    service=fxn.name,
+    role="roles/run.invoker",
+    member="allUsers",
+)

stack/cloud_function_historical_run.py

@@ -6,14 +6,19 @@
 import cloud_run_orchestrator
 import database
 import pulumi
-from pulumi_gcp import cloudfunctions, storage
+from pulumi_gcp import (
+    cloudfunctionsv2,
+    cloudrun,
+    storage,
+)
 from utils import construct_name, pulumi_create_zip
+from vpc_connector import vpc_connector
 
 stack = pulumi.get_stack()
 
 function_name = construct_name("cf-historical-run")
 config_values = {
-    "DB_URL": database.sql_instance_url,
+    "DB_URL": database.sql_instance_url_with_ip,
     "GCPPROJECT": pulumi.Config("gcp").require("project"),
     "GCPREGION": pulumi.Config("gcp").require("region"),
     "QUEUE": cloud_function_scene_relevancy.queue.name,
@@ -25,53 +30,71 @@
 }
 
 # The Cloud Function source code itself needs to be zipped up into an
-# archive, which we create using the pulumi.AssetArchive primitive.
+# archive.
 PATH_TO_SOURCE_CODE = "../cerulean_cloud/cloud_function_historical_run"
 package = pulumi_create_zip(
     dir_to_zip=PATH_TO_SOURCE_CODE,
     zip_filepath="../cloud_function_historical_run.zip",
 )
 archive = package.apply(lambda x: pulumi.FileAsset(x))
 
-# Create the single Cloud Storage object, which contains all of the function's
-# source code. ("main.py" and "requirements.txt".)
+# Create the Cloud Storage object containing the function's source code.
 source_archive_object = storage.BucketObject(
     construct_name("source-cf-historical-run"),
     name=f"handler.py-hr-{time.time():f}",
     bucket=cloud_function_scene_relevancy.bucket.name,
     source=archive,
 )
 
-apikey = cloudfunctions.FunctionSecretEnvironmentVariableArgs(
-    key="API_KEY",
-    secret=pulumi.Config("cerulean-cloud").require("keyname"),
-    version="latest",
-    project_id=pulumi.Config("gcp").require("project"),
-)
+apikey = {
+    "key": "API_KEY",
+    "secret": pulumi.Config("cerulean-cloud").require("keyname"),
+    "version": "latest",
+    "project_id": pulumi.Config("gcp").require("project"),
+}
 
-fxn = cloudfunctions.Function(
+# Create the Cloud Function (Gen2)
+fxn = cloudfunctionsv2.Function(
     function_name,
     name=function_name,
-    entry_point="main",
-    environment_variables=config_values,
-    region=pulumi.Config("gcp").require("region"),
-    runtime="python39",
-    source_archive_bucket=cloud_function_scene_relevancy.bucket.name,
-    source_archive_object=source_archive_object.name,
-    trigger_http=True,
-    service_account_email=cloud_function_scene_relevancy.cloud_function_service_account.email,
-    timeout=500,
-    secret_environment_variables=[apikey],
+    location=pulumi.Config("gcp").require("region"),
+    description="Cloud Function for Historical Run",
+    build_config={
+        "runtime": "python39",
+        "entry_point": "main",
+        "source": {
+            "storage_source": {
+                "bucket": cloud_function_scene_relevancy.bucket.name,
+                "object": source_archive_object.name,
+            },
+        },
+    },
+    service_config={
+        "environment_variables": config_values,
+        "timeout_seconds": 500,
+        "service_account_email": cloud_function_scene_relevancy.cloud_function_service_account.email,
+        "secret_environment_variables": [apikey],
+        "vpc_connector": vpc_connector.id,
+    },
     opts=pulumi.ResourceOptions(
         depends_on=[cloud_function_scene_relevancy.cloud_function_service_account_iam],
     ),
 )
 
-invoker = cloudfunctions.FunctionIamMember(
+invoker = cloudfunctionsv2.FunctionIamMember(
     construct_name("cf-historical-run-invoker"),
     project=fxn.project,
-    region=fxn.region,
+    location=fxn.location,
     cloud_function=fxn.name,
     role="roles/cloudfunctions.invoker",
     member="allUsers",
 )
+
+cloud_run_invoker = cloudrun.IamMember(
+    "cf-historical-run-run-invoker",
+    project=fxn.project,
+    location=fxn.location,
+    service=fxn.name,
+    role="roles/run.invoker",
+    member="allUsers",
+)