Dev

.github/workflows/deploy_pipeline.yml

@@ -21,6 +21,7 @@ jobs:
       # Read connection secrets
       SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
       SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }}
+      PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
 
     steps:
       # Checkout step is necessary if you want to use a config file from your repo

steps/01_setup_snowflake.sql

@@ -1,26 +1,27 @@
 USE ROLE ACCOUNTADMIN;
 
+
 CREATE OR ALTER WAREHOUSE QUICKSTART_WH 
   WAREHOUSE_SIZE = XSMALL 
   AUTO_SUSPEND = 300 
   AUTO_RESUME= TRUE;
 
 
 -- Separate database for git repository
-CREATE OR ALTER DATABASE QUICKSTART_COMMON;
+CREATE OR ALTER DATABASE QUICKSTART_{{environment}};
 
 
 -- API integration is needed for GitHub integration
 CREATE OR REPLACE API INTEGRATION git_api_integration
   API_PROVIDER = git_https_api
-  API_ALLOWED_PREFIXES = ('https://github.com/<insert GitHub username>') -- INSERT YOUR GITHUB USERNAME HERE
+  API_ALLOWED_PREFIXES = ('https://github.com/Hussainparbtani') -- INSERT YOUR GITHUB USERNAME HERE
   ENABLED = TRUE;
 
 
 -- Git repository object is similar to external stage
 CREATE OR REPLACE GIT REPOSITORY quickstart_common.public.quickstart_repo
   API_INTEGRATION = git_api_integration
-  ORIGIN = '<insert URL of forked GitHub repo>'; -- INSERT URL OF FORKED REPO HERE
+  ORIGIN = 'https://github.com/Hussainparbtani/sfguide-getting-started-with-snowflake-devops.git'; -- INSERT URL OF FORKED REPO HERE
 
 
 CREATE OR ALTER DATABASE QUICKSTART_PROD;

steps/03_harmonize_data.py

@@ -231,15 +231,42 @@ def main(df):
         group by city.geo_id, city.geo_name, city.total_population
         """,
     ),
-    # Placeholder: Add new view definition here
+    View(
+    name="attractions",
+    columns=[
+        ViewColumn(name="geo_id"),
+        ViewColumn(name="geo_name"),
+        ViewColumn(name="aquarium_cnt"),
+        ViewColumn(name="zoo_cnt"),
+        ViewColumn(name="korean_restaurant_cnt"),
+    ],
+    query="""
+    select
+        city.geo_id,
+        city.geo_name,
+        count(case when category_main = 'Aquarium' THEN 1 END) aquarium_cnt,
+        count(case when category_main = 'Zoo' THEN 1 END) zoo_cnt,
+        count(case when category_main = 'Korean Restaurant' THEN 1 END) korean_restaurant_cnt,
+    from us_addresses__poi.cybersyn.point_of_interest_index poi
+    join us_addresses__poi.cybersyn.point_of_interest_addresses_relationships poi_add 
+        on poi_add.poi_id = poi.poi_id
+    join us_addresses__poi.cybersyn.us_addresses address 
+        on address.address_id = poi_add.address_id
+    join major_us_cities city on city.geo_id = address.id_city
+    where true
+        and category_main in ('Aquarium', 'Zoo', 'Korean Restaurant')
+        and id_country = 'country/USA'
+    group by city.geo_id, city.geo_name
+    """,
+    ),
 ]
 
 
 # entry point for PythonAPI
 root = Root(Session.builder.getOrCreate())
 
 # create views in Snowflake
-silver_schema = root.databases["quickstart_prod"].schemas["silver"]
+silver_schema = silver_schema = root.databases[f"quickstart_{os.environ['environment']}"].schemas["silver"]
 silver_schema.user_defined_functions.create(
     map_city_to_airport, mode=CreateMode.or_replace
 )

steps/04_orchestrate_jobs.sql

@@ -1,5 +1,5 @@
 use role accountadmin;
-use schema quickstart_prod.gold;
+use schema quickstart_{{environment}}.gold;
 
 
 -- declarative target table of pipeline
@@ -8,12 +8,18 @@ create or alter table vacation_spots (
   , airport varchar
   , co2_emissions_kg_per_person float
   , punctual_pct float
-  , avg_temperature_air_f float
+  , avg_temperature_air_f float 
+  , new_column3 string
   , avg_relative_humidity_pct float
   , avg_cloud_cover_pct float
   , precipitation_probability_pct float
-  -- STEP 5: INSERT CHANGES HERE
-) data_retention_time_in_days = 1;
+  , aquarium_cnt int
+  , zoo_cnt int
+  , korean_restaurant_cnt int
+  , new_column string
+  , new_column2 string
+  , new_column3 string
+) data_retention_time_in_days = {{retention_time}};
 
 
 -- task to merge pipeline results into target table
@@ -26,6 +32,7 @@ create or alter task vacation_spots_update
     from silver.flights_from_home flight
     join silver.weather_joined_with_major_cities city on city.geo_name = flight.arrival_city
     -- STEP 5: INSERT CHANGES HERE
+    join silver.attractions att on att.geo_name = city.geo_name
   ) as harmonized_vacation_spots ON vacation_spots.city = harmonized_vacation_spots.arrival_city and vacation_spots.airport = harmonized_vacation_spots.arrival_airport
   WHEN MATCHED THEN
     UPDATE SET
@@ -36,6 +43,12 @@ create or alter task vacation_spots_update
       , vacation_spots.avg_cloud_cover_pct = harmonized_vacation_spots.avg_cloud_cover_pct
       , vacation_spots.precipitation_probability_pct = harmonized_vacation_spots.precipitation_probability_pct
       -- STEP 5: INSERT CHANGES HERE
+      , vacation_spots.aquarium_cnt = harmonized_vacation_spots.aquarium_cnt
+      , vacation_spots.zoo_cnt = harmonized_vacation_spots.zoo_cnt
+      , vacation_spots.korean_restaurant_cnt = harmonized_vacation_spots.korean_restaurant_cnt
+      , vacation_spots.new_column = '{{environment}}'
+      , vacation_spots.new_column2 = '{{environment}}'
+      , vacation_spots.new_column3 = '{{environment}}'
   WHEN NOT MATCHED THEN 
     INSERT VALUES (
         harmonized_vacation_spots.arrival_city
@@ -47,6 +60,12 @@ create or alter task vacation_spots_update
       , harmonized_vacation_spots.avg_cloud_cover_pct
       , harmonized_vacation_spots.precipitation_probability_pct
       -- STEP 5: INSERT CHANGES HERE
+      , harmonized_vacation_spots.aquarium_cnt
+      , harmonized_vacation_spots.zoo_cnt
+      , harmonized_vacation_spots.korean_restaurant_cnt
+      , '{{environment}}'
+      , '{{environment}}'
+      ,'{{environment}}'
     );
 
 
@@ -64,13 +83,15 @@ create or alter task email_notification
           and punctual_pct >= 50
           and avg_temperature_air_f >= 70
           -- STEP 5: INSERT CHANGES HERE
+          and korean_restaurant_cnt > 0
+          and (zoo_cnt > 0 or aquarium_cnt > 0)
         limit 10);
 
 
       if (:options = '[]') then
         CALL SYSTEM$SEND_EMAIL(
             'email_integration',
-            '<insert your email here>', -- INSERT YOUR EMAIL HERE
+            '[email protected]', -- INSERT YOUR EMAIL HERE
             'New data successfully processed: No suitable vacation spots found.',
             'The query did not return any results. Consider adjusting your filters.');
       end if;
@@ -83,14 +104,14 @@ create or alter task email_notification
 
       CALL SYSTEM$SEND_EMAIL(
         'email_integration',
-        '<insert your email here>', -- INSERT YOUR EMAIL HERE
+        '[email protected]', -- INSERT YOUR EMAIL HERE
         'New data successfully processed: The perfect place for your summer vacation has been found.',
         :response);
     exception
         when EXPRESSION_ERROR then
             CALL SYSTEM$SEND_EMAIL(
             'email_integration',
-            '<insert your email here>', -- INSERT YOUR EMAIL HERE
+            '<[email protected]', -- INSERT YOUR EMAIL HERE
             'New data successfully processed: Cortex LLM function inaccessible.',
             'It appears that the Cortex LLM functions are not available in your region');
     end;