diff --git a/.github/workflows/deploy_pipeline.yml b/.github/workflows/deploy_pipeline.yml index 3ef0f13c..9ff592b2 100644 --- a/.github/workflows/deploy_pipeline.yml +++ b/.github/workflows/deploy_pipeline.yml @@ -21,6 +21,7 @@ jobs: # Read connection secrets SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }} + PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }} steps: # Checkout step is necessary if you want to use a config file from your repo diff --git a/steps/01_setup_snowflake.sql b/steps/01_setup_snowflake.sql index 7e1bb6a6..ffe6d7a5 100644 --- a/steps/01_setup_snowflake.sql +++ b/steps/01_setup_snowflake.sql @@ -1,5 +1,6 @@ USE ROLE ACCOUNTADMIN; + CREATE OR ALTER WAREHOUSE QUICKSTART_WH WAREHOUSE_SIZE = XSMALL AUTO_SUSPEND = 300 @@ -7,20 +8,20 @@ CREATE OR ALTER WAREHOUSE QUICKSTART_WH -- Separate database for git repository -CREATE OR ALTER DATABASE QUICKSTART_COMMON; +CREATE OR ALTER DATABASE QUICKSTART_{{environment}}; -- API integration is needed for GitHub integration CREATE OR REPLACE API INTEGRATION git_api_integration API_PROVIDER = git_https_api - API_ALLOWED_PREFIXES = ('https://github.com/') -- INSERT YOUR GITHUB USERNAME HERE + API_ALLOWED_PREFIXES = ('https://github.com/Hussainparbtani') -- INSERT YOUR GITHUB USERNAME HERE ENABLED = TRUE; -- Git repository object is similar to external stage CREATE OR REPLACE GIT REPOSITORY quickstart_common.public.quickstart_repo API_INTEGRATION = git_api_integration - ORIGIN = ''; -- INSERT URL OF FORKED REPO HERE + ORIGIN = 'https://github.com/Hussainparbtani/sfguide-getting-started-with-snowflake-devops.git'; -- INSERT URL OF FORKED REPO HERE CREATE OR ALTER DATABASE QUICKSTART_PROD; diff --git a/steps/03_harmonize_data.py b/steps/03_harmonize_data.py index 83b62d48..abeec037 100644 --- a/steps/03_harmonize_data.py +++ b/steps/03_harmonize_data.py @@ -231,7 +231,34 @@ def main(df): group by city.geo_id, city.geo_name, city.total_population """, ), - # Placeholder: Add new view definition here + View( + name="attractions", + columns=[ + ViewColumn(name="geo_id"), + ViewColumn(name="geo_name"), + ViewColumn(name="aquarium_cnt"), + ViewColumn(name="zoo_cnt"), + ViewColumn(name="korean_restaurant_cnt"), + ], + query=""" + select + city.geo_id, + city.geo_name, + count(case when category_main = 'Aquarium' THEN 1 END) aquarium_cnt, + count(case when category_main = 'Zoo' THEN 1 END) zoo_cnt, + count(case when category_main = 'Korean Restaurant' THEN 1 END) korean_restaurant_cnt, + from us_addresses__poi.cybersyn.point_of_interest_index poi + join us_addresses__poi.cybersyn.point_of_interest_addresses_relationships poi_add + on poi_add.poi_id = poi.poi_id + join us_addresses__poi.cybersyn.us_addresses address + on address.address_id = poi_add.address_id + join major_us_cities city on city.geo_id = address.id_city + where true + and category_main in ('Aquarium', 'Zoo', 'Korean Restaurant') + and id_country = 'country/USA' + group by city.geo_id, city.geo_name + """, + ), ] @@ -239,7 +266,7 @@ def main(df): root = Root(Session.builder.getOrCreate()) # create views in Snowflake -silver_schema = root.databases["quickstart_prod"].schemas["silver"] +silver_schema = silver_schema = root.databases[f"quickstart_{os.environ['environment']}"].schemas["silver"] silver_schema.user_defined_functions.create( map_city_to_airport, mode=CreateMode.or_replace ) diff --git a/steps/04_orchestrate_jobs.sql b/steps/04_orchestrate_jobs.sql index e5e1d120..094c497a 100644 --- a/steps/04_orchestrate_jobs.sql +++ b/steps/04_orchestrate_jobs.sql @@ -1,5 +1,5 @@ use role accountadmin; -use schema quickstart_prod.gold; +use schema quickstart_{{environment}}.gold; -- declarative target table of pipeline @@ -8,12 +8,18 @@ create or alter table vacation_spots ( , airport varchar , co2_emissions_kg_per_person float , punctual_pct float - , avg_temperature_air_f float + , avg_temperature_air_f float + , new_column3 string , avg_relative_humidity_pct float , avg_cloud_cover_pct float , precipitation_probability_pct float - -- STEP 5: INSERT CHANGES HERE -) data_retention_time_in_days = 1; + , aquarium_cnt int + , zoo_cnt int + , korean_restaurant_cnt int + , new_column string + , new_column2 string + , new_column3 string +) data_retention_time_in_days = {{retention_time}}; -- task to merge pipeline results into target table @@ -26,6 +32,7 @@ create or alter task vacation_spots_update from silver.flights_from_home flight join silver.weather_joined_with_major_cities city on city.geo_name = flight.arrival_city -- STEP 5: INSERT CHANGES HERE + join silver.attractions att on att.geo_name = city.geo_name ) as harmonized_vacation_spots ON vacation_spots.city = harmonized_vacation_spots.arrival_city and vacation_spots.airport = harmonized_vacation_spots.arrival_airport WHEN MATCHED THEN UPDATE SET @@ -36,6 +43,12 @@ create or alter task vacation_spots_update , vacation_spots.avg_cloud_cover_pct = harmonized_vacation_spots.avg_cloud_cover_pct , vacation_spots.precipitation_probability_pct = harmonized_vacation_spots.precipitation_probability_pct -- STEP 5: INSERT CHANGES HERE + , vacation_spots.aquarium_cnt = harmonized_vacation_spots.aquarium_cnt + , vacation_spots.zoo_cnt = harmonized_vacation_spots.zoo_cnt + , vacation_spots.korean_restaurant_cnt = harmonized_vacation_spots.korean_restaurant_cnt + , vacation_spots.new_column = '{{environment}}' + , vacation_spots.new_column2 = '{{environment}}' + , vacation_spots.new_column3 = '{{environment}}' WHEN NOT MATCHED THEN INSERT VALUES ( harmonized_vacation_spots.arrival_city @@ -47,6 +60,12 @@ create or alter task vacation_spots_update , harmonized_vacation_spots.avg_cloud_cover_pct , harmonized_vacation_spots.precipitation_probability_pct -- STEP 5: INSERT CHANGES HERE + , harmonized_vacation_spots.aquarium_cnt + , harmonized_vacation_spots.zoo_cnt + , harmonized_vacation_spots.korean_restaurant_cnt + , '{{environment}}' + , '{{environment}}' + ,'{{environment}}' ); @@ -64,13 +83,15 @@ create or alter task email_notification and punctual_pct >= 50 and avg_temperature_air_f >= 70 -- STEP 5: INSERT CHANGES HERE + and korean_restaurant_cnt > 0 + and (zoo_cnt > 0 or aquarium_cnt > 0) limit 10); if (:options = '[]') then CALL SYSTEM$SEND_EMAIL( 'email_integration', - '', -- INSERT YOUR EMAIL HERE + 'hussain.parbtani@thebridge.com', -- INSERT YOUR EMAIL HERE 'New data successfully processed: No suitable vacation spots found.', 'The query did not return any results. Consider adjusting your filters.'); end if; @@ -83,14 +104,14 @@ create or alter task email_notification CALL SYSTEM$SEND_EMAIL( 'email_integration', - '', -- INSERT YOUR EMAIL HERE + 'hussain.parbtani@thebridge.com', -- INSERT YOUR EMAIL HERE 'New data successfully processed: The perfect place for your summer vacation has been found.', :response); exception when EXPRESSION_ERROR then CALL SYSTEM$SEND_EMAIL( 'email_integration', - '', -- INSERT YOUR EMAIL HERE + '