diff --git a/dags/config/gcs.yaml b/dags/config/gcs.yaml old mode 100644 new mode 100755 diff --git a/dags/config/gcs_dlp.yaml b/dags/config/gcs_dlp.yaml old mode 100644 new mode 100755 diff --git a/dags/config/gcs_hds_dlp.yaml b/dags/config/gcs_hds_dlp.yaml old mode 100644 new mode 100755 diff --git a/dags/config/gcs_template_source.yaml b/dags/config/gcs_template_source.yaml old mode 100644 new mode 100755 diff --git a/dags/config/gcs_template_source_incremental.yaml b/dags/config/gcs_template_source_incremental.yaml old mode 100644 new mode 100755 diff --git a/dags/parse_dags.py b/dags/parse_dags.py old mode 100644 new mode 100755 diff --git a/docker-compose.yaml b/docker-compose.yaml index 09fc87f2..b487b305 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -79,7 +79,7 @@ services: deploy: resources: reservations: - cpus: 2 + #cpus: 2 memory: 512M airflow-helper: diff --git a/docker/Dockerfile b/docker/Dockerfile index 17c746de..407ef781 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG AIRFLOW_IMAGE_NAME="2.1.1-python3.8" +ARG AIRFLOW_IMAGE_NAME="2.2.5-python3.8" FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}" diff --git a/docker/Dockerfile-ci b/docker/Dockerfile-ci index 129f0e0c..eb7e5ec9 100644 --- a/docker/Dockerfile-ci +++ b/docker/Dockerfile-ci @@ -1,5 +1,5 @@ # TODO: use multi-stage builds. This file is exactly the same as Dockerfile except the last line -ARG AIRFLOW_IMAGE_NAME="2.1.1-python3.8" +ARG AIRFLOW_IMAGE_NAME="2.2.5-python3.8" FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}" diff --git a/gcp_airflow_foundations/common/gcp/source_schema/bq.py b/gcp_airflow_foundations/common/gcp/source_schema/bq.py index d7d3af45..8b4270f1 100644 --- a/gcp_airflow_foundations/common/gcp/source_schema/bq.py +++ b/gcp_airflow_foundations/common/gcp/source_schema/bq.py @@ -1,7 +1,7 @@ import json import logging -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook def read_schema_from_bq( diff --git a/gcp_airflow_foundations/common/gcp/source_schema/gcs.py b/gcp_airflow_foundations/common/gcp/source_schema/gcs.py index 2e2f2825..f4bc3b62 100644 --- a/gcp_airflow_foundations/common/gcp/source_schema/gcs.py +++ b/gcp_airflow_foundations/common/gcp/source_schema/gcs.py @@ -1,7 +1,7 @@ import json import logging -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook from urllib.parse import urlparse diff --git a/gcp_airflow_foundations/operators/api/operators/twilio_operator.py b/gcp_airflow_foundations/operators/api/operators/twilio_operator.py index ea59320f..12004428 100644 --- a/gcp_airflow_foundations/operators/api/operators/twilio_operator.py +++ b/gcp_airflow_foundations/operators/api/operators/twilio_operator.py @@ -1,7 +1,7 @@ import json import logging -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.models.baseoperator import BaseOperator diff --git a/gcp_airflow_foundations/operators/facebook/operators/facebook_ads_to_gcs.py b/gcp_airflow_foundations/operators/facebook/operators/facebook_ads_to_gcs.py index 60ed1169..b03f7391 100644 --- a/gcp_airflow_foundations/operators/facebook/operators/facebook_ads_to_gcs.py +++ b/gcp_airflow_foundations/operators/facebook/operators/facebook_ads_to_gcs.py @@ -19,7 +19,7 @@ from gcp_airflow_foundations.enums.facebook import AccountLookupScope, ApiObject from airflow.models import BaseOperator, Variable -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from google.cloud import bigquery diff --git a/gcp_airflow_foundations/operators/gcp/bigquery/custom_bq_hook.py b/gcp_airflow_foundations/operators/gcp/bigquery/custom_bq_hook.py index bcabe21c..20d6a73f 100644 --- a/gcp_airflow_foundations/operators/gcp/bigquery/custom_bq_hook.py +++ b/gcp_airflow_foundations/operators/gcp/bigquery/custom_bq_hook.py @@ -3,7 +3,7 @@ import pyarrow.parquet as pq import pyarrow -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.utils.decorators import apply_defaults from google.cloud import bigquery diff --git a/gcp_airflow_foundations/operators/gcp/hds/hds_merge_table_operator.py b/gcp_airflow_foundations/operators/gcp/hds/hds_merge_table_operator.py index 1d8d4ecf..da5f20c8 100644 --- a/gcp_airflow_foundations/operators/gcp/hds/hds_merge_table_operator.py +++ b/gcp_airflow_foundations/operators/gcp/hds/hds_merge_table_operator.py @@ -8,7 +8,7 @@ ) from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.exceptions import AirflowException diff --git a/gcp_airflow_foundations/operators/gcp/ods/ods_merge_table_operator.py b/gcp_airflow_foundations/operators/gcp/ods/ods_merge_table_operator.py index 987e8aea..cbcd4e1b 100644 --- a/gcp_airflow_foundations/operators/gcp/ods/ods_merge_table_operator.py +++ b/gcp_airflow_foundations/operators/gcp/ods/ods_merge_table_operator.py @@ -8,7 +8,7 @@ ) from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.exceptions import AirflowException diff --git a/gcp_airflow_foundations/operators/gcp/schema_parsing/schema_parsing_operator.py b/gcp_airflow_foundations/operators/gcp/schema_parsing/schema_parsing_operator.py index d5a85125..79f281fb 100644 --- a/gcp_airflow_foundations/operators/gcp/schema_parsing/schema_parsing_operator.py +++ b/gcp_airflow_foundations/operators/gcp/schema_parsing/schema_parsing_operator.py @@ -7,7 +7,7 @@ ) from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.exceptions import AirflowException diff --git a/helpers/scripts/fernet.py b/helpers/scripts/fernet.py old mode 100644 new mode 100755 diff --git a/tests/__init__.py b/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tests/config/gcs_customer_data.yaml b/tests/config/gcs_customer_data.yaml old mode 100644 new mode 100755 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py old mode 100644 new mode 100755 diff --git a/tests/integration/config.ini b/tests/integration/config.ini old mode 100644 new mode 100755 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py old mode 100644 new mode 100755 diff --git a/tests/integration/dlp/__init__.py b/tests/integration/dlp/__init__.py old mode 100644 new mode 100755 diff --git a/tests/integration/dlp/test_dlp.py b/tests/integration/dlp/test_dlp.py old mode 100644 new mode 100755 diff --git a/tests/integration/hds/__init__.py b/tests/integration/hds/__init__.py old mode 100644 new mode 100755 diff --git a/tests/integration/hds/test_hds_upsert_scd2.py b/tests/integration/hds/test_hds_upsert_scd2.py old mode 100644 new mode 100755 diff --git a/tests/integration/hds/test_hds_upsert_snapshot.py b/tests/integration/hds/test_hds_upsert_snapshot.py old mode 100644 new mode 100755 diff --git a/tests/integration/ods/__init__.py b/tests/integration/ods/__init__.py old mode 100644 new mode 100755 diff --git a/tests/integration/ods/test_ods_upsert.py b/tests/integration/ods/test_ods_upsert.py old mode 100644 new mode 100755 diff --git a/tests/integration/operators/__init__.py b/tests/integration/operators/__init__.py old mode 100644 new mode 100755 diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini old mode 100644 new mode 100755 diff --git a/tests/integration/schema/__init__.py b/tests/integration/schema/__init__.py old mode 100644 new mode 100755 diff --git a/tests/integration/schema/config/bigquery.yaml b/tests/integration/schema/config/bigquery.yaml old mode 100644 new mode 100755 diff --git a/tests/integration/schema/test_schema_migration.py b/tests/integration/schema/test_schema_migration.py old mode 100644 new mode 100755 diff --git a/tests/integration/schema/test_schema_parsing.py b/tests/integration/schema/test_schema_parsing.py old mode 100644 new mode 100755 diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py old mode 100644 new mode 100755 diff --git a/tests/test_utils/bq_test_utils.py b/tests/test_utils/bq_test_utils.py old mode 100644 new mode 100755 index bcbcdce0..9dfe5596 --- a/tests/test_utils/bq_test_utils.py +++ b/tests/test_utils/bq_test_utils.py @@ -2,7 +2,7 @@ from google.cloud.bigquery import SchemaField import pandas from time import sleep -from airflow.contrib.hooks.bigquery_hook import BigQueryHook +from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook def insert_to_bq_from_csv(csv, project_id, dataset_id, table_id): diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py old mode 100644 new mode 100755 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/__init__.py b/tests/unit/dags/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/config/custom/invalid/custom.yaml b/tests/unit/dags/config/custom/invalid/custom.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/dags/config/custom/valid/custom.yaml b/tests/unit/dags/config/custom/valid/custom.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/dags/config/invalid/facebook_missing_location.yaml b/tests/unit/dags/config/invalid/facebook_missing_location.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/dags/config/valid/facebook.yaml b/tests/unit/dags/config/valid/facebook.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/dags/external_source.py b/tests/unit/dags/external_source.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/test_dag_builder.py b/tests/unit/dags/test_dag_builder.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/test_external_source.py b/tests/unit/dags/test_external_source.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/test_parse_dags.py b/tests/unit/dags/test_parse_dags.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/test_sample_dags.py b/tests/unit/dags/test_sample_dags.py old mode 100644 new mode 100755 diff --git a/tests/unit/dags/test_task_groups.py b/tests/unit/dags/test_task_groups.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/__init__.py b/tests/unit/operators/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/airflow/__init__.py b/tests/unit/operators/airflow/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/airflow/test_table_ingestion_sensor.py b/tests/unit/operators/airflow/test_table_ingestion_sensor.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/branch/test_cron_branch.py b/tests/unit/operators/branch/test_cron_branch.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/dlp/__init__.py b/tests/unit/operators/dlp/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/dlp/test_dlp.py b/tests/unit/operators/dlp/test_dlp.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/dlp/test_dlp_helpers.py b/tests/unit/operators/dlp/test_dlp_helpers.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/gcp/__init__.py b/tests/unit/operators/gcp/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/gcp/test_bigquery.py b/tests/unit/operators/gcp/test_bigquery.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/hds/__init__.py b/tests/unit/operators/hds/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/hds/test_merge_hds.py b/tests/unit/operators/hds/test_merge_hds.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/ods/__init__.py b/tests/unit/operators/ods/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/ods/test_merge_ods.py b/tests/unit/operators/ods/test_merge_ods.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/schema/__init__.py b/tests/unit/operators/schema/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/schema/config/gcs.yaml b/tests/unit/operators/schema/config/gcs.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/operators/schema/test_schema_migration.py b/tests/unit/operators/schema/test_schema_migration.py old mode 100644 new mode 100755 diff --git a/tests/unit/operators/schema/test_schema_parsing.py b/tests/unit/operators/schema/test_schema_parsing.py old mode 100644 new mode 100755 diff --git a/tests/unit/sources/__init__.py b/tests/unit/sources/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/sources/facebook/__init__.py b/tests/unit/sources/facebook/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/sources/facebook/facebook.yaml b/tests/unit/sources/facebook/facebook.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/sources/facebook/test_facebook.py b/tests/unit/sources/facebook/test_facebook.py old mode 100644 new mode 100755 diff --git a/tests/unit/sources/gcs/__init__.py b/tests/unit/sources/gcs/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/sources/gcs/gcs.yaml b/tests/unit/sources/gcs/gcs.yaml old mode 100644 new mode 100755 diff --git a/tests/unit/sources/gcs/test_gcs.py b/tests/unit/sources/gcs/test_gcs.py old mode 100644 new mode 100755 diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py old mode 100644 new mode 100755 diff --git a/tests/unit/utils/test_convert_config.py b/tests/unit/utils/test_convert_config.py old mode 100644 new mode 100755 diff --git a/tests/unit/utils/test_sql_helper_hds.py b/tests/unit/utils/test_sql_helper_hds.py old mode 100644 new mode 100755 diff --git a/tests/unit/utils/test_sql_helper_ods.py b/tests/unit/utils/test_sql_helper_ods.py old mode 100644 new mode 100755