From e3e614f3964c73e78fb8c7a0af9e1532c0fd7601 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Wed, 22 Jan 2025 12:17:37 +0000 Subject: [PATCH 1/2] fix(powerbi report server) Make report & chart patterns configurable --- .../source/powerbi_report_server/constants.py | 1 + .../powerbi_report_server/report_server.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py index 9f409793272dd..bdd59693b41f1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py @@ -66,6 +66,7 @@ class Constant: CORP_USER_INFO = "corpUserInfo" OWNERSHIP = "ownership" CORP_USER_KEY = "corpUserKey" + PLATFORM_NAME = "powerbi" API_ENDPOINTS = { diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py index 4764400215e12..a65c4f86bfa19 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py @@ -108,10 +108,21 @@ def host(self): class PowerBiReportServerDashboardSourceConfig(PowerBiReportServerAPIConfig): - platform_name: str = "powerbi" - platform_urn: str = builder.make_data_platform_urn(platform=platform_name) - report_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() - chart_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() + platform_name: str = pydantic.Field( + default=Constant.PLATFORM_NAME, hidden_from_docs=True + ) + platform_urn: str = pydantic.Field( + default=builder.make_data_platform_urn(platform=Constant.PLATFORM_NAME), + hidden_from_docs=True, + ) + report_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter PowerBI Reports in ingestion.", + ) + chart_pattern: AllowDenyPattern = pydantic.Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter PowerBI Charts in ingestion.", + ) def log_http_error(e: BaseException, message: str) -> Any: From 8f624008feb2c1579b5eff4975130986e9de3e70 Mon Sep 17 00:00:00 2001 From: Pedro Silva Date: Wed, 12 Feb 2025 13:16:24 +0000 Subject: [PATCH 2/2] Add logic to ignore PBiSserver reports --- .../source/powerbi_report_server/report_server.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py index a65c4f86bfa19..bd788cadc3bdd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py @@ -111,18 +111,10 @@ class PowerBiReportServerDashboardSourceConfig(PowerBiReportServerAPIConfig): platform_name: str = pydantic.Field( default=Constant.PLATFORM_NAME, hidden_from_docs=True ) - platform_urn: str = pydantic.Field( - default=builder.make_data_platform_urn(platform=Constant.PLATFORM_NAME), - hidden_from_docs=True, - ) report_pattern: AllowDenyPattern = pydantic.Field( default=AllowDenyPattern.allow_all(), description="Regex patterns to filter PowerBI Reports in ingestion.", ) - chart_pattern: AllowDenyPattern = pydantic.Field( - default=AllowDenyPattern.allow_all(), - description="Regex patterns to filter PowerBI Charts in ingestion.", - ) def log_http_error(e: BaseException, message: str) -> Any: @@ -552,6 +544,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: reports = self.powerbi_client.get_all_reports() for report in reports: + if not self.source_config.report_pattern.allowed(report.id): + self.report.report_dropped.append(f"{report.id} - {report.name}") + continue try: report.user_info = self.get_user_info(report) except pydantic.ValidationError as e: