diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/field_utils.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/field_utils.py new file mode 100644 index 00000000000000..11ca31ae861cbe --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/field_utils.py @@ -0,0 +1,169 @@ +import logging +from typing import Any, Dict, List, Union + +from datahub.ingestion.source.grafana.models import Panel +from datahub.metadata.schema_classes import ( + NumberTypeClass, + SchemaFieldClass, + SchemaFieldDataTypeClass, + StringTypeClass, + TimeTypeClass, +) + +logger = logging.getLogger(__name__) + + +def _deduplicate_fields(fields: List[SchemaFieldClass]) -> List[SchemaFieldClass]: + """Remove duplicate fields based on fieldPath while preserving order.""" + unique_fields = {field.fieldPath: field for field in fields} + return list(unique_fields.values()) + + +def extract_sql_column_fields(target: Dict[str, Any]) -> List[SchemaFieldClass]: + """Extract fields from SQL-style columns.""" + fields = [] + for col in target.get("sql", {}).get("columns", []): + for param in col.get("parameters", []): + if param.get("type") == "column" and param.get("name"): + field_type: Union[NumberTypeClass, StringTypeClass, TimeTypeClass] = ( + TimeTypeClass() + if col["type"] == "time" + else NumberTypeClass() + if col["type"] == "number" + else StringTypeClass() + ) + fields.append( + SchemaFieldClass( + fieldPath=param["name"], + type=SchemaFieldDataTypeClass(type=field_type), + nativeDataType=col["type"], + ) + ) + return fields + + +def extract_prometheus_fields(target: Dict[str, Any]) -> List[SchemaFieldClass]: + """Extract fields from Prometheus expressions.""" + expr = target.get("expr") + if expr: + legend = target.get("legendFormat", expr) + return [ + SchemaFieldClass( + fieldPath=legend, + type=SchemaFieldDataTypeClass(type=NumberTypeClass()), + nativeDataType="prometheus_metric", + ) + ] + return [] + + +def extract_raw_sql_fields(target: Dict[str, Any]) -> List[SchemaFieldClass]: + """Extract fields from raw SQL queries using SQL parsing.""" + raw_sql = target.get("rawSql", "").lower() + if not raw_sql: + return [] + + try: + sql = raw_sql.lower() + select_start = sql.index("select") + 6 # len("select") + from_start = sql.index("from") + select_part = sql[select_start:from_start].strip() + + columns = [col.strip().split()[-1].strip() for col in select_part.split(",")] + + return [ + ( + SchemaFieldClass( + # Capture the alias of the column if present or the name of the field + fieldPath=col.split(" as ")[-1].strip('"').strip("'"), + type=SchemaFieldDataTypeClass(type=StringTypeClass()), + nativeDataType="sql_column", + ) + ) + for col in columns + ] + except (IndexError, ValueError, StopIteration): + logger.warning(f"Failed to parse SQL {target.get('rawSql')}") + return [] + + +def extract_fields_from_panel(panel: Panel) -> List[SchemaFieldClass]: + """Extract all fields from a panel.""" + fields = [] + fields.extend(extract_fields_from_targets(panel.targets)) + fields.extend(get_fields_from_field_config(panel.field_config)) + fields.extend(get_fields_from_transformations(panel.transformations)) + return _deduplicate_fields(fields) + + +def extract_fields_from_targets( + targets: List[Dict[str, Any]], +) -> List[SchemaFieldClass]: + """Extract fields from panel targets.""" + fields = [] + for target in targets: + fields.extend(extract_sql_column_fields(target)) + fields.extend(extract_prometheus_fields(target)) + fields.extend(extract_raw_sql_fields(target)) + fields.extend(extract_time_format_fields(target)) + return fields + + +def extract_time_format_fields(target: Dict[str, Any]) -> List[SchemaFieldClass]: + """Extract fields from time series and table formats.""" + if target.get("format") in {"time_series", "table"}: + return [ + SchemaFieldClass( + fieldPath="time", + type=SchemaFieldDataTypeClass(type=TimeTypeClass()), + nativeDataType="timestamp", + ) + ] + return [] + + +def get_fields_from_field_config( + field_config: Dict[str, Any], +) -> List[SchemaFieldClass]: + """Extract fields from field configuration.""" + fields = [] + defaults = field_config.get("defaults", {}) + unit = defaults.get("unit") + if unit: + fields.append( + SchemaFieldClass( + fieldPath=f"value_{unit}", + type=SchemaFieldDataTypeClass(type=NumberTypeClass()), + nativeDataType="value", + ) + ) + for override in field_config.get("overrides", []): + if override.get("matcher", {}).get("id") == "byName": + field_name = override.get("matcher", {}).get("options") + if field_name: + fields.append( + SchemaFieldClass( + fieldPath=field_name, + type=SchemaFieldDataTypeClass(type=NumberTypeClass()), + nativeDataType="metric", + ) + ) + return fields + + +def get_fields_from_transformations( + transformations: List[Dict[str, Any]], +) -> List[SchemaFieldClass]: + """Extract fields from transformations.""" + fields = [] + for transform in transformations: + if transform.get("type") == "organize": + for field_name in transform.get("options", {}).get("indexByName", {}): + fields.append( + SchemaFieldClass( + fieldPath=field_name, + type=SchemaFieldDataTypeClass(type=StringTypeClass()), + nativeDataType="transformed", + ) + ) + return fields diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_api.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_api.py new file mode 100644 index 00000000000000..4badf7221bff58 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_api.py @@ -0,0 +1,129 @@ +"""API client for Grafana metadata extraction""" + +import logging +from typing import Dict, List, Optional, Union + +import requests +import urllib3.exceptions +from pydantic import SecretStr + +from datahub.ingestion.source.grafana.models import Dashboard, Folder +from datahub.ingestion.source.grafana.report import GrafanaSourceReport + +logger = logging.getLogger(__name__) + + +class GrafanaAPIClient: + """Client for making requests to Grafana API""" + + def __init__( + self, + base_url: str, + token: SecretStr, + verify_ssl: bool, + report: GrafanaSourceReport, + ) -> None: + self.base_url = base_url + self.verify_ssl = verify_ssl + self.report = report + self.session = self._create_session(token) + + def _create_session(self, token: SecretStr) -> requests.Session: + session = requests.Session() + session.headers.update( + { + "Authorization": f"Bearer {token.get_secret_value()}", + "Accept": "application/json", + "Content-Type": "application/json", + } + ) + session.verify = self.verify_ssl + + # If SSL verification is disabled, suppress the warnings + if not self.verify_ssl: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + self.report.warning("SSL Verification is recommended.") + + return session + + def get_folders(self) -> List[Folder]: + """Fetch all folders from Grafana with pagination.""" + folders: List[Folder] = [] + page = 1 + per_page = 100 + + while True: + try: + response = self.session.get( + f"{self.base_url}/api/folders", + params={"page": page, "limit": per_page}, + ) + response.raise_for_status() + + batch = response.json() + if not batch: + break + + folders.extend(Folder.parse_obj(folder) for folder in batch) + page += 1 + except requests.exceptions.RequestException as e: + self.report.report_failure( + message="Failed to fetch folders on page", + context=str(page), + exc=e, + ) + break + + return folders + + def get_dashboard(self, uid: str) -> Optional[Dashboard]: + """Fetch a specific dashboard by UID""" + try: + response = self.session.get(f"{self.base_url}/api/dashboards/uid/{uid}") + response.raise_for_status() + return Dashboard.parse_obj(response.json()) + except requests.exceptions.RequestException as e: + self.report.warning( + message="Failed to fetch dashboard", + context=uid, + exc=e, + ) + return None + + def get_dashboards(self) -> List[Dashboard]: + """Fetch all dashboards from search endpoint with pagination.""" + dashboards: List[Dashboard] = [] + page = 1 + per_page = 100 + + while True: + try: + params: Dict[str, Union[str, int]] = { + "type": "dash-db", + "page": page, + "limit": per_page, + } + response = self.session.get( + f"{self.base_url}/api/search", + params=params, + ) + response.raise_for_status() + + batch = response.json() + if not batch: + break + + for result in batch: + dashboard = self.get_dashboard(result["uid"]) + if dashboard: + dashboards.append(dashboard) + page += 1 + except requests.exceptions.RequestException as e: + self.report.report_failure( + message="Failed to fetch dashboards on page", + context=str(page), + exc=e, + ) + break + + return dashboards diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_config.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_config.py new file mode 100644 index 00000000000000..58e473a84e2701 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_config.py @@ -0,0 +1,59 @@ +from typing import Dict, Optional + +from pydantic import Field, SecretStr, validator + +from datahub.configuration.source_common import ( + DatasetLineageProviderConfigBase, + EnvConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StatefulStaleMetadataRemovalConfig, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) +from datahub.utilities import config_clean + + +class PlatformConnectionConfig( + EnvConfigMixin, + PlatformInstanceConfigMixin, +): + platform: str = Field(description="Upstream platform code (e.g. postgres, ms-sql)") + database: Optional[str] = Field(default=None, description="Database name") + database_schema: Optional[str] = Field(default=None, description="Schema name") + + +class GrafanaSourceConfig( + DatasetLineageProviderConfigBase, + StatefulIngestionConfigBase, + EnvConfigMixin, + PlatformInstanceConfigMixin, +): + platform: str = Field(default="grafana", hidden_from_docs=True) + url: str = Field( + description="URL of Grafana instance (e.g. https://grafana.company.com)" + ) + service_account_token: SecretStr = Field(description="Grafana API token") + verify_ssl: bool = Field( + default=True, + description="Verify SSL certificate for secure connections (https)", + ) + ingest_tags: bool = Field( + default=True, + description="Whether to ingest tags from Grafana dashboards and charts", + ) + ingest_owners: bool = Field( + default=True, + description="Whether to ingest owners from Grafana dashboards and charts", + ) + connection_to_platform_map: Dict[str, PlatformConnectionConfig] = Field( + default={}, + description="Map of Grafana connection names to their upstream platform details", + ) + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None + + @validator("url", allow_reuse=True) + def remove_trailing_slash(cls, v): + return config_clean.remove_trailing_slashes(v) diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py index 53f71046c25c0d..fa1c40ff0d7bed 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py @@ -1,131 +1,469 @@ from typing import Iterable, List, Optional -import requests -from pydantic import Field, SecretStr - -import datahub.emitter.mce_builder as builder -from datahub.configuration.source_common import PlatformInstanceConfigMixin +from datahub.emitter.mce_builder import ( + make_chart_urn, + make_container_urn, + make_data_platform_urn, + make_dataplatform_instance_urn, + make_dataset_urn_with_platform_instance, + make_schema_field_urn, + make_tag_urn, +) from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_dataset_to_container, gen_containers from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( + SourceCapability, SupportStatus, + capability, config_class, platform_name, support_status, ) from datahub.ingestion.api.source import MetadataWorkUnitProcessor -from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.grafana.field_utils import extract_fields_from_panel +from datahub.ingestion.source.grafana.grafana_api import GrafanaAPIClient +from datahub.ingestion.source.grafana.grafana_config import ( + GrafanaSourceConfig, +) +from datahub.ingestion.source.grafana.lineage import LineageExtractor +from datahub.ingestion.source.grafana.models import ( + Dashboard, + DashboardContainerKey, + Folder, + FolderKey, + Panel, +) +from datahub.ingestion.source.grafana.report import ( + GrafanaSourceReport, +) +from datahub.ingestion.source.grafana.snapshots import ( + build_chart_mce, + build_dashboard_mce, +) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, - StaleEntityRemovalSourceReport, - StatefulIngestionConfigBase, ) from datahub.ingestion.source.state.stateful_ingestion_base import ( - StatefulIngestionReport, StatefulIngestionSourceBase, ) -from datahub.metadata.com.linkedin.pegasus2avro.common import ChangeAuditStamps -from datahub.metadata.schema_classes import DashboardInfoClass, StatusClass - - -class GrafanaSourceConfig(StatefulIngestionConfigBase, PlatformInstanceConfigMixin): - url: str = Field( - default="", - description="Grafana URL in the format http://your-grafana-instance with no trailing slash", - ) - service_account_token: SecretStr = Field( - description="Service account token for Grafana" - ) - - -class GrafanaReport(StaleEntityRemovalSourceReport): - pass +from datahub.metadata.schema_classes import ( + DataPlatformInstanceClass, + DatasetPropertiesClass, + DatasetSnapshotClass, + GlobalTagsClass, + InputFieldClass, + InputFieldsClass, + MetadataChangeEventClass, + OtherSchemaClass, + SchemaFieldClass, + SchemaMetadataClass, + StatusClass, + TagAssociationClass, +) @platform_name("Grafana") @config_class(GrafanaSourceConfig) -@support_status(SupportStatus.TESTING) +@support_status(SupportStatus.CERTIFIED) +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability(SourceCapability.DELETION_DETECTION, "Enabled by default") +@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default") +@capability(SourceCapability.LINEAGE_FINE, "Enabled by default") +@capability(SourceCapability.OWNERSHIP, "Enabled by default") +@capability(SourceCapability.TAGS, "Enabled by default") class GrafanaSource(StatefulIngestionSourceBase): """ - This is an experimental source for Grafana. - Currently only ingests dashboards (no charts) + This plugin extracts metadata from Grafana and ingests it into DataHub. It connects to Grafana's API + to extract metadata about dashboards, charts, and data sources. The following types of metadata are extracted: + + - Container Entities: + - Folders: Top-level organizational units in Grafana + - Dashboards: Collections of panels and charts + - The full container hierarchy is preserved (Folders -> Dashboards -> Charts/Datasets) + + - Charts and Visualizations: + - All panel types (graphs, tables, stat panels, etc.) + - Chart configuration and properties + - Links to the original Grafana UI + - Custom properties including panel types and data source information + - Input fields and schema information when available + + - Data Sources and Datasets: + - Physical datasets representing Grafana's data sources + - Dataset schema information extracted from queries and panel configurations + - Support for various data source types (SQL, Prometheus, etc.) + - Custom properties including data source type and configuration + + - Lineage Information: + - Dataset-level lineage showing relationships between: + - Source data systems and Grafana datasets + - Grafana datasets and charts + - Column-level lineage for SQL-based data sources + - Support for external source systems through configurable platform mappings + + - Tags and Ownership: + - Dashboard and chart tags + - Ownership information derived from: + - Dashboard creators + - Technical owners based on dashboard UIDs + - Custom ownership assignments + + The source supports the following capabilities: + - Platform instance support for multi-Grafana deployments + - Stateful ingestion with support for soft-deletes + - Fine-grained lineage at both dataset and column levels + - Automated tag extraction + - Support for both HTTP and HTTPS connections with optional SSL verification + + Prerequisites: + 1. A running Grafana instance + 2. A service account token with permissions to: + - Read dashboards and folders + - Access data source configurations + - View user information + + A sample configuration file: + ```yaml + source: + type: grafana + config: + # Coordinates + platform_instance: production # optional + env: PROD # optional + url: https://grafana.company.com + service_account_token: ${GRAFANA_SERVICE_ACCOUNT_TOKEN} + + # SSL verification for HTTPS connections + verify_ssl: true # optional, default is true + + # Source type mapping for lineage + connection_to_platform_map: + postgres: + platform: postgres + database: grafana # optional + database_schema: grafana # optional + platform_instance: database_2 # optional + env: PROD # optional + mysql_uid_1: # Grafana datasource UID + platform: mysql + platform_instance: database_1 # optional + database: my_database # optional + ``` """ + config: GrafanaSourceConfig + report: GrafanaSourceReport + def __init__(self, config: GrafanaSourceConfig, ctx: PipelineContext): super().__init__(config, ctx) - self.source_config = config - self.report = GrafanaReport() - self.platform = "grafana" + self.config = config + self.ctx = ctx + self.platform = config.platform + self.platform_instance = self.config.platform_instance + self.env = self.config.env + self.report = GrafanaSourceReport() + + self.client = GrafanaAPIClient( + base_url=self.config.url, + token=self.config.service_account_token, + verify_ssl=self.config.verify_ssl, + report=self.report, + ) + + # Initialize lineage extractor with graph + self.lineage_extractor = LineageExtractor( + platform=self.config.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + connection_to_platform_map=self.config.connection_to_platform_map, + graph=self.ctx.graph, + report=self.report, + ) @classmethod - def create(cls, config_dict, ctx): + def create(cls, config_dict: dict, ctx: PipelineContext) -> "GrafanaSource": config = GrafanaSourceConfig.parse_obj(config_dict) return cls(config, ctx) def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: - return [ - *super().get_workunit_processors(), + processors = super().get_workunit_processors() + processors.append( StaleEntityRemovalHandler.create( - self, self.source_config, self.ctx - ).workunit_processor, - ] - - def get_report(self) -> StatefulIngestionReport: - return self.report + self, self.config, self.ctx + ).workunit_processor + ) + return processors def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - headers = { - "Authorization": f"Bearer {self.source_config.service_account_token.get_secret_value()}", - "Content-Type": "application/json", - } - try: - response = requests.get( - f"{self.source_config.url}/api/search", headers=headers + """Generate metadata work units""" + # Process folders + for folder in self.client.get_folders(): + self.report.report_folder_scanned() + yield from self._process_folder(folder) + + # Process dashboards and their panels + dashboards = self.client.get_dashboards() + for dashboard in dashboards: + self.report.report_dashboard_scanned() + yield from self._process_dashboard(dashboard) + + def _process_folder(self, folder: Folder) -> Iterable[MetadataWorkUnit]: + """Process Grafana folder metadata""" + folder_key = FolderKey( + platform=self.config.platform, + instance=self.config.platform_instance, + folder_id=folder.id, + ) + + yield from gen_containers( + container_key=folder_key, + name=folder.title, + sub_types=["Folder"], + description=folder.description, + ) + + def _process_dashboard(self, dashboard: Dashboard) -> Iterable[MetadataWorkUnit]: + """Process dashboard and its panels""" + chart_urns = [] + + # First create the dashboard container + dashboard_container_key = DashboardContainerKey( + platform=self.config.platform, + instance=self.config.platform_instance, + dashboard_id=dashboard.uid, + ) + + # Generate dashboard container first + yield from gen_containers( + container_key=dashboard_container_key, + name=dashboard.title, + sub_types=["Dashboard"], + description=dashboard.description, + ) + + # If dashboard is in a folder, add it to folder container + if dashboard.folder_id: + folder_key = FolderKey( + platform=self.config.platform, + instance=self.config.platform_instance, + folder_id=dashboard.folder_id, ) - response.raise_for_status() - except requests.exceptions.RequestException as e: - self.report.report_failure(f"Failed to fetch dashboards: {str(e)}") - return - res_json = response.json() - for item in res_json: - uid = item["uid"] - title = item["title"] - url_path = item["url"] - full_url = f"{self.source_config.url}{url_path}" - dashboard_urn = builder.make_dashboard_urn( - platform=self.platform, - name=uid, - platform_instance=self.source_config.platform_instance, + + yield from add_dataset_to_container( + container_key=folder_key, + dataset_urn=make_container_urn(dashboard_container_key), ) - yield from auto_workunit( - MetadataChangeProposalWrapper.construct_many( - entityUrn=dashboard_urn, - aspects=[ - DashboardInfoClass( - description="", - title=title, - charts=[], - lastModified=ChangeAuditStamps(), - externalUrl=full_url, - customProperties={ - key: str(value) - for key, value in { - "displayName": title, - "id": item["id"], - "uid": uid, - "title": title, - "uri": item["uri"], - "type": item["type"], - "folderId": item.get("folderId"), - "folderUid": item.get("folderUid"), - "folderTitle": item.get("folderTitle"), - }.items() - if value is not None - }, + # Process all panels first + for panel in dashboard.panels: + self.report.report_chart_scanned() + + # First emit the dataset for each panel's datasource + yield from self._process_panel_dataset( + panel, dashboard.uid, self.config.ingest_tags + ) + + # Process lineage + lineage = self.lineage_extractor.extract_panel_lineage(panel) + if lineage: + yield lineage.as_workunit() + + # Create chart MCE + dataset_urn, chart_mce = build_chart_mce( + panel=panel, + dashboard=dashboard, + platform=self.config.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + base_url=self.config.url, + ingest_tags=self.config.ingest_tags, + ) + chart_urns.append(chart_mce.urn) + + yield MetadataWorkUnit( + id=f"grafana-chart-{dashboard.uid}-{panel.id}", + mce=MetadataChangeEventClass(proposedSnapshot=chart_mce), + ) + + # Add chart to dashboard container + chart_urn = make_chart_urn( + self.platform, + f"{dashboard.uid}.{panel.id}", + self.platform_instance, + ) + if dataset_urn: + input_fields = extract_fields_from_panel(panel) + if input_fields: + yield from self._add_input_fields_to_chart( + chart_urn=chart_urn, + dataset_urn=dataset_urn, + input_fields=input_fields, + ) + + yield from add_dataset_to_container( + container_key=dashboard_container_key, + dataset_urn=chart_urn, + ) + + # Create dashboard MCE + dashboard_mce = build_dashboard_mce( + dashboard=dashboard, + platform=self.config.platform, + platform_instance=self.config.platform_instance, + chart_urns=chart_urns, + base_url=self.config.url, + ingest_owners=self.config.ingest_owners, + ingest_tags=self.config.ingest_tags, + ) + + yield MetadataWorkUnit( + id=f"grafana-dashboard-{dashboard.uid}", + mce=MetadataChangeEventClass(proposedSnapshot=dashboard_mce), + ) + + # Add dashboard entity to its container + yield from add_dataset_to_container( + container_key=dashboard_container_key, + dataset_urn=dashboard_mce.urn, + ) + + def _add_dashboard_to_folder( + self, dashboard: Dashboard + ) -> Iterable[MetadataWorkUnit]: + """Add dashboard to folder container""" + folder_key = FolderKey( + platform=self.config.platform, + instance=self.config.platform_instance, + folder_id=str(dashboard.folder_id), + ) + + dashboard_key = DashboardContainerKey( + platform=self.config.platform, + instance=self.config.platform_instance, + dashboard_id=dashboard.uid, + ) + + yield from add_dataset_to_container( + container_key=folder_key, + dataset_urn=dashboard_key.as_urn(), + ) + + def _add_input_fields_to_chart( + self, chart_urn: str, dataset_urn: str, input_fields: List[SchemaFieldClass] + ) -> Iterable[MetadataWorkUnit]: + """Add input fields aspect to chart""" + if not input_fields: + return + + yield MetadataChangeProposalWrapper( + entityUrn=chart_urn, + aspect=InputFieldsClass( + fields=[ + InputFieldClass( + schemaField=field, + schemaFieldUrn=make_schema_field_urn( + dataset_urn, field.fieldPath ), - StatusClass(removed=False), - ], - ) + ) + for field in input_fields + ] + ), + ).as_workunit() + + def _process_panel_dataset( + self, panel: Panel, dashboard_uid: str, ingest_tags: bool + ) -> Iterable[MetadataWorkUnit]: + """Process dataset metadata for a panel""" + if not panel.datasource: + return + + ds_type = panel.datasource.get("type", "unknown") + ds_uid = panel.datasource.get("uid", "unknown") + + # Build dataset name + dataset_name = f"{ds_type}.{ds_uid}.{panel.id}" + + # Create dataset URN + dataset_urn = make_dataset_urn_with_platform_instance( + platform=self.platform, + name=dataset_name, + platform_instance=self.platform_instance, + env=self.env, + ) + + # Create dataset snapshot + dataset_snapshot = DatasetSnapshotClass( + urn=dataset_urn, + aspects=[ + DataPlatformInstanceClass( + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn( + platform=self.platform, + instance=self.platform_instance, + ) + if self.platform_instance + else None, + ), + DatasetPropertiesClass( + name=f"{ds_uid} ({panel.title or panel.id})", + description="", + customProperties={ + "type": ds_type, + "uid": ds_uid, + "full_path": dataset_name, + }, + ), + StatusClass(removed=False), + ], + ) + + # Add schema metadata if available + schema_fields = extract_fields_from_panel(panel) + if schema_fields: + schema_metadata = SchemaMetadataClass( + schemaName=f"{ds_type}.{ds_uid}.{panel.id}", + platform=make_data_platform_urn(self.platform), + version=0, + fields=schema_fields, + hash="", + platformSchema=OtherSchemaClass(rawSchema=""), ) + dataset_snapshot.aspects.append(schema_metadata) + + if dashboard_uid and self.config.ingest_tags: + dashboard = self.client.get_dashboard(dashboard_uid) + if dashboard and dashboard.tags: + tags = [] + for tag in dashboard.tags: + if ":" in tag: + key, value = tag.split(":", 1) + tag_urn = make_tag_urn(f"{key}.{value}") + else: + tag_urn = make_tag_urn(tag) + tags.append(TagAssociationClass(tag=tag_urn)) + + if tags: + dataset_snapshot.aspects.append(GlobalTagsClass(tags=tags)) + + self.report.report_dataset_scanned() + yield MetadataWorkUnit( + id=f"grafana-dataset-{ds_uid}-{panel.id}", + mce=MetadataChangeEventClass(proposedSnapshot=dataset_snapshot), + ) + + # Add dataset to dashboard container + if dashboard_uid: + dashboard_key = DashboardContainerKey( + platform=self.platform, + instance=self.platform_instance, + dashboard_id=dashboard_uid, + ) + yield from add_dataset_to_container( + container_key=dashboard_key, + dataset_urn=dataset_urn, + ) + + def get_report(self) -> GrafanaSourceReport: + return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/lineage.py new file mode 100644 index 00000000000000..284e8a16dba0a2 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/lineage.py @@ -0,0 +1,206 @@ +import logging +from typing import Dict, List, Optional, Tuple + +from datahub.emitter.mce_builder import ( + make_dataset_urn_with_platform_instance, + make_schema_field_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import DataHubGraph +from datahub.ingestion.source.grafana.grafana_config import PlatformConnectionConfig +from datahub.ingestion.source.grafana.models import Panel +from datahub.ingestion.source.grafana.report import GrafanaSourceReport +from datahub.metadata.schema_classes import ( + DatasetLineageTypeClass, + FineGrainedLineageClass, + FineGrainedLineageDownstreamTypeClass, + FineGrainedLineageUpstreamTypeClass, + UpstreamClass, + UpstreamLineageClass, +) +from datahub.sql_parsing.sqlglot_lineage import ( + SqlParsingResult, + create_lineage_sql_parsed_result, +) + +logger = logging.getLogger(__name__) + + +class LineageExtractor: + """Handles extraction of lineage information from Grafana panels""" + + def __init__( + self, + platform: str, + platform_instance: Optional[str], + env: str, + connection_to_platform_map: Dict[str, PlatformConnectionConfig], + report: GrafanaSourceReport, + graph: Optional[DataHubGraph] = None, + ): + self.platform = platform + self.platform_instance = platform_instance + self.env = env + self.connection_map = connection_to_platform_map + self.graph = graph + self.report = report + + def extract_panel_lineage( + self, panel: Panel + ) -> Optional[MetadataChangeProposalWrapper]: + """Extract lineage information from a panel.""" + if not panel.datasource: + return None + + ds_type, ds_uid = self._extract_datasource_info(panel.datasource) + raw_sql = self._extract_raw_sql(panel.targets) + ds_urn = self._build_dataset_urn(ds_type, ds_uid, panel.id) + + # Handle platform-specific lineage + if ds_uid in self.connection_map: + if raw_sql: + parsed_sql = self._parse_sql(raw_sql, self.connection_map[ds_uid]) + if parsed_sql: + lineage = self._create_column_lineage(ds_urn, parsed_sql) + if lineage: + return lineage + + # Fall back to basic lineage if SQL parsing fails or no column lineage created + return self._create_basic_lineage( + ds_uid, self.connection_map[ds_uid], ds_urn + ) + + return None + + def _extract_datasource_info(self, datasource: Dict) -> Tuple[str, str]: + """Extract datasource type and UID.""" + return datasource.get("type", "unknown"), datasource.get("uid", "unknown") + + def _extract_raw_sql(self, targets: List[Dict]) -> Optional[str]: + """Extract raw SQL from panel targets.""" + for target in targets: + if target.get("rawSql"): + return target["rawSql"] + return None + + def _build_dataset_urn(self, ds_type: str, ds_uid: str, panel_id: str) -> str: + """Build dataset URN.""" + dataset_name = f"{ds_type}.{ds_uid}.{panel_id}" + return make_dataset_urn_with_platform_instance( + platform=self.platform, + name=dataset_name, + platform_instance=self.platform_instance, + env=self.env, + ) + + def _process_platform_lineage( + self, ds_uid: str, raw_sql: Optional[str], ds_urn: str + ) -> Optional[MetadataChangeProposalWrapper]: + """Process lineage for a specific platform.""" + platform_config = self.connection_map[ds_uid] + + if raw_sql: + parsed_sql = self._parse_sql(raw_sql, platform_config) + if parsed_sql: + return self._create_column_lineage(ds_urn, parsed_sql) + + # Basic lineage fallback + return self._create_basic_lineage(ds_uid, platform_config, ds_urn) + + def _create_basic_lineage( + self, ds_uid: str, platform_config: PlatformConnectionConfig, ds_urn: str + ) -> MetadataChangeProposalWrapper: + """Create basic upstream lineage.""" + name = ( + f"{platform_config.database}.{ds_uid}" + if platform_config.database + else ds_uid + ) + + upstream_urn = make_dataset_urn_with_platform_instance( + platform=platform_config.platform, + name=name, + platform_instance=platform_config.platform_instance, + env=platform_config.env, + ) + + logger.info(f"Generated upstream URN: {upstream_urn}") + + return MetadataChangeProposalWrapper( + entityUrn=ds_urn, + aspect=UpstreamLineageClass( + upstreams=[ + UpstreamClass( + dataset=upstream_urn, + type=DatasetLineageTypeClass.TRANSFORMED, + ) + ] + ), + ) + + def _parse_sql( + self, sql: str, platform_config: PlatformConnectionConfig + ) -> Optional[SqlParsingResult]: + """Parse SQL query for lineage information.""" + if not self.graph: + logger.warning("No DataHub graph specified for SQL parsing.") + return None + + try: + return create_lineage_sql_parsed_result( + query=sql, + platform=platform_config.platform, + platform_instance=platform_config.platform_instance, + env=platform_config.env, + default_db=platform_config.database, + default_schema=platform_config.database_schema, + graph=self.graph, + ) + except ValueError as e: + logger.error(f"SQL parsing error for query: {sql}", exc_info=e) + except Exception as e: + logger.exception(f"Unexpected error during SQL parsing: {sql}", exc_info=e) + + return None + + def _create_column_lineage( + self, + dataset_urn: str, + parsed_sql: SqlParsingResult, + ) -> Optional[MetadataChangeProposalWrapper]: + """Create column-level lineage""" + if not parsed_sql.column_lineage: + return None + + upstream_lineages = [] + for col_lineage in parsed_sql.column_lineage: + upstream_lineages.append( + FineGrainedLineageClass( + downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD, + downstreams=[ + make_schema_field_urn( + dataset_urn, col_lineage.downstream.column + ) + ], + upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, + upstreams=[ + make_schema_field_urn(upstream_dataset, col.column) + for col in col_lineage.upstreams + for upstream_dataset in parsed_sql.in_tables + ], + ) + ) + + return MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineageClass( + upstreams=[ + UpstreamClass( + dataset=table, + type=DatasetLineageTypeClass.TRANSFORMED, + ) + for table in parsed_sql.in_tables + ], + fineGrainedLineages=upstream_lineages, + ), + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/models.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/models.py new file mode 100644 index 00000000000000..de44f50f1f3ddd --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/models.py @@ -0,0 +1,86 @@ +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + +from datahub.emitter.mcp_builder import ContainerKey + + +class Panel(BaseModel): + """Represents a Grafana dashboard panel.""" + + id: str + title: str + description: str = "" + type: Optional[str] + targets: List[Dict[str, Any]] = Field(default_factory=list) + datasource: Optional[Dict[str, Any]] = None + field_config: Dict[str, Any] = Field(default_factory=dict, alias="fieldConfig") + transformations: List[Dict[str, Any]] = Field(default_factory=list) + + +class Dashboard(BaseModel): + """Represents a Grafana dashboard.""" + + uid: str + title: str + description: str = "" + version: Optional[str] + panels: List[Panel] + tags: List[str] + timezone: Optional[str] + refresh: Optional[str] = None + schema_version: Optional[str] = Field(default=None, alias="schemaVersion") + folder_id: Optional[str] = Field(default=None, alias="meta.folderId") + created_by: Optional[str] = None + + @staticmethod + def extract_panels(panels_data: List[Dict[str, Any]]) -> List[Panel]: + """Extract panels, including nested ones.""" + panels: List[Panel] = [] + for panel_data in panels_data: + if panel_data.get("type") == "row" and "panels" in panel_data: + panels.extend( + Panel.parse_obj(p) + for p in panel_data["panels"] + if p.get("type") != "row" + ) + elif panel_data.get("type") != "row": + panels.append(Panel.parse_obj(panel_data)) + return panels + + @classmethod + def parse_obj(cls, data: Dict[str, Any]) -> "Dashboard": + """Custom parsing to handle nested panel extraction.""" + dashboard_data = data.get("dashboard", {}) + panels = cls.extract_panels(dashboard_data.get("panels", [])) + + # Extract meta.folderId from nested structure + meta = dashboard_data.get("meta", {}) + folder_id = meta.get("folderId") + + # Create dashboard data without meta to avoid conflicts + dashboard_dict = {**dashboard_data, "panels": panels, "folder_id": folder_id} + if "meta" in dashboard_dict: + del dashboard_dict["meta"] + + return super().parse_obj(dashboard_dict) + + +class Folder(BaseModel): + """Represents a Grafana folder.""" + + id: str + title: str + description: Optional[str] = "" + + +class FolderKey(ContainerKey): + """Key for identifying a Grafana folder.""" + + folder_id: str + + +class DashboardContainerKey(ContainerKey): + """Key for identifying a Grafana dashboard.""" + + dashboard_id: str diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/report.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/report.py new file mode 100644 index 00000000000000..78ea371081489d --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/report.py @@ -0,0 +1,22 @@ +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) + + +class GrafanaSourceReport(StaleEntityRemovalSourceReport): + dashboards_scanned: int = 0 + charts_scanned: int = 0 + folders_scanned: int = 0 + datasets_scanned: int = 0 + + def report_dashboard_scanned(self) -> None: + self.dashboards_scanned += 1 + + def report_chart_scanned(self) -> None: + self.charts_scanned += 1 + + def report_folder_scanned(self) -> None: + self.folders_scanned += 1 + + def report_dataset_scanned(self) -> None: + self.datasets_scanned += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/snapshots.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/snapshots.py new file mode 100644 index 00000000000000..09de804c4666c7 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/snapshots.py @@ -0,0 +1,234 @@ +from typing import Dict, List, Optional, Tuple + +from datahub.emitter.mce_builder import ( + make_chart_urn, + make_dashboard_urn, + make_data_platform_urn, + make_dataplatform_instance_urn, + make_dataset_urn_with_platform_instance, + make_tag_urn, + make_user_urn, +) +from datahub.ingestion.source.grafana.models import Dashboard, Panel +from datahub.ingestion.source.grafana.types import CHART_TYPE_MAPPINGS +from datahub.metadata.schema_classes import ( + ChangeAuditStampsClass, + ChartInfoClass, + ChartSnapshotClass, + DashboardInfoClass, + DashboardSnapshotClass, + DataPlatformInstanceClass, + GlobalTagsClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + StatusClass, + TagAssociationClass, +) + + +def build_chart_mce( + panel: Panel, + dashboard: Dashboard, + platform: str, + platform_instance: Optional[str], + env: str, + base_url: str, + ingest_tags: bool, +) -> Tuple[Optional[str], ChartSnapshotClass]: + """Build chart metadata change event""" + ds_urn = None + chart_urn = make_chart_urn( + platform, + f"{dashboard.uid}.{panel.id}", + platform_instance, + ) + + chart_snapshot = ChartSnapshotClass( + urn=chart_urn, + aspects=[ + DataPlatformInstanceClass( + platform=make_data_platform_urn(platform), + instance=make_dataplatform_instance_urn( + platform=platform, + instance=platform_instance, + ) + if platform_instance + else None, + ), + StatusClass(removed=False), + ], + ) + + # Ensure title exists + title = panel.title or f"Panel {panel.id}" + + # Get input datasets + input_datasets = [] + if panel.datasource: + ds_type = panel.datasource.get("type", "unknown") + ds_uid = panel.datasource.get("uid", "unknown") + + # Add Grafana dataset + dataset_name = f"{ds_type}.{ds_uid}.{panel.id}" + ds_urn = make_dataset_urn_with_platform_instance( + platform=platform, + name=dataset_name, + platform_instance=platform_instance, + env=env, + ) + input_datasets.append(ds_urn) + + chart_info = ChartInfoClass( + type=CHART_TYPE_MAPPINGS.get(panel.type) if panel.type else None, + description=panel.description, + title=title, + lastModified=ChangeAuditStampsClass(), + chartUrl=f"{base_url}/d/{dashboard.uid}?viewPanel={panel.id}", + customProperties=_build_custom_properties(panel), + inputs=input_datasets, + ) + chart_snapshot.aspects.append(chart_info) + + if dashboard.tags and ingest_tags: + tags = [] + for tag in dashboard.tags: + # Handle both simple tags and key:value tags from Grafana + if ":" in tag: + key, value = tag.split(":", 1) + tag_urn = make_tag_urn(f"{key}.{value}") + else: + tag_urn = make_tag_urn(tag) + tags.append(TagAssociationClass(tag=tag_urn)) + + if tags: + chart_snapshot.aspects.append(GlobalTagsClass(tags=tags)) + + return ds_urn, chart_snapshot + + +def build_dashboard_mce( + dashboard: Dashboard, + platform: str, + platform_instance: Optional[str], + chart_urns: List[str], + base_url: str, + ingest_owners: bool, + ingest_tags: bool, +) -> DashboardSnapshotClass: + """Build dashboard metadata change event""" + dashboard_urn = make_dashboard_urn(platform, dashboard.uid, platform_instance) + + dashboard_snapshot = DashboardSnapshotClass( + urn=dashboard_urn, + aspects=[ + DataPlatformInstanceClass( + platform=make_data_platform_urn(platform), + instance=make_dataplatform_instance_urn( + platform=platform, + instance=platform_instance, + ) + if platform_instance + else None, + ), + ], + ) + + # Add basic info + dashboard_info = DashboardInfoClass( + description=dashboard.description, + title=dashboard.title, + charts=chart_urns, + lastModified=ChangeAuditStampsClass(), + dashboardUrl=f"{base_url}/d/{dashboard.uid}", + customProperties=_build_dashboard_properties(dashboard), + ) + dashboard_snapshot.aspects.append(dashboard_info) + + # Add ownership + if dashboard.uid and ingest_owners: + owner = _build_ownership(dashboard) + if owner: + dashboard_snapshot.aspects.append(owner) + + # Add tags + if dashboard.tags and ingest_tags: + tags = [TagAssociationClass(tag=make_tag_urn(tag)) for tag in dashboard.tags] + if tags: + dashboard_snapshot.aspects.append(GlobalTagsClass(tags=tags)) + + # Add status + dashboard_snapshot.aspects.append(StatusClass(removed=False)) + + return dashboard_snapshot + + +def _build_custom_properties(panel: Panel) -> Dict[str, str]: + """Build custom properties for chart""" + props = {} + + if panel.type: + props["type"] = panel.type + + if panel.datasource: + props["datasourceType"] = panel.datasource.get("type", "") + props["datasourceUid"] = panel.datasource.get("uid", "") + + for key in [ + "description", + "format", + "pluginVersion", + "repeatDirection", + "maxDataPoints", + ]: + value = getattr(panel, key, None) + if value: + props[key] = str(value) + + if panel.targets: + props["queryCount"] = str(len(panel.targets)) + + return props + + +def _build_dashboard_properties(dashboard: Dashboard) -> Dict[str, str]: + """Build custom properties for dashboard""" + props = {} + + if dashboard.timezone: + props["timezone"] = dashboard.timezone + + if dashboard.schema_version: + props["schema_version"] = dashboard.schema_version + + if dashboard.version: + props["version"] = dashboard.version + + if dashboard.refresh: + props["refresh"] = dashboard.refresh + + return props + + +def _build_ownership(dashboard: Dashboard) -> Optional[OwnershipClass]: + """Build ownership information""" + owners = [] + + if dashboard.uid: + owners.append( + OwnerClass( + owner=make_user_urn(dashboard.uid), + type=OwnershipTypeClass.TECHNICAL_OWNER, + ) + ) + + if dashboard.created_by: + owner_id = dashboard.created_by.split("@")[0] + owners.append( + OwnerClass( + owner=make_user_urn(owner_id), + type=OwnershipTypeClass.DATAOWNER, + ) + ) + + return OwnershipClass(owners=owners) if owners else None diff --git a/metadata-ingestion/src/datahub/ingestion/source/grafana/types.py b/metadata-ingestion/src/datahub/ingestion/source/grafana/types.py new file mode 100644 index 00000000000000..78046f2d7817fc --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/grafana/types.py @@ -0,0 +1,56 @@ +from datahub.metadata.schema_classes import ( + BooleanTypeClass, + ChartTypeClass, + DateTypeClass, + NumberTypeClass, + SchemaFieldDataTypeClass, + StringTypeClass, + TimeTypeClass, +) + +CHART_TYPE_MAPPINGS = { + "graph": ChartTypeClass.LINE, + "timeseries": ChartTypeClass.LINE, + "table": ChartTypeClass.TABLE, + "stat": ChartTypeClass.TEXT, + "gauge": ChartTypeClass.TEXT, + "bargauge": ChartTypeClass.TEXT, + "bar": ChartTypeClass.BAR, + "pie": ChartTypeClass.PIE, + "heatmap": ChartTypeClass.TABLE, + "histogram": ChartTypeClass.BAR, +} + + +class GrafanaTypeMapper: + """Maps Grafana types to DataHub types""" + + _TYPE_MAPPINGS = { + "string": SchemaFieldDataTypeClass(type=StringTypeClass()), + "number": SchemaFieldDataTypeClass(type=NumberTypeClass()), + "integer": SchemaFieldDataTypeClass(type=NumberTypeClass()), + "float": SchemaFieldDataTypeClass(type=NumberTypeClass()), + "boolean": SchemaFieldDataTypeClass(type=BooleanTypeClass()), + "time": SchemaFieldDataTypeClass(type=TimeTypeClass()), + "timestamp": SchemaFieldDataTypeClass(type=TimeTypeClass()), + "timeseries": SchemaFieldDataTypeClass(type=TimeTypeClass()), + "time_series": SchemaFieldDataTypeClass(type=TimeTypeClass()), + "datetime": SchemaFieldDataTypeClass(type=TimeTypeClass()), + "date": SchemaFieldDataTypeClass(type=DateTypeClass()), + } + + @classmethod + def get_field_type( + cls, grafana_type: str, default_type: str = "string" + ) -> SchemaFieldDataTypeClass: + return cls._TYPE_MAPPINGS.get( + grafana_type.lower(), + cls._TYPE_MAPPINGS.get(default_type, cls._TYPE_MAPPINGS["string"]), + ) + + @classmethod + def get_native_type(cls, grafana_type: str, default_type: str = "string") -> str: + grafana_type = grafana_type.lower() + if grafana_type in cls._TYPE_MAPPINGS: + return grafana_type + return default_type diff --git a/metadata-ingestion/tests/integration/grafana/dashboards/default-dashboard.json b/metadata-ingestion/tests/integration/grafana/dashboards/default-dashboard.json new file mode 100644 index 00000000000000..7fac8196880ab0 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/dashboards/default-dashboard.json @@ -0,0 +1,295 @@ +{ + "id": null, + "uid": "default", + "title": "Test Integration Dashboard", + "tags": ["test-tag", "integration-test"], + "timezone": "browser", + "schemaVersion": 36, + "version": 0, + "panels": [ + { + "id": 1, + "type": "text", + "title": "Dashboard Information", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 3 + }, + "options": { + "content": "# Test Integration Dashboard\nThis dashboard contains test panels for DataHub integration testing with both PostgreSQL metrics and Prometheus system metrics.", + "mode": "markdown" + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Response Times by Dimension", + "description": "Response times tracked across different dimensions", + "datasource": { + "type": "postgres", + "uid": "test-postgres" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "spanNulls": false + }, + "color": { + "mode": "palette-classic" + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "test-postgres" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, value, dimension FROM test_metrics WHERE metric = 'response_time' AND time > NOW() - interval '1 hour' ORDER BY time ASC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "time" + }, + { + "parameters": [ + { + "name": "value", + "type": "column" + } + ], + "type": "value" + } + ] + } + } + ] + }, + { + "id": 4, + "type": "table", + "title": "Recent Metrics", + "description": "Recent metrics from all sources", + "datasource": { + "type": "postgres", + "uid": "test-postgres" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "value" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 11 + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "test-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT time, metric, value, dimension FROM test_metrics WHERE time > NOW() - interval '1 hour' ORDER BY time DESC LIMIT 10;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "time" + }, + { + "parameters": [ + { + "name": "metric", + "type": "column" + } + ], + "type": "string" + }, + { + "parameters": [ + { + "name": "value", + "type": "column" + } + ], + "type": "number" + }, + { + "parameters": [ + { + "name": "dimension", + "type": "column" + } + ], + "type": "string" + } + ] + } + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Total Metrics Count", + "description": "Total number of metrics collected", + "datasource": { + "type": "postgres", + "uid": "test-postgres" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "test-postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM test_metrics WHERE time > NOW() - interval '1 hour';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "count", + "type": "column" + } + ], + "type": "number" + } + ] + } + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "System Metrics", + "description": "Prometheus system metrics", + "datasource": { + "type": "prometheus", + "uid": "test-prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10 + } + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 19 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "test-prometheus" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total[5m])", + "legendFormat": "CPU Usage", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "test-prometheus" + }, + "editorMode": "code", + "expr": "go_memstats_alloc_bytes", + "legendFormat": "Memory Usage", + "range": true, + "refId": "B" + } + ] + } + ], + "refresh": "5s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "timezone": "browser", + "description": "A comprehensive test dashboard for integration testing with various panel types and data sources" +} diff --git a/metadata-ingestion/tests/integration/grafana/default-dashboard.json b/metadata-ingestion/tests/integration/grafana/default-dashboard.json deleted file mode 100644 index 8ce40ad6acb13a..00000000000000 --- a/metadata-ingestion/tests/integration/grafana/default-dashboard.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "id": null, - "uid": "default", - "title": "Default Dashboard", - "tags": [], - "timezone": "browser", - "schemaVersion": 16, - "version": 0, - "panels": [ - { - "type": "text", - "title": "Welcome", - "gridPos": { - "x": 0, - "y": 0, - "w": 24, - "h": 5 - }, - "options": { - "content": "Welcome to your Grafana dashboard!", - "mode": "markdown" - } - } - ] -} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/grafana/docker-compose.yml b/metadata-ingestion/tests/integration/grafana/docker-compose.yml index 41995a1d49da60..f589a05ef1a045 100644 --- a/metadata-ingestion/tests/integration/grafana/docker-compose.yml +++ b/metadata-ingestion/tests/integration/grafana/docker-compose.yml @@ -10,22 +10,56 @@ services: - GF_SECURITY_ADMIN_PASSWORD=admin - GF_SECURITY_ADMIN_USER=admin - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_AUTH_DISABLE_LOGIN_FORM=false + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + - GF_FEATURE_TOGGLES_ENABLE=publicDashboards volumes: - grafana-storage:/var/lib/grafana - ./provisioning:/etc/grafana/provisioning - - ./default-dashboard.json:/var/lib/grafana/dashboards/default-dashboard.json + - ./dashboards:/var/lib/grafana/dashboards depends_on: - - postgres + postgres: + condition: service_healthy + prometheus: + condition: service_started + networks: + - grafana-network postgres: image: postgres:13 - container_name: grafana-postgres + container_name: postgres environment: POSTGRES_DB: grafana POSTGRES_USER: grafana POSTGRES_PASSWORD: grafana + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U grafana -d grafana" ] + interval: 10s + timeout: 5s + retries: 5 volumes: - postgres-storage:/var/lib/postgresql/data + - ./postgres-init:/docker-entrypoint-initdb.d + networks: + - grafana-network + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + networks: + - grafana-network + +networks: + grafana-network: + driver: bridge volumes: grafana-storage: diff --git a/metadata-ingestion/tests/integration/grafana/grafana_basic_mcps_golden.json b/metadata-ingestion/tests/integration/grafana/grafana_basic_mcps_golden.json new file mode 100644 index 00000000000000..97fb795ffdcdab --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/grafana_basic_mcps_golden.json @@ -0,0 +1,1404 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "grafana", + "dashboard_id": "default" + }, + "name": "Test Integration Dashboard", + "description": "A comprehensive test dashboard for integration testing with various panel types and data sources" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dashboard" + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,default.1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "text" + }, + "title": "Dashboard Information", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=1", + "inputs": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "postgres", + "uid": "test-postgres", + "full_path": "postgres.test-postgres.2" + }, + "name": "test-postgres (Response Times by Dimension)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "postgres.test-postgres.2", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "value_ms", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "value", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,default.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "timeseries", + "datasourceType": "postgres", + "datasourceUid": "test-postgres", + "description": "Response times tracked across different dimensions", + "queryCount": "1" + }, + "title": "Response Times by Dimension", + "description": "Response times tracked across different dimensions", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=2", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD)" + } + ], + "type": "LINE" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.2)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD),value)", + "schemaField": { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD),time)", + "schemaField": { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD),dimension)", + "schemaField": { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.2,PROD),value_ms)", + "schemaField": { + "fieldPath": "value_ms", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "value", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.2)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "postgres", + "uid": "test-postgres", + "full_path": "postgres.test-postgres.4" + }, + "name": "test-postgres (Recent Metrics)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "postgres.test-postgres.4", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "metric", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "metric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,default.4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "table", + "datasourceType": "postgres", + "datasourceUid": "test-postgres", + "description": "Recent metrics from all sources", + "queryCount": "1" + }, + "title": "Recent Metrics", + "description": "Recent metrics from all sources", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=4", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD)" + } + ], + "type": "TABLE" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD),metric)", + "schemaField": { + "fieldPath": "metric", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD),value)", + "schemaField": { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "metric", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD),dimension)", + "schemaField": { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.4,PROD),time)", + "schemaField": { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.5,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "postgres", + "uid": "test-postgres", + "full_path": "postgres.test-postgres.5" + }, + "name": "test-postgres (Total Metrics Count)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "postgres.test-postgres.5", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.5,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.5,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,default.5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "stat", + "datasourceType": "postgres", + "datasourceUid": "test-postgres", + "description": "Total number of metrics collected", + "queryCount": "1" + }, + "title": "Total Metrics Count", + "description": "Total number of metrics collected", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=5", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.5,PROD)" + } + ], + "type": "TEXT" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.5,PROD),count)", + "schemaField": { + "fieldPath": "count", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,postgres.test-postgres.5,PROD),time)", + "schemaField": { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,prometheus.test-prometheus.6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "prometheus", + "uid": "test-prometheus", + "full_path": "prometheus.test-prometheus.6" + }, + "name": "test-prometheus (System Metrics)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "prometheus.test-prometheus.6", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "CPU Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Memory Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,prometheus.test-prometheus.6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,prometheus.test-prometheus.6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,default.6)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "timeseries", + "datasourceType": "prometheus", + "datasourceUid": "test-prometheus", + "description": "Prometheus system metrics", + "queryCount": "2" + }, + "title": "System Metrics", + "description": "Prometheus system metrics", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=6", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,prometheus.test-prometheus.6,PROD)" + } + ], + "type": "LINE" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.6)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,prometheus.test-prometheus.6,PROD),CPU Usage)", + "schemaField": { + "fieldPath": "CPU Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,prometheus.test-prometheus.6,PROD),Memory Usage)", + "schemaField": { + "fieldPath": "Memory Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.6)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,default.6)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(grafana,default)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "timezone": "browser", + "schema_version": "36", + "version": "1", + "refresh": "5s" + }, + "title": "Test Integration Dashboard", + "description": "A comprehensive test dashboard for integration testing with various panel types and data sources", + "charts": [ + "urn:li:chart:(grafana,default.1)", + "urn:li:chart:(grafana,default.2)", + "urn:li:chart:(grafana,default.4)", + "urn:li:chart:(grafana,default.5)", + "urn:li:chart:(grafana,default.6)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://localhost:3000/d/default" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(grafana,default)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(grafana,default)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099", + "urn": "urn:li:container:15a4d4f049d1fb26674e0b66dfa58099" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json b/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json index 1447e840eac8cd..0d755ee1cb6e08 100644 --- a/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json +++ b/metadata-ingestion/tests/integration/grafana/grafana_mcps_golden.json @@ -1,45 +1,29 @@ [ { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(grafana,default)", + "entityType": "container", + "entityUrn": "urn:li:container:ae0ac23df7f392b003891eb008eea810", "changeType": "UPSERT", - "aspectName": "dashboardInfo", + "aspectName": "containerProperties", "aspect": { "json": { "customProperties": { - "displayName": "Default Dashboard", - "id": "1", - "uid": "default", - "title": "Default Dashboard", - "uri": "db/default-dashboard", - "type": "dash-db" + "platform": "grafana", + "instance": "local-grafana", + "dashboard_id": "default" }, - "externalUrl": "http://localhost:3000/d/default/default-dashboard", - "title": "Default Dashboard", - "description": "", - "charts": [], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } + "name": "Test Integration Dashboard", + "description": "A comprehensive test dashboard for integration testing with various panel types and data sources" } }, "systemMetadata": { "lastObserved": 1720785600000, - "runId": "grafana-test-simple", + "runId": "grafana-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(grafana,default)", + "entityType": "container", + "entityUrn": "urn:li:container:ae0ac23df7f392b003891eb008eea810", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -49,7 +33,1695 @@ }, "systemMetadata": { "lastObserved": 1720785600000, - "runId": "grafana-test-simple", + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dashboard" + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,local-grafana.default.1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "text" + }, + "title": "Dashboard Information", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=1", + "inputs": [] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "postgres", + "uid": "test-postgres", + "full_path": "postgres.test-postgres.2" + }, + "name": "test-postgres (Response Times by Dimension)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "postgres.test-postgres.2", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "value_ms", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "value", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,local.grafana.test-postgres,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,local-grafana.default.2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "timeseries", + "datasourceType": "postgres", + "datasourceUid": "test-postgres", + "description": "Response times tracked across different dimensions", + "queryCount": "1" + }, + "title": "Response Times by Dimension", + "description": "Response times tracked across different dimensions", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=2", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD)" + } + ], + "type": "LINE" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.2)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD),value)", + "schemaField": { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD),time)", + "schemaField": { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD),dimension)", + "schemaField": { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.2,PROD),value_ms)", + "schemaField": { + "fieldPath": "value_ms", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "value", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.2)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.2)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "postgres", + "uid": "test-postgres", + "full_path": "postgres.test-postgres.4" + }, + "name": "test-postgres (Recent Metrics)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "postgres.test-postgres.4", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "metric", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "metric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,local.grafana.test-postgres,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,local-grafana.default.4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "table", + "datasourceType": "postgres", + "datasourceUid": "test-postgres", + "description": "Recent metrics from all sources", + "queryCount": "1" + }, + "title": "Recent Metrics", + "description": "Recent metrics from all sources", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=4", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD)" + } + ], + "type": "TABLE" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD),metric)", + "schemaField": { + "fieldPath": "metric", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD),value)", + "schemaField": { + "fieldPath": "value", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "metric", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD),dimension)", + "schemaField": { + "fieldPath": "dimension", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.4,PROD),time)", + "schemaField": { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "postgres", + "uid": "test-postgres", + "full_path": "postgres.test-postgres.5" + }, + "name": "test-postgres (Total Metrics Count)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "postgres.test-postgres.5", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,local.grafana.test-postgres,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,local-grafana.default.5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "stat", + "datasourceType": "postgres", + "datasourceUid": "test-postgres", + "description": "Total number of metrics collected", + "queryCount": "1" + }, + "title": "Total Metrics Count", + "description": "Total number of metrics collected", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=5", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD)" + } + ], + "type": "TEXT" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD),count)", + "schemaField": { + "fieldPath": "count", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "sql_column", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.postgres.test-postgres.5,PROD),time)", + "schemaField": { + "fieldPath": "time", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "type": "prometheus", + "uid": "test-prometheus", + "full_path": "prometheus.test-prometheus.6" + }, + "name": "test-prometheus (System Metrics)", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "prometheus.test-prometheus.6", + "platform": "urn:li:dataPlatform:grafana", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "CPU Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Memory Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:prometheus,local.test-prometheus,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(grafana,local-grafana.default.6)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "type": "timeseries", + "datasourceType": "prometheus", + "datasourceUid": "test-prometheus", + "description": "Prometheus system metrics", + "queryCount": "2" + }, + "title": "System Metrics", + "description": "Prometheus system metrics", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "chartUrl": "http://localhost:3000/d/default?viewPanel=6", + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD)" + } + ], + "type": "LINE" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.6)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD),CPU Usage)", + "schemaField": { + "fieldPath": "CPU Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:grafana,local-grafana.prometheus.test-prometheus.6,PROD),Memory Usage)", + "schemaField": { + "fieldPath": "Memory Usage", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "prometheus_metric", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.6)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(grafana,local-grafana.default.6)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(grafana,local-grafana.default)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:grafana", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "timezone": "browser", + "schema_version": "36", + "version": "1", + "refresh": "5s" + }, + "title": "Test Integration Dashboard", + "description": "A comprehensive test dashboard for integration testing with various panel types and data sources", + "charts": [ + "urn:li:chart:(grafana,local-grafana.default.1)", + "urn:li:chart:(grafana,local-grafana.default.2)", + "urn:li:chart:(grafana,local-grafana.default.4)", + "urn:li:chart:(grafana,local-grafana.default.5)", + "urn:li:chart:(grafana,local-grafana.default.6)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "http://localhost:3000/d/default" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:default", + "type": "TECHNICAL_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:test-tag" + }, + { + "tag": "urn:li:tag:integration-test" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(grafana,local-grafana.default)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(grafana,local-grafana.default)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:grafana,local-grafana)" + }, + { + "id": "urn:li:container:ae0ac23df7f392b003891eb008eea810", + "urn": "urn:li:container:ae0ac23df7f392b003891eb008eea810" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:integration-test", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "integration-test" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:test-tag", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "test-tag" + } + }, + "systemMetadata": { + "lastObserved": 1720785600000, + "runId": "grafana-test", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/grafana/postgres-init/init.sql b/metadata-ingestion/tests/integration/grafana/postgres-init/init.sql new file mode 100644 index 00000000000000..452b89c07f6f99 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/postgres-init/init.sql @@ -0,0 +1,74 @@ +-- Connect to the grafana database +\connect grafana + +-- Create test metrics table +CREATE TABLE IF NOT EXISTS test_metrics ( + id SERIAL PRIMARY KEY, + time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + metric VARCHAR(50), + value NUMERIC, + dimension VARCHAR(50) +); + +-- Create indexes for better query performance +CREATE INDEX IF NOT EXISTS idx_test_metrics_time ON test_metrics(time); +CREATE INDEX IF NOT EXISTS idx_test_metrics_metric ON test_metrics(metric); + +-- Create a function to generate random data +CREATE OR REPLACE FUNCTION generate_random_metrics() +RETURNS TABLE ( + metric_time TIMESTAMP WITH TIME ZONE, + metric_name VARCHAR(50), + metric_value NUMERIC, + metric_dimension VARCHAR(50) +) AS $$ +BEGIN + RETURN QUERY + SELECT + NOW() - (i || ' minutes')::interval, + (CASE (random() * 2)::integer + WHEN 0 THEN 'response_time' + WHEN 1 THEN 'error_rate' + ELSE 'cpu_usage' + END)::VARCHAR(50), + (random() * 100)::numeric, + (CASE (random() * 2)::integer + WHEN 0 THEN 'api' + WHEN 1 THEN 'web' + ELSE 'mobile' + END)::VARCHAR(50) + FROM generate_series(0, 60) i; +END; +$$ LANGUAGE plpgsql; + +-- Insert initial static test data +INSERT INTO test_metrics (time, metric, value, dimension) VALUES + (NOW() - interval '1 hour', 'response_time', 100, 'api'), + (NOW() - interval '50 minutes', 'response_time', 150, 'api'), + (NOW() - interval '40 minutes', 'response_time', 120, 'api'), + (NOW() - interval '30 minutes', 'response_time', 200, 'web'), + (NOW() - interval '20 minutes', 'response_time', 180, 'web'), + (NOW() - interval '10 minutes', 'response_time', 90, 'mobile'), + (NOW(), 'response_time', 110, 'mobile'); + +-- Insert random test data +INSERT INTO test_metrics (time, metric, value, dimension) +SELECT metric_time, metric_name, metric_value, metric_dimension +FROM generate_random_metrics(); + +-- Create a view for common aggregations +CREATE OR REPLACE VIEW metric_summaries AS +SELECT + metric, + dimension, + AVG(value) as avg_value, + MAX(value) as max_value, + MIN(value) as min_value, + COUNT(*) as count +FROM test_metrics +WHERE time > NOW() - interval '1 hour' +GROUP BY metric, dimension; + +-- Grant necessary permissions +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO grafana; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO grafana; \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/grafana/prometheus.yml b/metadata-ingestion/tests/integration/grafana/prometheus.yml new file mode 100644 index 00000000000000..7aad7f688d2910 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'grafana' + static_configs: + - targets: ['grafana:3000'] diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml index e6d4aa3a45a39d..3a57e3a5dfc3ed 100644 --- a/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml +++ b/metadata-ingestion/tests/integration/grafana/provisioning/dashboards/dashboard.yaml @@ -7,5 +7,7 @@ providers: type: file disableDeletion: false updateIntervalSeconds: 10 + allowUiUpdates: true options: path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml index 9ba65ec1a54bc6..93135eb07edbf5 100644 --- a/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml +++ b/metadata-ingestion/tests/integration/grafana/provisioning/datasources/datasource.yaml @@ -8,5 +8,22 @@ datasources: database: grafana user: grafana password: grafana + uid: test-postgres jsonData: sslmode: disable + postgresVersion: 1300 + timescaledb: false + maxIdleConns: 100 + maxOpenConns: 100 + connMaxLifetime: 14400 + secureJsonData: + password: grafana + + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + uid: test-prometheus + jsonData: + httpMethod: GET + timeInterval: "15s" \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/notifiers/notifiers.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/notifiers/notifiers.yaml new file mode 100644 index 00000000000000..c5ce856237d759 --- /dev/null +++ b/metadata-ingestion/tests/integration/grafana/provisioning/notifiers/notifiers.yaml @@ -0,0 +1,3 @@ +apiVersion: 1 + +notifiers: [] diff --git a/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml b/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml index a6c259aac77abd..d01bd2a053950f 100644 --- a/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml +++ b/metadata-ingestion/tests/integration/grafana/provisioning/service_accounts/service_accounts.yaml @@ -1,6 +1,8 @@ service_accounts: - name: 'example-service-account' role: 'Admin' + orgId: 1 apiKeys: - keyName: 'example-api-key' role: 'Admin' + secondsToLive: 86400 diff --git a/metadata-ingestion/tests/integration/grafana/test_grafana.py b/metadata-ingestion/tests/integration/grafana/test_grafana.py index cbac965884365d..5b7e2f387b62f4 100644 --- a/metadata-ingestion/tests/integration/grafana/test_grafana.py +++ b/metadata-ingestion/tests/integration/grafana/test_grafana.py @@ -14,7 +14,6 @@ FROZEN_TIME = "2024-07-12 12:00:00" - logger = logging.getLogger(__name__) @@ -65,33 +64,27 @@ def test_resources_dir(pytestconfig): @pytest.fixture(scope="module") def test_api_key(): - # Example usage: url = "http://localhost:3000" admin_user = "admin" admin_password = "admin" grafana_client = GrafanaClient(url, admin_user, admin_password) - # Step 1: Create the service account service_account = grafana_client.create_service_account( - name="example-service-account", role="Viewer" + name="example-service-account", role="Admin" ) if service_account: - print(f"Service Account Created: {service_account}") - - # Step 2: Create the API key for the service account api_key = grafana_client.create_api_key( service_account_id=service_account["id"], key_name="example-api-key", role="Admin", ) if api_key: - print("Service Account API Key:", api_key) return api_key else: - print("Failed to create API key for the service account") + pytest.fail("Failed to create API key for the service account") else: - print("Failed to create service account") + pytest.fail("Failed to create service account") @pytest.fixture(scope="module") @@ -107,68 +100,98 @@ def loaded_grafana(docker_compose_runner, test_resources_dir): ) yield docker_services - # The Grafana image can be large, so we remove it after the test. cleanup_image("grafana/grafana") -@freeze_time(FROZEN_TIME) -def test_grafana_dashboard(loaded_grafana, pytestconfig, tmp_path, test_resources_dir): - # Wait for Grafana to be up and running - url = "http://localhost:3000/api/health" - for i in range(30): +def wait_for_grafana(url: str, max_attempts: int = 30, sleep_time: int = 5) -> bool: + """Helper function to wait for Grafana to start""" + for i in range(max_attempts): logging.info("waiting for Grafana to start...") - time.sleep(5) - resp = requests.get(url) - if resp.status_code == 200: - logging.info(f"Grafana started after waiting {i * 5} seconds") - break - else: - pytest.fail("Grafana did not start in time") + try: + resp = requests.get(url) + if resp.status_code == 200: + logging.info(f"Grafana started after waiting {i * sleep_time} seconds") + return True + except requests.exceptions.RequestException: + pass + time.sleep(sleep_time) - # Check if the default dashboard is loaded - dashboard_url = "http://localhost:3000/api/dashboards/uid/default" - resp = requests.get(dashboard_url, auth=("admin", "admin")) - assert resp.status_code == 200, "Failed to load default dashboard" - dashboard = resp.json() + pytest.fail("Grafana did not start in time") - assert dashboard["dashboard"]["title"] == "Default Dashboard", ( - "Default dashboard title mismatch" - ) - assert any(panel["type"] == "text" for panel in dashboard["dashboard"]["panels"]), ( - "Default dashboard missing text panel" - ) - # Verify the output. (You can add further checks here if needed) - logging.info("Default dashboard verified successfully") +@freeze_time(FROZEN_TIME) +def test_grafana_basic_ingest( + loaded_grafana, pytestconfig, tmp_path, test_resources_dir, test_api_key +): + """Test ingestion with lineage enabled""" + wait_for_grafana("http://localhost:3000/api/health") + + with fs_helpers.isolated_filesystem(tmp_path): + pipeline = Pipeline.create( + { + "run_id": "grafana-test", + "source": { + "type": "grafana", + "config": { + "url": "http://localhost:3000", + "service_account_token": test_api_key, + "ingest_tags": False, + "ingest_owners": False, + }, + }, + "sink": { + "type": "file", + "config": {"filename": "./grafana_basic_mcps.json"}, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + mce_helpers.check_golden_file( + pytestconfig, + output_path="grafana_basic_mcps.json", + golden_path=test_resources_dir / "grafana_basic_mcps_golden.json", + ignore_paths=[ + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['lastModified'\]", + ], + ) @freeze_time(FROZEN_TIME) def test_grafana_ingest( loaded_grafana, pytestconfig, tmp_path, test_resources_dir, test_api_key ): - # Wait for Grafana to be up and running - url = "http://localhost:3000/api/health" - for i in range(30): - logging.info("waiting for Grafana to start...") - time.sleep(5) - resp = requests.get(url) - if resp.status_code == 200: - logging.info(f"Grafana started after waiting {i * 5} seconds") - break - else: - pytest.fail("Grafana did not start in time") + """Test ingestion with lineage enabled""" + wait_for_grafana("http://localhost:3000/api/health") - # Run the metadata ingestion pipeline. with fs_helpers.isolated_filesystem(tmp_path): - # Run grafana ingestion run. pipeline = Pipeline.create( { - "run_id": "grafana-test-simple", + "run_id": "grafana-test", "source": { "type": "grafana", "config": { "url": "http://localhost:3000", "service_account_token": test_api_key, + "ingest_tags": True, + "ingest_owners": True, + "connection_to_platform_map": { + "test-postgres": { + "platform": "postgres", + "database": "grafana", + "platform_instance": "local", + "env": "PROD", + }, + "test-prometheus": { + "platform": "prometheus", + "platform_instance": "local", + "env": "PROD", + }, + }, + "platform_instance": "local-grafana", + "env": "PROD", }, }, "sink": { @@ -180,12 +203,12 @@ def test_grafana_ingest( pipeline.run() pipeline.raise_from_status() - # Verify the output. mce_helpers.check_golden_file( pytestconfig, output_path="grafana_mcps.json", golden_path=test_resources_dir / "grafana_mcps_golden.json", ignore_paths=[ - r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['last_event_time'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['lastModified'\]", ], ) diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_api.py b/metadata-ingestion/tests/unit/grafana/test_grafana_api.py new file mode 100644 index 00000000000000..e5555b32e9a0b6 --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_api.py @@ -0,0 +1,183 @@ +from unittest.mock import MagicMock, patch + +import pytest +import requests +from pydantic import SecretStr + +from datahub.ingestion.source.grafana.grafana_api import GrafanaAPIClient +from datahub.ingestion.source.grafana.models import Dashboard, Folder +from datahub.ingestion.source.grafana.report import GrafanaSourceReport + + +@pytest.fixture +def mock_session(): + with patch("requests.Session") as session: + yield session.return_value + + +@pytest.fixture +def api_client(mock_session): + report = GrafanaSourceReport() + return GrafanaAPIClient( + base_url="http://grafana.test", + token=SecretStr("test-token"), + verify_ssl=True, + report=report, + ) + + +def test_create_session(mock_session): + report = GrafanaSourceReport() + GrafanaAPIClient( + base_url="http://grafana.test", + token=SecretStr("test-token"), + verify_ssl=True, + report=report, + ) + + # Verify headers were properly set + mock_session.headers.update.assert_called_once_with( + { + "Authorization": "Bearer test-token", + "Accept": "application/json", + "Content-Type": "application/json", + } + ) + + # Verify SSL verification was set + assert mock_session.verify is True + + +def test_get_folders_success(api_client, mock_session): + # First call returns folders + first_response = MagicMock() + first_response.json.return_value = [ + {"id": "1", "title": "Folder 1", "description": ""}, + {"id": "2", "title": "Folder 2", "description": ""}, + ] + first_response.raise_for_status.return_value = None + + # Second call returns empty list to end pagination + second_response = MagicMock() + second_response.json.return_value = [] + second_response.raise_for_status.return_value = None + + mock_session.get.side_effect = [first_response, second_response] + + folders = api_client.get_folders() + + assert len(folders) == 2 + assert all(isinstance(f, Folder) for f in folders) + assert folders[0].id == "1" + assert folders[0].title == "Folder 1" + + +def test_get_folders_error(api_client, mock_session): + mock_session.get.side_effect = requests.exceptions.RequestException("API Error") + + folders = api_client.get_folders() + + assert len(folders) == 0 + assert len(api_client.report.failures) == 1 + + +def test_get_dashboard_success(api_client, mock_session): + mock_response = MagicMock() + mock_response.json.return_value = { + "dashboard": { + "uid": "test-uid", + "title": "Test Dashboard", + "description": "", + "version": "1", + "panels": [], + "tags": [], + "schemaVersion": "1.0", + "timezone": "utc", + "refresh": None, + "meta": {"folderId": "123"}, + } + } + mock_session.get.return_value = mock_response + + dashboard = api_client.get_dashboard("test-uid") + + assert isinstance(dashboard, Dashboard) + assert dashboard.uid == "test-uid" + assert dashboard.title == "Test Dashboard" + + +def test_get_dashboard_error(api_client, mock_session): + mock_session.get.side_effect = requests.exceptions.RequestException("API Error") + + dashboard = api_client.get_dashboard("test-uid") + + assert dashboard is None + assert len(api_client.report.warnings) == 1 + + +def test_get_dashboards_success(api_client, mock_session): + # Mock search response + search_response = MagicMock() + search_response.raise_for_status.return_value = None + search_response.json.return_value = [{"uid": "dash1"}, {"uid": "dash2"}] + + # Mock individual dashboard responses + dash1_response = MagicMock() + dash1_response.raise_for_status.return_value = None + dash1_response.json.return_value = { + "dashboard": { + "uid": "dash1", + "title": "Dashboard 1", + "description": "", + "version": "1", + "panels": [], + "tags": [], + "timezone": "utc", + "schemaVersion": "1.0", + "meta": {"folderId": None}, + } + } + + # Mock dashboard2 response + dash2_response = MagicMock() + dash2_response.raise_for_status.return_value = None + dash2_response.json.return_value = { + "dashboard": { + "uid": "dash2", + "title": "Dashboard 2", + "description": "", + "version": "1", + "panels": [], + "tags": [], + "timezone": "utc", + "schemaVersion": "1.0", + "meta": {"folderId": None}, + } + } + + # Empty response to end pagination + empty_response = MagicMock() + empty_response.json.return_value = [] + empty_response.raise_for_status.return_value = None + + mock_session.get.side_effect = [ + search_response, + dash1_response, + dash2_response, + empty_response, + ] + + dashboards = api_client.get_dashboards() + + assert len(dashboards) == 2 + assert dashboards[0].uid == "dash1" + assert dashboards[0].title == "Dashboard 1" + + +def test_get_dashboards_error(api_client, mock_session): + mock_session.get.side_effect = requests.exceptions.RequestException("API Error") + + dashboards = api_client.get_dashboards() + + assert len(dashboards) == 0 + assert len(api_client.report.failures) == 1 diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_field_utils.py b/metadata-ingestion/tests/unit/grafana/test_grafana_field_utils.py new file mode 100644 index 00000000000000..42172d8298ffa3 --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_field_utils.py @@ -0,0 +1,107 @@ +from datahub.ingestion.source.grafana.field_utils import ( + extract_prometheus_fields, + extract_raw_sql_fields, + extract_sql_column_fields, + extract_time_format_fields, + get_fields_from_field_config, + get_fields_from_transformations, +) +from datahub.metadata.schema_classes import ( + NumberTypeClass, + StringTypeClass, + TimeTypeClass, +) + + +def test_extract_sql_column_fields(): + target = { + "sql": { + "columns": [ + { + "type": "time", + "parameters": [{"type": "column", "name": "timestamp"}], + }, + {"type": "number", "parameters": [{"type": "column", "name": "value"}]}, + {"type": "string", "parameters": [{"type": "column", "name": "name"}]}, + ] + } + } + + fields = extract_sql_column_fields(target) + + assert len(fields) == 3 + assert fields[0].fieldPath == "timestamp" + assert isinstance(fields[0].type.type, TimeTypeClass) + assert fields[1].fieldPath == "value" + assert isinstance(fields[1].type.type, NumberTypeClass) + assert fields[2].fieldPath == "name" + assert isinstance(fields[2].type.type, StringTypeClass) + + +def test_extract_prometheus_fields(): + target = { + "expr": "sum(rate(http_requests_total[5m]))", + "legendFormat": "HTTP Requests", + } + + fields = extract_prometheus_fields(target) + + assert len(fields) == 1 + assert fields[0].fieldPath == "HTTP Requests" + assert isinstance(fields[0].type.type, NumberTypeClass) + assert fields[0].nativeDataType == "prometheus_metric" + + +def test_extract_raw_sql_fields(): + target = { + "rawSql": "SELECT name as user_name, count as request_count FROM requests" + } + + fields = extract_raw_sql_fields(target) + assert len(fields) == 2 + assert fields[0].fieldPath == "user_name" + assert fields[1].fieldPath == "request_count" + + +def test_extract_raw_sql_fields_invalid(): + target = {"rawSql": "INVALID SQL"} + + fields = extract_raw_sql_fields(target) + assert len(fields) == 0 + + +def test_extract_time_format_fields(): + target = {"format": "time_series"} + fields = extract_time_format_fields(target) + + assert len(fields) == 1 + assert fields[0].fieldPath == "time" + assert isinstance(fields[0].type.type, TimeTypeClass) + assert fields[0].nativeDataType == "timestamp" + + +def test_get_fields_from_field_config(): + field_config = { + "defaults": {"unit": "bytes"}, + "overrides": [{"matcher": {"id": "byName", "options": "memory_usage"}}], + } + + fields = get_fields_from_field_config(field_config) + + assert len(fields) == 2 + assert fields[0].fieldPath == "value_bytes" + assert fields[1].fieldPath == "memory_usage" + + +def test_get_fields_from_transformations(): + transformations = [ + { + "type": "organize", + "options": {"indexByName": {"user": "user", "value": "value"}}, + } + ] + + fields = get_fields_from_transformations(transformations) + assert len(fields) == 2 + field_paths = {f.fieldPath for f in fields} + assert field_paths == {"user", "value"} diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_lineage.py b/metadata-ingestion/tests/unit/grafana/test_grafana_lineage.py new file mode 100644 index 00000000000000..eceb6428c64913 --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_lineage.py @@ -0,0 +1,193 @@ +from unittest.mock import MagicMock + +import pytest + +from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.source.grafana.grafana_config import PlatformConnectionConfig +from datahub.ingestion.source.grafana.lineage import LineageExtractor +from datahub.ingestion.source.grafana.models import Panel +from datahub.ingestion.source.grafana.report import GrafanaSourceReport +from datahub.metadata.schema_classes import ( + DatasetLineageTypeClass, + UpstreamLineageClass, +) + + +@pytest.fixture +def mock_graph(): + return MagicMock() + + +@pytest.fixture +def mock_report(): + return GrafanaSourceReport() + + +@pytest.fixture +def lineage_extractor(mock_graph, mock_report): + return LineageExtractor( + platform="grafana", + platform_instance="test-instance", + env="PROD", + connection_to_platform_map={ + "postgres_uid": PlatformConnectionConfig( + platform="postgres", + database="test_db", + database_schema="public", + ), + "mysql_uid": PlatformConnectionConfig( + platform="mysql", + database="test_db", + ), + }, + report=mock_report, + graph=mock_graph, + ) + + +def test_extract_panel_lineage_no_datasource(lineage_extractor): + panel = Panel(id="1", title="Test Panel", type="graph", datasource=None, targets=[]) + + lineage = lineage_extractor.extract_panel_lineage(panel) + assert lineage is None + + +def test_extract_panel_lineage_unknown_datasource(lineage_extractor): + panel = Panel( + id="1", + title="Test Panel", + type="graph", + datasource={"type": "unknown", "uid": "unknown_uid"}, + targets=[], + ) + + lineage = lineage_extractor.extract_panel_lineage(panel) + assert lineage is None + + +def test_extract_panel_lineage_postgres(lineage_extractor): + panel = Panel( + id="1", + title="Test Panel", + type="graph", + datasource={"type": "postgres", "uid": "postgres_uid"}, + targets=[ + { + "rawSql": "SELECT value, timestamp FROM test_table", + "format": "table", + "sql": { + "columns": [ + { + "type": "number", + "parameters": [{"type": "column", "name": "value"}], + }, + { + "type": "time", + "parameters": [{"type": "column", "name": "timestamp"}], + }, + ] + }, + } + ], + ) + + lineage = lineage_extractor.extract_panel_lineage(panel) + assert lineage is not None, "Lineage should not be None" + assert isinstance(lineage, MetadataChangeProposalWrapper) + assert isinstance(lineage.aspect, UpstreamLineageClass) + assert len(lineage.aspect.upstreams) == 1 + assert lineage.aspect.upstreams[0].type == DatasetLineageTypeClass.TRANSFORMED + + +def test_extract_panel_lineage_mysql(lineage_extractor): + panel = Panel( + id="1", + title="Test Panel", + type="graph", + datasource={"type": "mysql", "uid": "mysql_uid"}, + targets=[ + { + "rawSql": "SELECT value, timestamp FROM test_table", + "format": "table", + "sql": { + "columns": [ + { + "type": "number", + "parameters": [{"type": "column", "name": "value"}], + }, + { + "type": "time", + "parameters": [{"type": "column", "name": "timestamp"}], + }, + ] + }, + } + ], + ) + + lineage = lineage_extractor.extract_panel_lineage(panel) + assert lineage is not None, "Lineage should not be None" + assert isinstance(lineage, MetadataChangeProposalWrapper) + assert isinstance(lineage.aspect, UpstreamLineageClass) + assert len(lineage.aspect.upstreams) == 1 + + +def test_extract_panel_lineage_prometheus(lineage_extractor): + panel = Panel( + id="1", + title="Test Panel", + type="graph", + datasource={"type": "prometheus", "uid": "prom_uid"}, + targets=[{"expr": "rate(http_requests_total[5m])"}], + ) + + lineage = lineage_extractor.extract_panel_lineage(panel) + assert lineage is None + + +def test_create_basic_lineage(lineage_extractor): + ds_uid = "postgres_uid" + ds_urn = make_dataset_urn_with_platform_instance( + platform="grafana", + name="test_dataset", + platform_instance="test-instance", + env="PROD", + ) + + platform_config = PlatformConnectionConfig( + platform="postgres", + database="test_db", + database_schema="public", + ) + + lineage = lineage_extractor._create_basic_lineage(ds_uid, platform_config, ds_urn) + + assert isinstance(lineage, MetadataChangeProposalWrapper) + assert isinstance(lineage.aspect, UpstreamLineageClass) + assert len(lineage.aspect.upstreams) == 1 + + +def test_create_column_lineage(lineage_extractor, mock_graph): + mock_parsed_sql = MagicMock() + mock_parsed_sql.in_tables = [ + "urn:li:dataset:(postgres,test_db.public.test_table,PROD)" + ] + mock_parsed_sql.column_lineage = [ + MagicMock( + downstream=MagicMock(column="test_col"), + upstreams=[MagicMock(column="source_col")], + ) + ] + + ds_urn = make_dataset_urn_with_platform_instance( + platform="grafana", + name="test_dataset", + platform_instance="test-instance", + env="PROD", + ) + + lineage = lineage_extractor._create_column_lineage(ds_urn, mock_parsed_sql) + assert isinstance(lineage, MetadataChangeProposalWrapper) + assert isinstance(lineage.aspect, UpstreamLineageClass) + assert lineage.aspect.fineGrainedLineages is not None diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_models.py b/metadata-ingestion/tests/unit/grafana/test_grafana_models.py new file mode 100644 index 00000000000000..3f5093673cf140 --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_models.py @@ -0,0 +1,95 @@ +from typing import Any, Dict + +from datahub.ingestion.source.grafana.models import ( + Dashboard, + Folder, + Panel, +) + + +def test_panel_basic(): + panel_data: Dict[str, Any] = { + "id": "1", + "title": "Test Panel", + "description": "Test Description", + "type": "graph", + "targets": [], + "datasource": None, + "fieldConfig": {}, + "transformations": [], + } + + panel = Panel.parse_obj(panel_data) + assert panel.id == "1" + assert panel.title == "Test Panel" + assert panel.description == "Test Description" + assert panel.type == "graph" + assert len(panel.targets) == 0 + + +def test_panel_with_datasource(): + panel_data = { + "id": "1", + "title": "Test Panel", + "datasource": {"type": "postgres", "uid": "abc123"}, + } + + panel = Panel.parse_obj(panel_data) + assert panel.datasource == {"type": "postgres", "uid": "abc123"} + + +def test_dashboard_basic(): + dashboard_data = { + "dashboard": { + "uid": "dash1", + "title": "Test Dashboard", + "description": "Test Description", + "version": "1", + "panels": [], + "tags": ["test"], + "timezone": "utc", + "schemaVersion": "1.0", + "meta": {"folderId": "123"}, + } + } + + dashboard = Dashboard.parse_obj(dashboard_data) + assert dashboard.uid == "dash1" + assert dashboard.title == "Test Dashboard" + assert dashboard.version == "1" + assert dashboard.tags == ["test"] + + +def test_dashboard_nested_panels(): + dashboard_data = { + "dashboard": { + "uid": "dash1", + "title": "Test Dashboard", + "description": "", + "version": "1", + "timezone": "utc", + "schemaVersion": "1.0", + "panels": [ + { + "type": "row", + "panels": [{"id": "1", "title": "Nested Panel", "type": "graph"}], + }, + {"id": "2", "title": "Top Level Panel", "type": "graph"}, + ], + "tags": [], + } + } + + dashboard = Dashboard.parse_obj(dashboard_data) + assert len(dashboard.panels) == 2 + assert dashboard.panels[0].title == "Nested Panel" + assert dashboard.panels[1].title == "Top Level Panel" + + +def test_folder(): + folder_data = {"id": "1", "title": "Test Folder", "description": "Test Description"} + + folder = Folder.parse_obj(folder_data) + assert folder.id == "1" + assert folder.title == "Test Folder" + assert folder.description == "Test Description" diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_report.py b/metadata-ingestion/tests/unit/grafana/test_grafana_report.py new file mode 100644 index 00000000000000..f7f4021e45f955 --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_report.py @@ -0,0 +1,55 @@ +from datahub.ingestion.source.grafana.report import GrafanaSourceReport + + +def test_grafana_report_initialization(): + report = GrafanaSourceReport() + assert report.dashboards_scanned == 0 + assert report.charts_scanned == 0 + assert report.folders_scanned == 0 + assert report.datasets_scanned == 0 + + +def test_report_dashboard_scanned(): + report = GrafanaSourceReport() + report.report_dashboard_scanned() + assert report.dashboards_scanned == 1 + report.report_dashboard_scanned() + assert report.dashboards_scanned == 2 + + +def test_report_chart_scanned(): + report = GrafanaSourceReport() + report.report_chart_scanned() + assert report.charts_scanned == 1 + report.report_chart_scanned() + assert report.charts_scanned == 2 + + +def test_report_folder_scanned(): + report = GrafanaSourceReport() + report.report_folder_scanned() + assert report.folders_scanned == 1 + report.report_folder_scanned() + assert report.folders_scanned == 2 + + +def test_report_dataset_scanned(): + report = GrafanaSourceReport() + report.report_dataset_scanned() + assert report.datasets_scanned == 1 + report.report_dataset_scanned() + assert report.datasets_scanned == 2 + + +def test_multiple_report_types(): + report = GrafanaSourceReport() + + report.report_dashboard_scanned() + report.report_chart_scanned() + report.report_folder_scanned() + report.report_dataset_scanned() + + assert report.dashboards_scanned == 1 + assert report.charts_scanned == 1 + assert report.folders_scanned == 1 + assert report.datasets_scanned == 1 diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_snapshots.py b/metadata-ingestion/tests/unit/grafana/test_grafana_snapshots.py new file mode 100644 index 00000000000000..85147e6bc7171e --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_snapshots.py @@ -0,0 +1,185 @@ +import pytest + +from datahub.ingestion.source.grafana.models import Dashboard, Panel +from datahub.ingestion.source.grafana.snapshots import ( + _build_custom_properties, + _build_dashboard_properties, + _build_ownership, + build_chart_mce, + build_dashboard_mce, +) +from datahub.metadata.schema_classes import ( + ChartInfoClass, + DashboardInfoClass, + GlobalTagsClass, + OwnershipClass, + StatusClass, +) + + +@pytest.fixture +def mock_panel(): + return Panel( + id="1", + title="Test Panel", + description="Test Description", + type="graph", + targets=[{"query": "SELECT * FROM test"}], + datasource={"type": "mysql", "uid": "test_uid"}, + ) + + +@pytest.fixture +def mock_dashboard(): + return Dashboard( + uid="dash1", + title="Test Dashboard", + description="Test Description", + version="1", + panels=[], + tags=["tag1", "environment:prod"], + timezone="UTC", + schemaVersion="1.0", + meta={"folderId": "123"}, + created_by="test@test.com", + ) + + +def test_build_custom_properties(): + panel = Panel( + id="1", + title="Test Panel", + type="graph", + description="Test Description", + targets=[{"query": "test"}], + datasource={"type": "mysql", "uid": "test_uid"}, + ) + + props = _build_custom_properties(panel) + assert props["type"] == "graph" + assert props["datasourceType"] == "mysql" + assert props["datasourceUid"] == "test_uid" + assert props["description"] == "Test Description" + assert props["queryCount"] == "1" + + +def test_build_dashboard_properties(mock_dashboard): + props = _build_dashboard_properties(mock_dashboard) + assert props["version"] == "1" + assert props["schema_version"] == "1.0" + assert props["timezone"] == "UTC" + + +def test_build_ownership(mock_dashboard): + ownership = _build_ownership(mock_dashboard) + assert isinstance(ownership, OwnershipClass) + assert len(ownership.owners) == 2 + assert any(owner.owner.split(":")[-1] == "dash1" for owner in ownership.owners) + assert any(owner.owner.split(":")[-1] == "test" for owner in ownership.owners) + + +def test_build_chart_mce(mock_panel, mock_dashboard): + dataset_urn, chart_mce = build_chart_mce( + panel=mock_panel, + dashboard=mock_dashboard, + platform="grafana", + platform_instance="test-instance", + env="PROD", + base_url="http://grafana.test", + ingest_tags=True, + ) + + assert dataset_urn is not None + assert chart_mce.urn.startswith("urn:li:chart") + + chart_info = next( + aspect for aspect in chart_mce.aspects if isinstance(aspect, ChartInfoClass) + ) + assert chart_info.title == "Test Panel" + assert chart_info.description == "Test Description" + assert ( + chart_info.chartUrl is not None and "http://grafana.test" in chart_info.chartUrl + ) + assert chart_info.inputs is not None and dataset_urn in chart_info.inputs + + status = next( + aspect for aspect in chart_mce.aspects if isinstance(aspect, StatusClass) + ) + assert status.removed is False + + +def test_build_dashboard_mce(mock_dashboard): + chart_urns = ["urn:li:chart:(grafana,chart1)", "urn:li:chart:(grafana,chart2)"] + + dashboard_mce = build_dashboard_mce( + dashboard=mock_dashboard, + platform="grafana", + platform_instance="test-instance", + chart_urns=chart_urns, + base_url="http://grafana.test", + ingest_owners=True, + ingest_tags=True, + ) + + assert dashboard_mce.urn.startswith("urn:li:dashboard") + + dashboard_info = next( + aspect + for aspect in dashboard_mce.aspects + if isinstance(aspect, DashboardInfoClass) + ) + assert dashboard_info.title == "Test Dashboard" + assert dashboard_info.description == "Test Description" + assert dashboard_info.charts is not None and set(dashboard_info.charts) == set( + chart_urns + ) + assert ( + dashboard_info.dashboardUrl is not None + and "http://grafana.test" in dashboard_info.dashboardUrl + ) + + tags = next( + aspect + for aspect in dashboard_mce.aspects + if isinstance(aspect, GlobalTagsClass) + ) + assert len(tags.tags) == 2 + + ownership = next( + aspect for aspect in dashboard_mce.aspects if isinstance(aspect, OwnershipClass) + ) + assert len(ownership.owners) == 2 + + +def test_build_chart_mce_no_tags(mock_panel, mock_dashboard): + mock_dashboard.tags = [] # Ensure it's an empty list rather than None + dataset_urn, chart_mce = build_chart_mce( + panel=mock_panel, + dashboard=mock_dashboard, + platform="grafana", + platform_instance="test-instance", + env="PROD", + base_url="http://grafana.test", + ingest_tags=True, + ) + + assert not any(isinstance(aspect, GlobalTagsClass) for aspect in chart_mce.aspects) + + +def test_build_dashboard_mce_no_owners(mock_dashboard): + mock_dashboard.created_by = "" + mock_dashboard.uid = "" + + dashboard_mce = build_dashboard_mce( + dashboard=mock_dashboard, + platform="grafana", + platform_instance="test-instance", + chart_urns=[], + base_url="http://grafana.test", + ingest_owners=True, + ingest_tags=True, + ) + + assert not any( + isinstance(aspect, OwnershipClass) for aspect in dashboard_mce.aspects + ) diff --git a/metadata-ingestion/tests/unit/grafana/test_grafana_source.py b/metadata-ingestion/tests/unit/grafana/test_grafana_source.py new file mode 100644 index 00000000000000..ef3a2e0b055681 --- /dev/null +++ b/metadata-ingestion/tests/unit/grafana/test_grafana_source.py @@ -0,0 +1,152 @@ +from unittest.mock import patch + +import pytest + +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.grafana.grafana_config import GrafanaSourceConfig +from datahub.ingestion.source.grafana.grafana_source import GrafanaSource +from datahub.ingestion.source.grafana.models import Dashboard, Folder, Panel + + +@pytest.fixture +def mock_config(): + return GrafanaSourceConfig( + url="http://grafana.test", + service_account_token="test-token", + platform_instance="test-instance", + ) + + +@pytest.fixture +def mock_context(): + return PipelineContext(run_id="test") + + +@pytest.fixture +def mock_source(mock_config, mock_context): + return GrafanaSource(mock_config, mock_context) + + +@pytest.fixture +def mock_folder(): + return Folder(id="1", title="Test Folder", description="Test Description") + + +@pytest.fixture +def mock_dashboard(): + return Dashboard( + uid="dash1", + title="Test Dashboard", + description="Test Description", + version="1", + panels=[ + Panel( + id="1", + title="Test Panel", + type="graph", + datasource={"type": "postgres", "uid": "postgres_uid"}, + targets=[{"rawSql": "SELECT * FROM test_table"}], + ) + ], + tags=["test"], + schemaVersion="1.0", + meta={"folderId": "1"}, + ) + + +def test_source_initialization(mock_source): + assert mock_source.platform == "grafana" + assert mock_source.config.url == "http://grafana.test" + assert mock_source.platform_instance == "test-instance" + + +@patch("datahub.ingestion.source.grafana.grafana_source.GrafanaAPIClient") +def test_process_folder(mock_api, mock_source, mock_folder): + workunit_list = list(mock_source._process_folder(mock_folder)) + + assert len(workunit_list) > 0 + assert all(isinstance(w, MetadataWorkUnit) for w in workunit_list) + + +@patch("datahub.ingestion.source.grafana.grafana_source.GrafanaAPIClient") +def test_process_dashboard(mock_api, mock_source, mock_dashboard): + workunit_list = list(mock_source._process_dashboard(mock_dashboard)) + + assert len(workunit_list) > 0 + assert all(isinstance(w, MetadataWorkUnit) for w in workunit_list) + assert mock_source.report.charts_scanned == 1 + + +@patch("datahub.ingestion.source.grafana.grafana_source.GrafanaAPIClient") +def test_process_panel_dataset(mock_api, mock_source, mock_dashboard): + panel = mock_dashboard.panels[0] + workunit_list = list( + mock_source._process_panel_dataset( + panel=panel, dashboard_uid=mock_dashboard.uid, ingest_tags=True + ) + ) + + assert len(workunit_list) > 0 + assert all(isinstance(w, MetadataWorkUnit) for w in workunit_list) + assert mock_source.report.datasets_scanned == 1 + + +@patch("datahub.ingestion.source.grafana.grafana_source.GrafanaAPIClient") +def test_process_dashboard_with_folder(mock_api, mock_source, mock_dashboard): + workunit_list = list(mock_source._process_dashboard(mock_dashboard)) + + assert len(workunit_list) > 0 + # Verify folder container relationship is created + container_workunits = [wu for wu in workunit_list if "container" in wu.id] + assert len(container_workunits) > 0 + + +@patch("datahub.ingestion.source.grafana.grafana_source.GrafanaAPIClient") +def test_source_get_workunits_internal( + mock_api, mock_source, mock_folder, mock_dashboard +): + # Create a mock API client instance + mock_api_instance = mock_api.return_value + mock_api_instance.get_folders.return_value = [mock_folder] + mock_api_instance.get_dashboards.return_value = [mock_dashboard] + + # Set the mock API client on the source + mock_source.client = mock_api_instance + + # Run the test + workunit_list = list(mock_source.get_workunits_internal()) + + # Verify results + assert len(workunit_list) > 0 + assert mock_source.report.folders_scanned == 1 + assert mock_source.report.dashboards_scanned == 1 + assert mock_source.report.charts_scanned == 1 + + # Verify API calls + mock_api_instance.get_folders.assert_called_once() + mock_api_instance.get_dashboards.assert_called_once() + + +def test_source_get_report(mock_source): + report = mock_source.get_report() + assert report.dashboards_scanned == 0 + assert report.charts_scanned == 0 + assert report.folders_scanned == 0 + assert report.datasets_scanned == 0 + + +@patch("datahub.ingestion.source.grafana.grafana_source.GrafanaAPIClient") +def test_source_close(mock_api, mock_source): + mock_source.close() + # Verify any cleanup is performed correctly + + +def test_source_platform_instance_none(): + config = GrafanaSourceConfig( + url="http://grafana.test", + service_account_token="test-token", + ) + ctx = PipelineContext(run_id="test") + source = GrafanaSource(config, ctx) + assert source.platform_instance is None