feat(ingestion/grafana): Add datasets and charts to dashboards with lineage and tags. Lineage back to source

datahub-web-react/src/app/ingest/source/builder/constants.ts

@@ -15,6 +15,7 @@ import elasticsearchLogo from '@images/elasticsearchlogo.png';
 import feastLogo from '@images/feastlogo.png';
 import fivetranLogo from '@images/fivetranlogo.png';
 import glueLogo from '@images/gluelogo.png';
+import grafanaLogo from '@images/grafana.png';
 import hiveLogo from '@images/hivelogo.png';
 import kafkaLogo from '@images/kafkalogo.png';
 import lookerLogo from '@images/lookerlogo.svg';
@@ -70,6 +71,8 @@ export const FEAST_LEGACY = 'feast-legacy';
 export const FEAST_URN = `urn:li:dataPlatform:${FEAST}`;
 export const GLUE = 'glue';
 export const GLUE_URN = `urn:li:dataPlatform:${GLUE}`;
+export const GRAFANA = 'grafana';
+export const GRAFANA_URN = `urn:li:dataPlatform:${GRAFANA}`;
 export const HIVE = 'hive';
 export const HIVE_URN = `urn:li:dataPlatform:${HIVE}`;
 export const KAFKA = 'kafka';
@@ -160,6 +163,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [ELASTICSEARCH_URN]: elasticsearchLogo,
     [FEAST_URN]: feastLogo,
     [GLUE_URN]: glueLogo,
+    [GRAFANA_URN]: grafanaLogo,
     [HIVE_URN]: hiveLogo,
     [KAFKA_URN]: kafkaLogo,
     [LOOKER_URN]: lookerLogo,

datahub-web-react/src/app/ingest/source/builder/sources.json

@@ -349,6 +349,13 @@
         "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/hex/",
         "recipe": "source:\n  type: hex\n  config:\n    workspace_name: # Your Hex Workspace name\n     token: # Your PAT or Workspace token"
     },
+    {
+        "urn": "urn:li:dataPlatform:grafana",
+        "name": "grafana",
+        "displayName": "Grafana",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/grafana/",
+        "recipe": "source:\n    type: \"grafana\"\n    config:\n        url: \"https://grafana.company.com:443\"\n        service_account_token: token\n        verify_ssl: true"
+    },
     {
         "urn": "urn:li:dataPlatform:preset",
         "name": "preset",

metadata-ingestion/docs/sources/grafana/grafana_pre.md

@@ -0,0 +1,110 @@
+### Concept Mapping
+
+| Source Concept              | DataHub Concept                                           | Notes                                                                    |
+| --------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------------ |
+| `"grafana"`                 | [Data Platform](../../metamodel/entities/dataPlatform.md) |                                                                          |
+| Grafana Folder              | [Container](../../metamodel/entities/container.md)        | Subtype `Folder`                                                         |
+| Grafana Dashboard           | [Container](../../metamodel/entities/container.md)        | Subtype `Dashboard`                                                      |
+| Grafana Panel/Visualization | [Chart](../../metamodel/entities/chart.md)                | Various types mapped based on panel type (e.g., graph → LINE, pie → PIE) |
+| Grafana Data Source         | [Dataset](../../metamodel/entities/dataset.md)            | Created for each panel's data source                                     |
+| Dashboard Owner             | [Corp User](../../metamodel/entities/corpuser.md)         | Derived from dashboard UID and creator                                   |
+| Dashboard Tags              | [Tag](../../metamodel/entities/tag.md)                    | Supports both simple tags and key:value tags                             |
+
+### Compatibility
+
+The connector supports extracting metadata from any Grafana instance accessible via API. For SQL-based data sources, column-level lineage can be extracted when the queries are parseable. The connector supports various panel types and their transformations, and can work with both standalone Grafana instances and those integrated with other platforms.
+
+For optimal lineage extraction from SQL-based data sources:
+
+- Database/schema information should be properly configured in the connection settings
+- The platform mapping (`connection_to_platform_map`) should be configured to match your data sources
+
+### Prerequisites:
+
+The Grafana source supports two extraction modes based on your permission level:
+
+#### Enhanced Mode (Default)
+
+For full metadata extraction including lineage, containers, and detailed panel information:
+
+1. A running Grafana instance
+2. A service account token with **Admin permissions** to:
+   - Read dashboards and folders
+   - Access data source configurations
+   - View user information
+   - Access detailed dashboard metadata
+   - Read panel configurations and transformations
+
+#### Basic Mode (Limited Permissions)
+
+For users with limited permissions who only need basic dashboard metadata:
+
+1. A running Grafana instance
+2. A service account token with **Viewer permissions** to:
+   - Read dashboards (via `/api/search` endpoint)
+   - Basic dashboard metadata access
+
+To enable basic mode, set `basic_mode: true` in your configuration. This provides backwards compatibility with the original simple connector behavior.
+
+**Note:** Basic mode extracts only dashboard entities without folder hierarchy, panel details, lineage information, or schema metadata. It's recommended to use enhanced mode when possible for complete metadata extraction.
+
+#### Configuration Examples
+
+Enhanced Mode (Default):
+
+```yaml
+source:
+  type: grafana
+  config:
+    url: "https://grafana.company.com"
+    service_account_token: "your_admin_token"
+    # basic_mode: false  # Default - full extraction
+```
+
+Basic Mode (Limited Permissions):
+
+```yaml
+source:
+  type: grafana
+  config:
+    url: "https://grafana.company.com"
+    service_account_token: "your_viewer_token"
+    basic_mode: true # Enable basic mode for limited permissions
+```
+
+#### Lineage Configuration
+
+The Grafana source can extract lineage information between charts and their data sources. You can control lineage extraction using these configuration options:
+
+```yaml
+source:
+  type: grafana
+  config:
+    url: "https://grafana.company.com"
+    service_account_token: "your_token"
+
+    # Lineage extraction (default: true)
+    include_lineage: true
+
+    # Column-level lineage from SQL queries (default: true)
+    # Only applicable when include_lineage is true
+    include_column_lineage: true
+
+    # Platform mappings for lineage extraction
+    connection_to_platform_map:
+      postgres_datasource_uid:
+        platform: postgres
+        platform_instance: my_postgres
+        env: PROD
+        database: analytics
+        database_schema: public
+```
+
+**Lineage Features:**
+
+- **Dataset-level lineage**: Links charts to their underlying data sources
+- **Column-level lineage**: Extracts field-to-field relationships from SQL queries
+- **Platform mapping**: Maps Grafana data sources to their actual platforms for accurate lineage
+- **SQL parsing**: Supports parsing of SQL queries for detailed lineage extraction
+
+**Performance Note:** Lineage extraction can be disabled (`include_lineage: false`) to improve ingestion performance when lineage information is not needed.

metadata-ingestion/docs/sources/grafana/grafana_recipe.yml

@@ -0,0 +1,26 @@
+source:
+  type: grafana
+  config:
+    # Coordinates
+    platform_instance: production # optional
+    env: PROD # optional
+    url: https://grafana.company.com
+    service_account_token: ${GRAFANA_SERVICE_ACCOUNT_TOKEN}
+
+    # SSL verification for HTTPS connections
+    verify_ssl: true # optional, default is true
+
+    # Source type mapping for lineage
+    connection_to_platform_map:
+      postgres:
+        platform: postgres
+        database: grafana # optional
+        database_schema: grafana # optional
+        platform_instance: database_2 # optional
+        env: PROD # optional
+      mysql_uid_1: # Grafana datasource UID
+        platform: mysql
+        platform_instance: database_1 # optional
+        database: my_database # optional
+sink:
+  # sink configs

metadata-ingestion/src/datahub/ingestion/autogenerated/capability_summary.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2025-07-14T09:20:09.632850+00:00",
+  "generated_at": "2025-07-24T11:08:33.966929+00:00",
   "generated_by": "metadata-ingestion/scripts/capability_summary.py",
   "plugin_details": {
     "abs": {
@@ -956,17 +956,47 @@
     },
     "grafana": {
       "capabilities": [
+        {
+          "capability": "LINEAGE_FINE",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
         {
           "capability": "DELETION_DETECTION",
-          "description": "Enabled by default via stateful ingestion",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
+        {
+          "capability": "OWNERSHIP",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
+        {
+          "capability": "TAGS",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
+        {
+          "capability": "PLATFORM_INSTANCE",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
+        {
+          "capability": "LINEAGE_COARSE",
+          "description": "Enabled by default",
           "subtype_modifier": null,
           "supported": true
         }
       ],
       "classname": "datahub.ingestion.source.grafana.grafana_source.GrafanaSource",
       "platform_id": "grafana",
       "platform_name": "Grafana",
-      "support_status": "TESTING"
+      "support_status": "CERTIFIED"
     },
     "hana": {
       "capabilities": [

metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py

@@ -69,6 +69,8 @@ class BIContainerSubTypes(StrEnum):
     SIGMA_WORKSPACE = "Sigma Workspace"
     SIGMA_WORKBOOK = "Sigma Workbook"
     MODE_COLLECTION = "Collection"
+    GRAFANA_FOLDER = "Folder"
+    GRAFANA_DASHBOARD = "Dashboard"
 
 
 class FlowContainerSubTypes(StrEnum):