-
Notifications
You must be signed in to change notification settings - Fork 3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(ingestion/grafana): Add datasets and charts to dashboards with lineage and tags. Lineage back to source #12417
Open
acrylJonny
wants to merge
33
commits into
datahub-project:master
Choose a base branch
from
acrylJonny:grafana-improvements
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+6,379
−184
Open
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit
Hold shift + click to select a range
d4be2de
initial commit
acrylJonny 660daf8
Delete grafana2
acrylJonny 6cfbf5f
Merge branch 'master' into grafana-improvements
acrylJonny 23e4eca
second commit
acrylJonny 5468909
updating tests and adding tags and ownership options
acrylJonny e684eed
Merge branch 'master' into grafana-improvements
acrylJonny eda52c4
error updates and better formatting of docs
acrylJonny 507c811
Update grafana_api.py
acrylJonny 0e9f0dc
updating to use basemodel. Cleaning up code
acrylJonny c832e91
test updates
acrylJonny 0fe68d2
Merge branch 'master' into grafana-improvements
acrylJonny c0e63d7
reorder libraries to be alphabetical
acrylJonny 64a3bae
Update grafana_source.py
acrylJonny 09b1ed8
adding unit tests
acrylJonny b06fd12
Merge branch 'master' into grafana-improvements
acrylJonny e0533ad
Adding optionals
acrylJonny 27a9ceb
removing sqlparse requirement
acrylJonny a7e9e19
test updates
acrylJonny 4714b24
Merge branch 'master' into grafana-improvements
acrylJonny f0f88c6
Merge branch 'master' into grafana-improvements
acrylJonny 7646b89
addressing comments
acrylJonny 6bd09ed
Merge branch 'master' into grafana-improvements
acrylJonny 51310f9
addressing comments
acrylJonny 54d142f
Improving overall experience
acrylJonny b4b7409
Merge branch 'master' into grafana-improvements
acrylJonny b4da306
Update constants.ts
acrylJonny 40b75b8
Merge branch 'grafana-improvements' of https://github.com/acrylJonny/…
acrylJonny 9bd5e7d
Merge branch 'master' into grafana-improvements
acrylJonny 1dbe6f9
Update grafana_pre.md
acrylJonny 8e150cd
test file updates
acrylJonny addfd15
Merge branch 'master' into grafana-improvements
acrylJonny 40c2ff9
Merge branch 'master' into grafana-improvements
acrylJonny ac2e4cd
Merge branch 'master' into grafana-improvements
acrylJonny File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
1,188 changes: 1,117 additions & 71 deletions
1,188
metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py
Large diffs are not rendered by default.
Oops, something went wrong.
295 changes: 295 additions & 0 deletions
295
metadata-ingestion/tests/integration/grafana/dashboards/default-dashboard.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,295 @@ | ||
{ | ||
"id": null, | ||
"uid": "default", | ||
"title": "Test Integration Dashboard", | ||
"tags": ["test-tag", "integration-test"], | ||
"timezone": "browser", | ||
"schemaVersion": 36, | ||
"version": 0, | ||
"panels": [ | ||
{ | ||
"id": 1, | ||
"type": "text", | ||
"title": "Dashboard Information", | ||
"gridPos": { | ||
"x": 0, | ||
"y": 0, | ||
"w": 24, | ||
"h": 3 | ||
}, | ||
"options": { | ||
"content": "# Test Integration Dashboard\nThis dashboard contains test panels for DataHub integration testing with both PostgreSQL metrics and Prometheus system metrics.", | ||
"mode": "markdown" | ||
} | ||
}, | ||
{ | ||
"id": 2, | ||
"type": "timeseries", | ||
"title": "Response Times by Dimension", | ||
"description": "Response times tracked across different dimensions", | ||
"datasource": { | ||
"type": "postgres", | ||
"uid": "test-postgres" | ||
}, | ||
"fieldConfig": { | ||
"defaults": { | ||
"custom": { | ||
"drawStyle": "line", | ||
"lineInterpolation": "smooth", | ||
"spanNulls": false | ||
}, | ||
"color": { | ||
"mode": "palette-classic" | ||
}, | ||
"unit": "ms" | ||
} | ||
}, | ||
"gridPos": { | ||
"h": 8, | ||
"w": 12, | ||
"x": 0, | ||
"y": 3 | ||
}, | ||
"targets": [ | ||
{ | ||
"datasource": { | ||
"type": "postgres", | ||
"uid": "test-postgres" | ||
}, | ||
"editorMode": "code", | ||
"format": "time_series", | ||
"rawQuery": true, | ||
"rawSql": "SELECT time, value, dimension FROM test_metrics WHERE metric = 'response_time' AND time > NOW() - interval '1 hour' ORDER BY time ASC;", | ||
"refId": "A", | ||
"sql": { | ||
"columns": [ | ||
{ | ||
"parameters": [], | ||
"type": "time" | ||
}, | ||
{ | ||
"parameters": [ | ||
{ | ||
"name": "value", | ||
"type": "column" | ||
} | ||
], | ||
"type": "value" | ||
} | ||
] | ||
} | ||
} | ||
] | ||
}, | ||
{ | ||
"id": 4, | ||
"type": "table", | ||
"title": "Recent Metrics", | ||
"description": "Recent metrics from all sources", | ||
"datasource": { | ||
"type": "postgres", | ||
"uid": "test-postgres" | ||
}, | ||
"fieldConfig": { | ||
"defaults": { | ||
"custom": { | ||
"align": "auto", | ||
"displayMode": "auto" | ||
} | ||
}, | ||
"overrides": [ | ||
{ | ||
"matcher": { | ||
"id": "byName", | ||
"options": "value" | ||
}, | ||
"properties": [ | ||
{ | ||
"id": "custom.width", | ||
"value": 150 | ||
} | ||
] | ||
} | ||
] | ||
}, | ||
"gridPos": { | ||
"h": 8, | ||
"w": 24, | ||
"x": 0, | ||
"y": 11 | ||
}, | ||
"targets": [ | ||
{ | ||
"datasource": { | ||
"type": "postgres", | ||
"uid": "test-postgres" | ||
}, | ||
"editorMode": "code", | ||
"format": "table", | ||
"rawQuery": true, | ||
"rawSql": "SELECT time, metric, value, dimension FROM test_metrics WHERE time > NOW() - interval '1 hour' ORDER BY time DESC LIMIT 10;", | ||
"refId": "A", | ||
"sql": { | ||
"columns": [ | ||
{ | ||
"parameters": [], | ||
"type": "time" | ||
}, | ||
{ | ||
"parameters": [ | ||
{ | ||
"name": "metric", | ||
"type": "column" | ||
} | ||
], | ||
"type": "string" | ||
}, | ||
{ | ||
"parameters": [ | ||
{ | ||
"name": "value", | ||
"type": "column" | ||
} | ||
], | ||
"type": "number" | ||
}, | ||
{ | ||
"parameters": [ | ||
{ | ||
"name": "dimension", | ||
"type": "column" | ||
} | ||
], | ||
"type": "string" | ||
} | ||
] | ||
} | ||
} | ||
] | ||
}, | ||
{ | ||
"id": 5, | ||
"type": "stat", | ||
"title": "Total Metrics Count", | ||
"description": "Total number of metrics collected", | ||
"datasource": { | ||
"type": "postgres", | ||
"uid": "test-postgres" | ||
}, | ||
"fieldConfig": { | ||
"defaults": { | ||
"mappings": [], | ||
"thresholds": { | ||
"mode": "absolute", | ||
"steps": [ | ||
{ | ||
"color": "green", | ||
"value": null | ||
} | ||
] | ||
} | ||
} | ||
}, | ||
"gridPos": { | ||
"h": 8, | ||
"w": 12, | ||
"x": 12, | ||
"y": 3 | ||
}, | ||
"targets": [ | ||
{ | ||
"datasource": { | ||
"type": "postgres", | ||
"uid": "test-postgres" | ||
}, | ||
"editorMode": "code", | ||
"format": "table", | ||
"rawQuery": true, | ||
"rawSql": "SELECT COUNT(*) as count FROM test_metrics WHERE time > NOW() - interval '1 hour';", | ||
"refId": "A", | ||
"sql": { | ||
"columns": [ | ||
{ | ||
"parameters": [ | ||
{ | ||
"name": "count", | ||
"type": "column" | ||
} | ||
], | ||
"type": "number" | ||
} | ||
] | ||
} | ||
} | ||
] | ||
}, | ||
{ | ||
"id": 6, | ||
"type": "timeseries", | ||
"title": "System Metrics", | ||
"description": "Prometheus system metrics", | ||
"datasource": { | ||
"type": "prometheus", | ||
"uid": "test-prometheus" | ||
}, | ||
"fieldConfig": { | ||
"defaults": { | ||
"custom": { | ||
"drawStyle": "line", | ||
"lineWidth": 1, | ||
"fillOpacity": 10 | ||
} | ||
} | ||
}, | ||
"gridPos": { | ||
"h": 8, | ||
"w": 24, | ||
"x": 0, | ||
"y": 19 | ||
}, | ||
"targets": [ | ||
{ | ||
"datasource": { | ||
"type": "prometheus", | ||
"uid": "test-prometheus" | ||
}, | ||
"editorMode": "code", | ||
"expr": "rate(process_cpu_seconds_total[5m])", | ||
"legendFormat": "CPU Usage", | ||
"range": true, | ||
"refId": "A" | ||
}, | ||
{ | ||
"datasource": { | ||
"type": "prometheus", | ||
"uid": "test-prometheus" | ||
}, | ||
"editorMode": "code", | ||
"expr": "go_memstats_alloc_bytes", | ||
"legendFormat": "Memory Usage", | ||
"range": true, | ||
"refId": "B" | ||
} | ||
] | ||
} | ||
], | ||
"refresh": "5s", | ||
"time": { | ||
"from": "now-1h", | ||
"to": "now" | ||
}, | ||
"timepicker": { | ||
"refresh_intervals": [ | ||
"5s", | ||
"10s", | ||
"30s", | ||
"1m", | ||
"5m", | ||
"15m", | ||
"30m", | ||
"1h" | ||
] | ||
}, | ||
"timezone": "browser", | ||
"description": "A comprehensive test dashboard for integration testing with various panel types and data sources" | ||
} |
25 changes: 0 additions & 25 deletions
25
metadata-ingestion/tests/integration/grafana/default-dashboard.json
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Looks good!
Thinking on the future troubleshooting, adding more instrumentation may help (some metrics in the report or info/debug log traces)