Skip to content

Add Spark single application monitoring public API swagger files - Spark Advisor API #166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Add Spark single application monitoring public API swagger files - Spark Advisor API #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
901a9b0
Add Spark single application monitoring public API swagger files - Sp…
May 23, 2025
0617a67
Add "detail" property and related definitions
Jun 4, 2025
0b1be8a
Wording changes, including advices -> advice
Jun 4, 2025
496c69e
Change onto yaml
Jun 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
347 changes: 347 additions & 0 deletions docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,6 +772,190 @@
}
}
}
},
"/v1/workspaces/{workspaceId}/{itemType}/{itemId}/livySessions/{livyId}/applications/{appId}/{attemptId}/advice": {
"get": {
"tags": [
"SparkAdvisor"
],
"summary": "Get advice list",
"operationId": "SparkAdvisor_List",
"produces": [
"application/json"
],
"parameters": [
{
"name": "workspaceId",
"in": "path",
"required": true,
"type": "string",
"format": "uuid"
},
{
"name": "itemType",
"in": "path",
"required": true,
"type": "string",
"enum": ["notebooks", "sparkJobDefinitions", "lakehouses"]
},
{
"name": "itemId",
"in": "path",
"required": true,
"type": "string",
"format": "uuid"
},
{
"name": "livyId",
"in": "path",
"required": true,
"type": "string",
"format": "uuid"
},
{
"name": "appId",
"in": "path",
"required": true,
"type": "string"
},
{
"name": "attemptId",
"in": "path",
"required": false,
"type": "integer",
"format": "int32"
},
{
"name": "stageId",
"in": "query",
"description": "The specific stage ID to get advice on",
"required": false,
"type": "integer",
"format": "int64"
},
{
"name": "jobId",
"in": "query",
"description": "The specific job ID to get advice on",
"required": false,
"type": "integer",
"format": "int64"
},
{
"name": "exectionId",
"in": "query",
"description": "The specific exection (SQL query) ID to get advice on",
"required": false,
"type": "integer",
"format": "int64"
},
{
"name": "jobGroupId",
"in": "query",
"description": "The specific job group (statement) ID to get advice on",
"required": false,
"type": "string"
},
{
"name": "executorId",
"in": "query",
"description": "The specific executor ID to get advice on",
"required": false,
"type": "string"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/AdviseDataJson"
}
}
}
}
}
},
"/v1/workspaces/{workspaceId}/{itemType}/{itemId}/livySessions/{livyId}/applications/{appId}/{attemptId}/advice/{filter}/{byId}": {
"get": {
"tags": [
"SparkAdvisor"
],
"summary": "Get advice by filter",
"operationId": "SparkAdvisor_ByFilter",
"produces": [
"application/json"
],
"parameters": [
{
"name": "workspaceId",
"in": "path",
"required": true,
"type": "string",
"format": "uuid"
},
{
"name": "itemType",
"in": "path",
"required": true,
"type": "string",
"enum": ["notebooks", "sparkJobDefinitions", "lakehouses"]
},
{
"name": "itemId",
"in": "path",
"required": true,
"type": "string",
"format": "uuid"
},
{
"name": "livyId",
"in": "path",
"required": true,
"type": "string",
"format": "uuid"
},
{
"name": "appId",
"in": "path",
"required": true,
"type": "string"
},
{
"name": "attemptId",
"in": "path",
"required": false,
"type": "integer",
"format": "int32"
},
{
"name": "filter",
"in": "path",
"required": true,
"type": "string",
"enum": ["stages", "jobs", "executions", "jobGroups", "executors"]
},
{
"name": "byId",
"in": "path",
"description": "The specific ID to filter. Type is \"int64\" if it follows the filter of \"stages\", \"jobs\" or \"executions\"; Type is \"string\" if it follows the filter of \"jobGroups\" or \"executors\"",
"required": true,
"type": "object"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/AdviseDataJson"
}
}
}
}
}
}
},
"definitions": {
Expand Down Expand Up @@ -1049,6 +1233,169 @@
"$ref": "#/definitions/ResourceUsageSnapshotData"
}
}
},
"AdviseDataJson": {
"type": "object",
"properties": {
"id": {
"description": "The ID of the advice",
"type": "integer",
"format": "int64"
},
"name": {
"description": "The name of the advice",
"type": "string"
},
"description": {
"description": "The description of the advice",
"type": "string"
},
"helplink": {
"description": "The web page or documentation to reference or help further",
"type": "string"
},
"detail": {
"description": "More details in rich text format",
"$ref": "#/definitions/JsonAdviseDetail"
},
"source": {
"description": "Could be \"System\", \"User\" or \"Dependency\"",
"type": "string"
},
"level": {
"description": "Could be \"info\", \"warn\" or \"error\"",
"type": "string"
},
"stageId": {
"description": "The stage ID which the advice is provided on. Ignore stageId if its value is -1",
"type": "integer",
"format": "int64"
},
"jobId": {
"description": "The job ID which the advice is provided on. Ignore jobId if its value is -1",
"type": "integer",
"format": "int64"
},
"executionId": {
"description": "The execution (SQL query) ID which the advice is provided on. Ignore executionId if its value is -1",
"type": "integer",
"format": "int64"
},
"jobGroupId": {
"description": "The job group (statement) ID which the advice is provided on",
"type": "string"
},
"executorId": {
"description": "The executor ID which the advice is provided on",
"type": "string"
}
}
},
"JsonAdviseDetail": {
"type": "object",
"properties": {
"data": {
"oneOf": {
"$ref": "#/definitions/TaskError",
"$ref": "#/definitions/DataSkew",
"$ref": "#/definitions/TimeSkew"
}
}
}
},
"TaskError": {
"type": "object",
"properties": {
"name": {
"description": "The name of the error or exception",
"type": "string"
},
"tsg": {
"description": "The trouble shooting guide to follow",
"type": "string"
},
"taskTypes": {
"description": "The types of Spark tasks, could be \"ResultTask\", \"ShuffleMapTask\"...",
"type": "string"
},
"helplink": {
"description": "The web page or documentation to reference or help further",
"type": "string"
},
"executorIds": {
"description": "The executor IDs",
"type": "string"
},
"stageIds": {
"description": "The stage IDs",
"type": "string"
},
"count": {
"description": "The task count",
"type": "integer",
"format": "int32"
}
}
},
"DataSkew": {
"type": "object",
"properties": {
"name": {
"description": "The stage name",
"type": "string"
},
"stageId": {
"description": "The stage ID",
"type": "string"
},
"skewedTaskPercentage": {
"description": "The percentage of skewed tasks",
"type": "string"
},
"maxDataRead": {
"description": "Max data read, in MiB",
"type": "string"
},
"meanDataRead": {
"description": "Mean data read, in MiB",
"type": "string"
},
"taskDataReadSkewness": {
"description": "The data read skewness of tasks within the stage, measured by \"Pearson's Second Coefficient of Skewness\"",
"type": "number",
"format": "double"
}
}
},
"TimeSkew": {
"type": "object",
"properties": {
"name": {
"description": "The stage name",
"type": "string"
},
"stageId": {
"description": "The stage ID",
"type": "string"
},
"skewedTaskPercentage": {
"description": "The percentage of skewed tasks",
"type": "string"
},
"maxTaskDuration": {
"description": "Max duration, in second",
"type": "string"
},
"meanTaskDuration": {
"description": "Mean duration, in second",
"type": "string"
},
"taskDurationSkewness": {
"description": "The duration skewness of tasks within the stage, measured by \"Pearson's Second Coefficient of Skewness\"",
"type": "number",
"format": "double"
}
}
}
}
}
Loading