diff --git a/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.json b/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.json index ce3565a5..89b1099c 100644 --- a/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.json +++ b/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.json @@ -772,6 +772,190 @@ } } } + }, + "/v1/workspaces/{workspaceId}/{itemType}/{itemId}/livySessions/{livyId}/applications/{appId}/{attemptId}/advice": { + "get": { + "tags": [ + "SparkAdvisor" + ], + "summary": "Get advice list", + "operationId": "SparkAdvisor_List", + "produces": [ + "application/json" + ], + "parameters": [ + { + "name": "workspaceId", + "in": "path", + "required": true, + "type": "string", + "format": "uuid" + }, + { + "name": "itemType", + "in": "path", + "required": true, + "type": "string", + "enum": ["notebooks", "sparkJobDefinitions", "lakehouses"] + }, + { + "name": "itemId", + "in": "path", + "required": true, + "type": "string", + "format": "uuid" + }, + { + "name": "livyId", + "in": "path", + "required": true, + "type": "string", + "format": "uuid" + }, + { + "name": "appId", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "attemptId", + "in": "path", + "required": false, + "type": "integer", + "format": "int32" + }, + { + "name": "stageId", + "in": "query", + "description": "The specific stage ID to get advice on", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "jobId", + "in": "query", + "description": "The specific job ID to get advice on", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "exectionId", + "in": "query", + "description": "The specific exection (SQL query) ID to get advice on", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "jobGroupId", + "in": "query", + "description": "The specific job group (statement) ID to get advice on", + "required": false, + "type": "string" + }, + { + "name": "executorId", + "in": "query", + "description": "The specific executor ID to get advice on", + "required": false, + "type": "string" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/AdviseDataJson" + } + } + } + } + } + }, + "/v1/workspaces/{workspaceId}/{itemType}/{itemId}/livySessions/{livyId}/applications/{appId}/{attemptId}/advice/{filter}/{byId}": { + "get": { + "tags": [ + "SparkAdvisor" + ], + "summary": "Get advice by filter", + "operationId": "SparkAdvisor_ByFilter", + "produces": [ + "application/json" + ], + "parameters": [ + { + "name": "workspaceId", + "in": "path", + "required": true, + "type": "string", + "format": "uuid" + }, + { + "name": "itemType", + "in": "path", + "required": true, + "type": "string", + "enum": ["notebooks", "sparkJobDefinitions", "lakehouses"] + }, + { + "name": "itemId", + "in": "path", + "required": true, + "type": "string", + "format": "uuid" + }, + { + "name": "livyId", + "in": "path", + "required": true, + "type": "string", + "format": "uuid" + }, + { + "name": "appId", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "attemptId", + "in": "path", + "required": false, + "type": "integer", + "format": "int32" + }, + { + "name": "filter", + "in": "path", + "required": true, + "type": "string", + "enum": ["stages", "jobs", "executions", "jobGroups", "executors"] + }, + { + "name": "byId", + "in": "path", + "description": "The specific ID to filter. Type is \"int64\" if it follows the filter of \"stages\", \"jobs\" or \"executions\"; Type is \"string\" if it follows the filter of \"jobGroups\" or \"executors\"", + "required": true, + "type": "object" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/AdviseDataJson" + } + } + } + } + } } }, "definitions": { @@ -1049,6 +1233,169 @@ "$ref": "#/definitions/ResourceUsageSnapshotData" } } + }, + "AdviseDataJson": { + "type": "object", + "properties": { + "id": { + "description": "The ID of the advice", + "type": "integer", + "format": "int64" + }, + "name": { + "description": "The name of the advice", + "type": "string" + }, + "description": { + "description": "The description of the advice", + "type": "string" + }, + "helplink": { + "description": "The web page or documentation to reference or help further", + "type": "string" + }, + "detail": { + "description": "More details in rich text format", + "$ref": "#/definitions/JsonAdviseDetail" + }, + "source": { + "description": "Could be \"System\", \"User\" or \"Dependency\"", + "type": "string" + }, + "level": { + "description": "Could be \"info\", \"warn\" or \"error\"", + "type": "string" + }, + "stageId": { + "description": "The stage ID which the advice is provided on. Ignore stageId if its value is -1", + "type": "integer", + "format": "int64" + }, + "jobId": { + "description": "The job ID which the advice is provided on. Ignore jobId if its value is -1", + "type": "integer", + "format": "int64" + }, + "executionId": { + "description": "The execution (SQL query) ID which the advice is provided on. Ignore executionId if its value is -1", + "type": "integer", + "format": "int64" + }, + "jobGroupId": { + "description": "The job group (statement) ID which the advice is provided on", + "type": "string" + }, + "executorId": { + "description": "The executor ID which the advice is provided on", + "type": "string" + } + } + }, + "JsonAdviseDetail": { + "type": "object", + "properties": { + "data": { + "oneOf": { + "$ref": "#/definitions/TaskError", + "$ref": "#/definitions/DataSkew", + "$ref": "#/definitions/TimeSkew" + } + } + } + }, + "TaskError": { + "type": "object", + "properties": { + "name": { + "description": "The name of the error or exception", + "type": "string" + }, + "tsg": { + "description": "The trouble shooting guide to follow", + "type": "string" + }, + "taskTypes": { + "description": "The types of Spark tasks, could be \"ResultTask\", \"ShuffleMapTask\"...", + "type": "string" + }, + "helplink": { + "description": "The web page or documentation to reference or help further", + "type": "string" + }, + "executorIds": { + "description": "The executor IDs", + "type": "string" + }, + "stageIds": { + "description": "The stage IDs", + "type": "string" + }, + "count": { + "description": "The task count", + "type": "integer", + "format": "int32" + } + } + }, + "DataSkew": { + "type": "object", + "properties": { + "name": { + "description": "The stage name", + "type": "string" + }, + "stageId": { + "description": "The stage ID", + "type": "string" + }, + "skewedTaskPercentage": { + "description": "The percentage of skewed tasks", + "type": "string" + }, + "maxDataRead": { + "description": "Max data read, in MiB", + "type": "string" + }, + "meanDataRead": { + "description": "Mean data read, in MiB", + "type": "string" + }, + "taskDataReadSkewness": { + "description": "The data read skewness of tasks within the stage, measured by \"Pearson's Second Coefficient of Skewness\"", + "type": "number", + "format": "double" + } + } + }, + "TimeSkew": { + "type": "object", + "properties": { + "name": { + "description": "The stage name", + "type": "string" + }, + "stageId": { + "description": "The stage ID", + "type": "string" + }, + "skewedTaskPercentage": { + "description": "The percentage of skewed tasks", + "type": "string" + }, + "maxTaskDuration": { + "description": "Max duration, in second", + "type": "string" + }, + "meanTaskDuration": { + "description": "Mean duration, in second", + "type": "string" + }, + "taskDurationSkewness": { + "description": "The duration skewness of tasks within the stage, measured by \"Pearson's Second Coefficient of Skewness\"", + "type": "number", + "format": "double" + } + } } } } \ No newline at end of file diff --git a/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.yaml b/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.yaml index 90542eff..bf15b302 100644 --- a/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.yaml +++ b/docs-samples/data-engineering/API-Swaggers/Spark-application-monitoring-API/swagger.yaml @@ -569,6 +569,145 @@ paths: description: OK schema: $ref: '#/definitions/ResourceUsageSnapshot' + /v1/workspaces/{workspaceId}/{itemType}/{itemId}/livySessions/{livyId}/applications/{appId}/{attemptId}/advice: + get: + tags: + - SparkAdvisor + summary: Get advice list + operationId: SparkAdvisor_List + produces: + - application/json + parameters: + - name: workspaceId + in: path + required: true + type: string + format: uuid + - name: itemType + in: path + required: true + type: string + enum: + - notebooks + - sparkJobDefinitions + - lakehouses + - name: itemId + in: path + required: true + type: string + format: uuid + - name: livyId + in: path + required: true + type: string + format: uuid + - name: appId + in: path + required: true + type: string + - name: attemptId + in: path + required: false + type: integer + format: int32 + - name: stageId + in: query + description: The specific stage ID to get advice on + required: false + type: integer + format: int64 + - name: jobId + in: query + description: The specific job ID to get advice on + required: false + type: integer + format: int64 + - name: exectionId + in: query + description: The specific exection (SQL query) ID to get advice on + required: false + type: integer + format: int64 + - name: jobGroupId + in: query + description: The specific job group (statement) ID to get advice on + required: false + type: string + - name: executorId + in: query + description: The specific executor ID to get advice on + required: false + type: string + responses: + '200': + description: OK + schema: + type: array + items: + $ref: '#/definitions/AdviseDataJson' + /v1/workspaces/{workspaceId}/{itemType}/{itemId}/livySessions/{livyId}/applications/{appId}/{attemptId}/advice/{filter}/{byId}: + get: + tags: + - SparkAdvisor + summary: Get advice by filter + operationId: SparkAdvisor_ByFilter + produces: + - application/json + parameters: + - name: workspaceId + in: path + required: true + type: string + format: uuid + - name: itemType + in: path + required: true + type: string + enum: + - notebooks + - sparkJobDefinitions + - lakehouses + - name: itemId + in: path + required: true + type: string + format: uuid + - name: livyId + in: path + required: true + type: string + format: uuid + - name: appId + in: path + required: true + type: string + - name: attemptId + in: path + required: false + type: integer + format: int32 + - name: filter + in: path + required: true + type: string + enum: + - stages + - jobs + - executions + - jobGroups + - executors + - name: byId + in: path + description: The specific ID to filter. Type is "int64" if it follows the filter of "stages", "jobs" or "executions"; Type is "string" if it follows the filter of "jobGroups" or "executors" + required: true + type: object + responses: + '200': + description: OK + schema: + type: array + items: + $ref: '#/definitions/AdviseDataJson' definitions: FileMetadata: description: Metadata of a log file @@ -765,4 +904,124 @@ definitions: type: integer format: int64 data: - $ref: '#/definitions/ResourceUsageSnapshotData' \ No newline at end of file + $ref: '#/definitions/ResourceUsageSnapshotData' + AdviseDataJson: + type: object + properties: + id: + description: The ID of the advice + type: integer + format: int64 + name: + description: The name of the advice + type: string + description: + description: The description of the advice + type: string + helplink: + description: The web page or documentation to reference or help further + type: string + detail: + description: More details in rich text format + $ref: '#/definitions/JsonAdviseDetail' + source: + description: Could be "System", "User" or "Dependency" + type: string + level: + description: Could be "info", "warn" or "error" + type: string + stageId: + description: The stage ID which the advice is provided on. Ignore stageId if its value is -1 + type: integer + format: int64 + jobId: + description: The job ID which the advice is provided on. Ignore jobId if its value is -1 + type: integer + format: int64 + executionId: + description: The execution (SQL query) ID which the advice is provided on. Ignore executionId if its value is -1 + type: integer + format: int64 + jobGroupId: + description: The job group (statement) ID which the advice is provided on + type: string + executorId: + description: The executor ID which the advice is provided on + type: string + JsonAdviseDetail: + type: object + properties: + data: + oneOf: + - $ref: '#/definitions/TaskError' + - $ref: '#/definitions/DataSkew' + - $ref: '#/definitions/TimeSkew' + TaskError: + type: object + properties: + name: + description: The name of the error or exception + type: string + tsg: + description: The trouble shooting guide to follow + type: string + taskTypes: + description: 'The types of Spark tasks, could be "ResultTask", "ShuffleMapTask"...' + type: string + helplink: + description: The web page or documentation to reference or help further + type: string + executorIds: + description: The executor IDs + type: string + stageIds: + description: The stage IDs + type: string + count: + description: The task count + type: integer + format: int32 + DataSkew: + type: object + properties: + name: + description: The stage name + type: string + stageId: + description: The stage ID + type: string + skewedTaskPercentage: + description: The percentage of skewed tasks + type: string + maxDataRead: + description: 'Max data read, in MiB' + type: string + meanDataRead: + description: 'Mean data read, in MiB' + type: string + taskDataReadSkewness: + description: The data read skewness of tasks within the stage, measured by "Pearson's Second Coefficient of Skewness" + type: number + format: double + TimeSkew: + type: object + properties: + name: + description: The stage name + type: string + stageId: + description: The stage ID + type: string + skewedTaskPercentage: + description: The percentage of skewed tasks + type: string + maxTaskDuration: + description: 'Max duration, in second' + type: string + meanTaskDuration: + description: 'Mean duration, in second' + type: string + taskDurationSkewness: + description: The duration skewness of tasks within the stage, measured by "Pearson's Second Coefficient of Skewness" + type: number + format: double \ No newline at end of file