Skip to content

Commit 0525b78

Browse files
zsimjeezsimjee
authored
Merge pull request #3 from guardrails-ai/get-guard-step
Get Guard Step
2 parents f5eed01 + 73094ea commit 0525b78

File tree

2 files changed

+232
-13
lines changed

2 files changed

+232
-13
lines changed

‎service-specs/guardrails-service-spec.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ components:
218218
required: false
219219
schema:
220220
type: string
221+
format: password
221222
schemas:
222223
HttpError:
223224
type: object
@@ -306,11 +307,15 @@ components:
306307
type: object
307308
properties:
308309
schema:
309-
type: object
310-
additionalProperties:
311-
$ref: "#/components/schemas/DataType"
310+
oneOf:
311+
- $ref: "#/components/schemas/JsonSchema"
312+
- $ref: "#/components/schemas/DataType"
312313
required:
313314
- schema
315+
JsonSchema:
316+
type: object
317+
additionalProperties:
318+
$ref: "#/components/schemas/DataType"
314319
DataType:
315320
type: object
316321
properties:

‎service-specs/reporting-service-spec.yml‎

Lines changed: 224 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ paths:
2424
application/json:
2525
schema:
2626
$ref: "#/components/schemas/HttpError"
27-
/metrics/guards:
27+
/guards/metrics:
2828
get:
29-
operationId: getMetricsForGuards
29+
operationId: getGuardMetrics
3030
summary: Returns an array of GuardMetrics for a given time range. If startTime or endTime are ommitted, a five minute duration is used by default. Default page size is 10.
3131
parameters:
3232
- $ref: "#/components/parameters/StartTime"
@@ -45,7 +45,90 @@ paths:
4545
schema:
4646
type: array
4747
items:
48-
$ref: "#/components/schemas/GuardMetric"
48+
$ref: "#/components/schemas/GuardMetrics"
49+
default:
50+
description: Unexpected error
51+
content:
52+
application/json:
53+
schema:
54+
$ref: "#/components/schemas/HttpError"
55+
/guards/{guardName}/runs/{requestId}/metrics:
56+
get:
57+
operationId: getGuardRunMetrics
58+
summary: Returns an instance of GuardRunMetrics for a given request.
59+
parameters:
60+
- $ref: "#/components/parameters/GuardName"
61+
- $ref: "#/components/parameters/RequestId"
62+
tags:
63+
- Metrics
64+
security:
65+
- apiKeyAuth: []
66+
- bearerAuth: []
67+
responses:
68+
"200":
69+
description: Returns metrics for Guard request
70+
content:
71+
application/json:
72+
schema:
73+
type: array
74+
items:
75+
$ref: "#/components/schemas/GuardRunMetrics"
76+
default:
77+
description: Unexpected error
78+
content:
79+
application/json:
80+
schema:
81+
$ref: "#/components/schemas/HttpError"
82+
/validators/metrics:
83+
get:
84+
operationId: getValidatorMetrics
85+
summary: Returns an array of ValidatorMetrics for a given time range. If startTime or endTime are ommitted, a five minute duration is used by default. Default page size is 10, but this page size applied to guard steps not validator metrics. Have to think this one out a little more.
86+
parameters:
87+
- $ref: "#/components/parameters/StartTime"
88+
- $ref: "#/components/parameters/EndTime"
89+
- $ref: "#/components/parameters/PageNumber"
90+
tags:
91+
- Metrics
92+
security:
93+
- apiKeyAuth: []
94+
- bearerAuth: []
95+
responses:
96+
"200":
97+
description: Returns metrics for Guards
98+
content:
99+
application/json:
100+
schema:
101+
type: array
102+
items:
103+
$ref: "#/components/schemas/ValidatorMetrics"
104+
default:
105+
description: Unexpected error
106+
content:
107+
application/json:
108+
schema:
109+
$ref: "#/components/schemas/HttpError"
110+
/validators/{validatorName}/runs/{requestId}/instances/{validatorInstanceId}/metrics:
111+
get:
112+
operationId: getValidatorRunMetrics
113+
summary: Returns an arary of ValidatorRunMetrics for a given request.
114+
parameters:
115+
- $ref: "#/components/parameters/ValidatorName"
116+
- $ref: "#/components/parameters/RequestId"
117+
- $ref: "#/components/parameters/ValidatorInstanceId"
118+
tags:
119+
- Metrics
120+
security:
121+
- apiKeyAuth: []
122+
- bearerAuth: []
123+
responses:
124+
"200":
125+
description: Returns metrics for Guard request
126+
content:
127+
application/json:
128+
schema:
129+
type: array
130+
items:
131+
$ref: "#/components/schemas/ValidatorRunMetrics"
49132
default:
50133
description: Unexpected error
51134
content:
@@ -101,7 +184,7 @@ components:
101184
required:
102185
- status
103186
- message
104-
GuardMetric:
187+
GuardMetrics:
105188
type: object
106189
properties:
107190
requestId:
@@ -118,7 +201,7 @@ components:
118201
type: number
119202
description: The total span duration of the guard run in seconds
120203
status:
121-
$ref: "#/components/schemas/GuardStatus"
204+
$ref: "#/components/schemas/Status"
122205
reasks:
123206
type: number
124207
description: The number of times the LLM was reasked
@@ -141,33 +224,164 @@ components:
141224
- tokens
142225
- rawLlmOuput
143226
- validatedOutput
144-
GuardStatus:
227+
Status:
145228
type: string
146229
enum: [
147230
"pass",
148231
"fail"
149232
]
233+
GuardRunMetrics:
234+
allOf:
235+
- $ref: '#/components/schemas/GuardMetrics'
236+
- $ref: '#/components/schemas/GuardRunMetricDetails'
237+
GuardRunMetricDetails:
238+
type: object
239+
properties:
240+
prompt:
241+
type: string
242+
instructions:
243+
type: string
244+
steps:
245+
type: array
246+
items:
247+
$ref: '#/components/schemas/GuardRunStep'
248+
required:
249+
- steps
250+
GuardRunStep:
251+
type: object
252+
properties:
253+
stepNumber:
254+
type: number
255+
validatorMetrics:
256+
type: array
257+
items:
258+
$ref: '#/components/schemas/ValidatorMetrics'
259+
required:
260+
- stepNumber
261+
- validatorMetrics
262+
ValidatorMetrics:
263+
type: object
264+
properties:
265+
instanceId:
266+
type: string
267+
name:
268+
type: string
269+
requestId:
270+
type: string
271+
guardName:
272+
type: string
273+
startTime:
274+
type: string
275+
format: date-time
276+
endTime:
277+
type: string
278+
format: date-time
279+
durationInMillis:
280+
type: number
281+
input:
282+
type: string
283+
output:
284+
type: string
285+
status:
286+
$ref: "#/components/schemas/Status"
287+
tokens:
288+
type: number
289+
description: The number of tokens consumed if the validator calls an LLM. Only applies to validators that call LLMs.
290+
required:
291+
- instanceId
292+
- name
293+
- requestId
294+
- guardName
295+
- startTime
296+
- endTime
297+
- durationInMillis
298+
- input
299+
- output
300+
- status
301+
ValidatorRunMetrics:
302+
allOf:
303+
- $ref: '#/components/schemas/ValidatorMetrics'
304+
- $ref: '#/components/schemas/ValidatorRunMetricDetails'
305+
ValidatorRunMetricDetails:
306+
type: object
307+
properties:
308+
stepNumber:
309+
type: integer
310+
resultType:
311+
type: string
312+
failAction:
313+
$ref: '#/components/schemas/OnFailOptions'
314+
parameters:
315+
type: object
316+
properties: {}
317+
additionalProperties:
318+
type: string
319+
required:
320+
- stepNumber
321+
- resultType
322+
- failAction
323+
- parameters
324+
OnFailOptions:
325+
type: string
326+
enum: [
327+
"exception",
328+
"filter",
329+
"fix",
330+
"fix_reask",
331+
"noop",
332+
"reask",
333+
"refrain"
334+
]
150335
parameters:
151336
StartTime:
152337
name: startTime
153338
in: query
154-
description: Used to specify to start of a time range
339+
description: Used to specify the start of a time range
155340
required: false
156341
schema:
157342
type: string
158343
format: date-time
159344
EndTime:
160345
name: endTime
161346
in: query
162-
description: Used to specify to end of a time range
347+
description: Used to specify the end of a time range
163348
required: false
164349
schema:
165350
type: string
166351
format: date-time
167352
PageNumber:
168353
name: page
169354
in: query
170-
description: Used to specify the page number for paginated endpoints
355+
description: Used to specify the page number for paginated endpoints. Pages are zero-indexed.
171356
required: false
172357
schema:
173-
type: integer
358+
type: integer
359+
default: 0
360+
GuardName:
361+
name: guardName
362+
in: path
363+
description: Guard name
364+
required: true
365+
schema:
366+
type: string
367+
RequestId:
368+
name: requestId
369+
in: path
370+
description: Request id aka trace id for a validate-{guard-name} span
371+
required: true
372+
schema:
373+
type: string
374+
ValidatorName:
375+
name: validatorName
376+
in: path
377+
description: the validator's registered name
378+
required: true
379+
schema:
380+
type: string
381+
ValidatorInstanceId:
382+
name: validatorInstanceId
383+
in: path
384+
description: the instanceId from ValidatorMetrics
385+
required: true
386+
schema:
387+
type: string

0 commit comments

Comments
 (0)