Skip to content

Commit a6609c4

Browse files
michelle192837michelle192837
committed
Use SearchConfiguredTargets when searching targets.
If searching for a target (as opposed to labels or other identifiers), we want to search for those results using a difference method (SearchConfiguredTargets, rather than SearchInvocations). Add the search method and use it if the query contains a target atom (and update the Prow label accordingly for this search). Moved the query code into a separate package so it can be used in config validation without a circular dependency.
1 parent 837ecf5 commit a6609c4

10 files changed

Lines changed: 205 additions & 44 deletions

File tree

‎config/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ go_library(
1313
visibility = ["//visibility:public"],
1414
deps = [
1515
"//pb/config:go_default_library",
16+
"//pkg/updater/resultstore/query:go_default_library",
1617
"//util/gcs:go_default_library",
1718
"//util/queue:go_default_library",
1819
"@com_github_golang_protobuf//proto:go_default_library",

‎config/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/golang/protobuf/proto"
2929

3030
configpb "github.com/GoogleCloudPlatform/testgrid/pb/config"
31+
"github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore/query"
3132
multierror "github.com/hashicorp/go-multierror"
3233
)
3334

@@ -269,6 +270,9 @@ func validateResultStoreSource(tg *configpb.TestGroup) error {
269270
if rs.GetProject() == "" {
270271
return errors.New("project ID in resultstore_config cannot be empty")
271272
}
273+
if _, err := query.TranslateQuery(rs.GetQuery()); err != nil {
274+
return fmt.Errorf("invalid ResultStore query %q: %v", rs.GetQuery(), err)
275+
}
272276
}
273277
return nil
274278
}

‎config/config_test.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,47 @@ func TestValidateResultStoreSource(t *testing.T) {
498498
},
499499
err: false,
500500
},
501+
{
502+
name: "valid query",
503+
tg: &configpb.TestGroup{
504+
ResultSource: &configpb.TestGroup_ResultSource{
505+
ResultSourceConfig: &configpb.TestGroup_ResultSource_ResultstoreConfig{
506+
ResultstoreConfig: &configpb.ResultStoreConfig{
507+
Project: "my-project",
508+
Query: `target:"my-job"`,
509+
},
510+
},
511+
},
512+
},
513+
err: false,
514+
},
515+
{
516+
name: "invalid query",
517+
tg: &configpb.TestGroup{
518+
ResultSource: &configpb.TestGroup_ResultSource{
519+
ResultSourceConfig: &configpb.TestGroup_ResultSource_ResultstoreConfig{
520+
ResultstoreConfig: &configpb.ResultStoreConfig{
521+
Project: "my-project",
522+
Query: `label:foo bar`,
523+
},
524+
},
525+
},
526+
},
527+
err: true,
528+
},
529+
{
530+
name: "query without project",
531+
tg: &configpb.TestGroup{
532+
ResultSource: &configpb.TestGroup_ResultSource{
533+
ResultSourceConfig: &configpb.TestGroup_ResultSource_ResultstoreConfig{
534+
ResultstoreConfig: &configpb.ResultStoreConfig{
535+
Query: `target:"my-job"`,
536+
},
537+
},
538+
},
539+
},
540+
err: true,
541+
},
501542
{
502543
name: "gcs_prefix and ResultStore defined",
503544
tg: &configpb.TestGroup{

‎pkg/updater/resultstore/BUILD.bazel‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ go_library(
44
name = "go_default_library",
55
srcs = [
66
"client.go",
7-
"query.go",
87
"resultstore.go",
98
],
109
importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore",
@@ -15,6 +14,7 @@ go_library(
1514
"//pb/state:go_default_library",
1615
"//pb/test_status:go_default_library",
1716
"//pkg/updater:go_default_library",
17+
"//pkg/updater/resultstore/query:go_default_library",
1818
"//util/gcs:go_default_library",
1919
"@com_github_sirupsen_logrus//:go_default_library",
2020
"@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto",
@@ -28,10 +28,7 @@ go_library(
2828

2929
go_test(
3030
name = "go_default_test",
31-
srcs = [
32-
"query_test.go",
33-
"resultstore_test.go",
34-
],
31+
srcs = ["resultstore_test.go"],
3532
embed = [":go_default_library"],
3633
deps = [
3734
"//pb/config:go_default_library",
@@ -59,7 +56,10 @@ filegroup(
5956

6057
filegroup(
6158
name = "all-srcs",
62-
srcs = [":package-srcs"],
59+
srcs = [
60+
":package-srcs",
61+
"//pkg/updater/resultstore/query:all-srcs",
62+
],
6363
tags = ["automanaged"],
6464
visibility = ["//visibility:public"],
6565
)

‎pkg/updater/resultstore/client.go‎

Lines changed: 62 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333

3434
type resultStoreClient interface {
3535
SearchInvocations(context.Context, *resultstore.SearchInvocationsRequest, ...grpc.CallOption) (*resultstore.SearchInvocationsResponse, error)
36+
SearchConfiguredTargets(context.Context, *resultstore.SearchConfiguredTargetsRequest, ...grpc.CallOption) (*resultstore.SearchConfiguredTargetsResponse, error)
3637
ExportInvocation(context.Context, *resultstore.ExportInvocationRequest, ...grpc.CallOption) (*resultstore.ExportInvocationResponse, error)
3738
}
3839

@@ -81,35 +82,76 @@ func Connect(ctx context.Context, serviceAccountPath string) (*grpc.ClientConn,
8182
}
8283

8384
// Search finds all the invocations that satisfies the query condition within a project.
84-
func (c *DownloadClient) Search(ctx context.Context, log logrus.FieldLogger, query, projectID string, fields ...string) ([]string, error) {
85-
var ids []string
85+
func (c *DownloadClient) Search(ctx context.Context, log logrus.FieldLogger, query, projectID string) ([]string, error) {
86+
var invIDs []string
8687
nextPageToken := ""
87-
fieldMaskCtx := fieldMask(
88-
ctx,
89-
"next_page_token",
90-
"invocations.id",
91-
)
88+
searchTargets := strings.Contains(query, "id.target_id=")
9289
for {
93-
req := &resultstore.SearchInvocationsRequest{
94-
Query: query,
95-
ProjectId: projectID,
96-
PageStart: &resultstore.SearchInvocationsRequest_PageToken{
97-
PageToken: nextPageToken,
98-
},
90+
var ids []string
91+
var err error
92+
if searchTargets {
93+
ids, nextPageToken, err = c.targetSearch(ctx, log, query, projectID, nextPageToken)
94+
} else {
95+
ids, nextPageToken, err = c.invocationSearch(ctx, log, query, projectID, nextPageToken)
9996
}
100-
resp, err := c.client.SearchInvocations(fieldMaskCtx, req)
10197
if err != nil {
10298
return nil, err
10399
}
104-
for _, inv := range resp.GetInvocations() {
105-
ids = append(ids, inv.Id.GetInvocationId())
106-
}
107-
if resp.GetNextPageToken() == "" {
100+
invIDs = append(invIDs, ids...)
101+
if nextPageToken == "" {
108102
break
109103
}
110-
nextPageToken = resp.GetNextPageToken()
111104
}
112-
return ids, nil
105+
return invIDs, nil
106+
}
107+
108+
func (c *DownloadClient) invocationSearch(ctx context.Context, log logrus.FieldLogger, query, projectID, nextPageToken string) ([]string, string, error) {
109+
fieldMaskCtx := fieldMask(
110+
ctx,
111+
"next_page_token",
112+
"invocations.id",
113+
)
114+
req := &resultstore.SearchInvocationsRequest{
115+
Query: query,
116+
ProjectId: projectID,
117+
PageStart: &resultstore.SearchInvocationsRequest_PageToken{
118+
PageToken: nextPageToken,
119+
},
120+
}
121+
resp, err := c.client.SearchInvocations(fieldMaskCtx, req)
122+
if err != nil {
123+
return nil, "", err
124+
}
125+
var ids []string
126+
for _, inv := range resp.GetInvocations() {
127+
ids = append(ids, inv.GetId().GetInvocationId())
128+
}
129+
return ids, resp.GetNextPageToken(), err
130+
}
131+
132+
func (c *DownloadClient) targetSearch(ctx context.Context, log logrus.FieldLogger, query, projectID, nextPageToken string) ([]string, string, error) {
133+
fieldMaskCtx := fieldMask(
134+
ctx,
135+
"next_page_token",
136+
"configured_targets.id",
137+
)
138+
req := &resultstore.SearchConfiguredTargetsRequest{
139+
Query: query,
140+
ProjectId: projectID,
141+
Parent: "invocations/-/targets/-",
142+
PageStart: &resultstore.SearchConfiguredTargetsRequest_PageToken{
143+
PageToken: nextPageToken,
144+
},
145+
}
146+
resp, err := c.client.SearchConfiguredTargets(fieldMaskCtx, req)
147+
if err != nil {
148+
return nil, "", err
149+
}
150+
var ids []string
151+
for _, target := range resp.GetConfiguredTargets() {
152+
ids = append(ids, target.GetId().GetInvocationId())
153+
}
154+
return ids, resp.GetNextPageToken(), err
113155
}
114156

115157
// FetchResult provides a interface to store Resultstore invocation data.

‎pkg/updater/resultstore/query/BUILD.bazel‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["query.go"],
6+
importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore/query",
7+
visibility = ["//visibility:public"],
8+
)
9+
10+
go_test(
11+
name = "go_default_test",
12+
srcs = ["query_test.go"],
13+
embed = [":go_default_library"],
14+
)
15+
16+
filegroup(
17+
name = "package-srcs",
18+
srcs = glob(["**"]),
19+
tags = ["automanaged"],
20+
visibility = ["//visibility:private"],
21+
)
22+
23+
filegroup(
24+
name = "all-srcs",
25+
srcs = [":package-srcs"],
26+
tags = ["automanaged"],
27+
visibility = ["//visibility:public"],
28+
)

pkg/updater/resultstore/query.go renamed to pkg/updater/resultstore/query/query.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package resultstore
17+
package query
1818

1919
import (
2020
"fmt"
@@ -47,7 +47,7 @@ var (
4747
queryRe = regexp.MustCompile(`^target:".*"$`)
4848
)
4949

50-
func translateQuery(simpleQuery string) (string, error) {
50+
func TranslateQuery(simpleQuery string) (string, error) {
5151
if simpleQuery == "" {
5252
return "", nil
5353
}

pkg/updater/resultstore/query_test.go renamed to pkg/updater/resultstore/query/query_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package resultstore
17+
package query
1818

1919
import (
2020
"testing"
@@ -135,7 +135,7 @@ func TestTranslateQuery(t *testing.T) {
135135

136136
for _, tc := range cases {
137137
t.Run(tc.name, func(t *testing.T) {
138-
got, err := translateQuery(tc.query)
138+
got, err := TranslateQuery(tc.query)
139139
if tc.want != got {
140140
t.Errorf("translateQuery(%q) differed; got %q, want %q", tc.query, got, tc.want)
141141
}

‎pkg/updater/resultstore/resultstore.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"time"
2727

2828
"github.com/GoogleCloudPlatform/testgrid/pkg/updater"
29+
"github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore/query"
2930
"github.com/GoogleCloudPlatform/testgrid/util/gcs"
3031
"github.com/sirupsen/logrus"
3132

@@ -890,19 +891,22 @@ func queryAfter(query string, when time.Time) string {
890891
}
891892

892893
const (
894+
// Use this when searching invocations, e.g. if query does not search for a target.
893895
prowLabel = `invocation_attributes.labels:"prow"`
896+
// Use this when searching for a configured target, e.g. if query contains `target:"<target>"`.
897+
prowTargetLabel = `invocation.invocation_attributes.labels:"prow"`
894898
)
895899

896900
func queryProw(baseQuery string, stop time.Time) (string, error) {
897901
// TODO: ResultStore use is assumed to be Prow-only at the moment. Make this more flexible in future.
898902
if baseQuery == "" {
899903
return queryAfter(prowLabel, stop), nil
900904
}
901-
query, err := translateQuery(baseQuery)
905+
query, err := query.TranslateQuery(baseQuery)
902906
if err != nil {
903907
return "", err
904908
}
905-
return queryAfter(fmt.Sprintf("%s %s", query, prowLabel), stop), nil
909+
return queryAfter(fmt.Sprintf("%s %s", query, prowTargetLabel), stop), nil
906910
}
907911

908912
func search(ctx context.Context, log logrus.FieldLogger, client *DownloadClient, rsConfig *configpb.ResultStoreConfig, stop time.Time) ([]string, error) {

0 commit comments

Comments
 (0)