diff --git a/common/scanning/credscanning/fields.go b/common/scanning/credscanning/fields.go index 9dfa078006..cc7e30596d 100644 --- a/common/scanning/credscanning/fields.go +++ b/common/scanning/credscanning/fields.go @@ -186,10 +186,13 @@ func getFields(info providers.ProviderInfo, withRequiredWorkspace = info.Oauth2Opts.ExplicitWorkspaceRequired } + workspaceMode := optionalType if info.RequiresWorkspace() || withRequiredWorkspace { - lists.Add(requiredType, Fields.Workspace) + workspaceMode = requiredType } + lists.Add(workspaceMode, Fields.Workspace) + return lists, nil } diff --git a/providers/microsoft/connector.go b/providers/microsoft/connector.go index 15503dac14..2ecd9a669a 100644 --- a/providers/microsoft/connector.go +++ b/providers/microsoft/connector.go @@ -11,6 +11,7 @@ import ( "github.com/amp-labs/connectors/internal/components/schema" "github.com/amp-labs/connectors/internal/components/writer" "github.com/amp-labs/connectors/providers" + "github.com/amp-labs/connectors/providers/microsoft/internal/batch" "github.com/amp-labs/connectors/providers/microsoft/internal/metadata" ) @@ -28,6 +29,9 @@ type Connector struct { components.Reader components.Writer components.Deleter + + // Dependent services. + batchStrategy *batch.Strategy } // NewConnector creates a new Microsoft connector. It defaults to the Microsoft @@ -49,7 +53,8 @@ func NewConnectorForProvider(provider providers.Provider, params common.Connecto // nolint:funlen func constructor(base *components.Connector) (*Connector, error) { connector := &Connector{ - Connector: base, + Connector: base, + batchStrategy: batch.NewStrategy(base.JSONHTTPClient(), base.ProviderInfo()), } connector.SchemaProvider = schema.NewOpenAPISchemaProvider(connector.ProviderContext.Module(), metadata.Schemas) diff --git a/providers/microsoft/read-by-ids.go b/providers/microsoft/read-by-ids.go new file mode 100644 index 0000000000..be6ccdc2cd --- /dev/null +++ b/providers/microsoft/read-by-ids.go @@ -0,0 +1,64 @@ +package microsoft + +import ( + "context" + "fmt" + "net/http" + + "github.com/amp-labs/connectors" + "github.com/amp-labs/connectors/common" + "github.com/amp-labs/connectors/common/readhelper" + "github.com/amp-labs/connectors/internal/datautils" + "github.com/amp-labs/connectors/providers/microsoft/internal/batch" +) + +var _ connectors.BatchRecordReaderConnector = (*Connector)(nil) + +// GetRecordsByIds scoped reading of records given their ids. +// nolint:revive +func (c *Connector) GetRecordsByIds(ctx context.Context, + objectName string, recordIds []string, + fields []string, associations []string, +) ([]common.ReadResultRow, error) { + if len(recordIds) == 0 { + return nil, common.ErrMissingObjects + } + + batchParams, requestIdentifiers, err := c.paramsForBatchRead(objectName, recordIds) + if err != nil { + return nil, err + } + + batchResponse := batch.Execute[map[string]any](ctx, c.batchStrategy, batchParams) + if err = batchResponse.JoinedErr(); err != nil { + return nil, err + } + + marshaler := readhelper.MakeGetMarshaledDataWithId(readhelper.NewIdField("id")) + uniqueFields := datautils.NewSetFromList(fields).List() + + return marshaler(batchResponse.GetInOrder(requestIdentifiers), uniqueFields) +} + +func (c *Connector) paramsForBatchRead( + objectName string, identifiers []string, +) (*batch.Params, []batch.RequestID, error) { + batchParams := &batch.Params{} + + requestIdentifiers := make([]batch.RequestID, len(identifiers)) + for index, identifier := range identifiers { + url, err := c.getURL(objectName) + if err != nil { + return nil, nil, err + } + + url.AddPath(identifier) + requestIdentifier := batch.RequestID(fmt.Sprintf("%v_%v", objectName, identifier)) + requestIdentifiers[index] = requestIdentifier + batchParams.WithRequest(requestIdentifier, http.MethodGet, url, nil, map[string]any{ + "Content-Type": "application/json", + }) + } + + return batchParams, requestIdentifiers, nil +} diff --git a/providers/microsoft/read-by-ids_test.go b/providers/microsoft/read-by-ids_test.go new file mode 100644 index 0000000000..aaae962a7c --- /dev/null +++ b/providers/microsoft/read-by-ids_test.go @@ -0,0 +1,115 @@ +package microsoft + +import ( + "net/http" + "testing" + + "github.com/amp-labs/connectors" + "github.com/amp-labs/connectors/common" + "github.com/amp-labs/connectors/test/utils/mockutils/mockcond" + "github.com/amp-labs/connectors/test/utils/mockutils/mockserver" + "github.com/amp-labs/connectors/test/utils/testroutines" + "github.com/amp-labs/connectors/test/utils/testutils" +) + +func TestGetRecordsByIds(t *testing.T) { // nolint:funlen,cyclop + t.Parallel() + + responseMessages := testutils.DataFromFile(t, "read/messages/batch-by-ids.json") + + tests := []testroutines.ReadByIds{ + { + Name: "Empty record identifiers", + Server: mockserver.Dummy(), + ExpectedErrs: []error{common.ErrMissingObjects}, + }, + { + Name: "Read messages by identifiers", + Input: testroutines.ReadByIdsParams{ + ObjectName: "me/messages", + RecordIds: []string{"msg1", "msg2", "msg3"}, + Fields: []string{"subject"}, + }, + Server: mockserver.Conditional{ + Setup: mockserver.ContentJSON(), + If: mockcond.And{ + mockcond.MethodPOST(), + mockcond.Path("/v1.0/$batch"), + mockcond.Body(`{ + "requests": [ + { + "id": "me/messages_msg1", + "method": "GET", + "url": "/me/messages/msg1", + "headers": { + "Content-Type": "application/json" + } + }, + { + "id": "me/messages_msg2", + "method": "GET", + "url": "/me/messages/msg2", + "headers": { + "Content-Type": "application/json" + } + }, + { + "id": "me/messages_msg3", + "method": "GET", + "url": "/me/messages/msg3", + "headers": { + "Content-Type": "application/json" + } + } + ] + }`), + }, + Then: mockserver.Response(http.StatusOK, responseMessages), + }.Server(), + Comparator: testroutines.ComparatorSubsetReadByIds, + Expected: []common.ReadResultRow{{ + Id: "msg1", + Fields: map[string]any{ + "subject": "Hello", + }, + Raw: map[string]any{ + "id": "msg1", + "subject": "Hello", + "bodyPreview": "Hi there", + }, + }, { + Id: "msg2", + Fields: map[string]any{ + "subject": "Meeting", + }, + Raw: map[string]any{ + "id": "msg2", + "subject": "Meeting", + "bodyPreview": "See you soon", + }, + }, { + Id: "msg3", + Fields: map[string]any{ + "subject": "Lunch", + }, + Raw: map[string]any{ + "id": "msg3", + "subject": "Lunch", + "bodyPreview": "Hungry?", + }, + }}, + ExpectedErrs: nil, + }, + } + + for _, tt := range tests { // nolint:dupl + // nolint:varnamelen + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + tt.Run(t, func() (connectors.BatchRecordReaderConnector, error) { + return constructTestConnector(tt.Server.URL) + }) + }) + } +} diff --git a/providers/microsoft/test/read/messages/batch-by-ids.json b/providers/microsoft/test/read/messages/batch-by-ids.json new file mode 100644 index 0000000000..d7bb50b0cc --- /dev/null +++ b/providers/microsoft/test/read/messages/batch-by-ids.json @@ -0,0 +1,193 @@ +{ + "responses": [ + { + "id": "me/messages_msg1", + "status": 200, + "headers": { + "Cache-Control": "private", + "Content-Type": "application/json; odata.metadata=minimal; odata.streaming=true; IEEE754Compatible=false; charset=utf-8" + }, + "body": { + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('ae87552f-48fc-4dec-9322-65040bf9fdfd')/messages/$entity", + "@odata.etag": "W/\"msg1-etag\"", + "id": "msg1", + "createdDateTime": "2026-04-21T01:12:24Z", + "lastModifiedDateTime": "2026-04-21T01:12:24Z", + "changeKey": "msg1-changekey", + "categories": [], + "receivedDateTime": "2026-04-21T01:12:24Z", + "sentDateTime": "2026-04-21T01:12:24Z", + "hasAttachments": false, + "internetMessageId": "", + "subject": "Hello", + "bodyPreview": "Hi there", + "importance": "normal", + "parentFolderId": "folder1", + "conversationId": "conv1", + "conversationIndex": "idx1", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": true, + "isDraft": true, + "webLink": "https://outlook.office365.com/owa/?ItemID=msg1", + "inferenceClassification": "focused", + "body": { + "contentType": "text", + "content": "Hi there" + }, + "sender": { + "emailAddress": { + "name": "Alice", + "address": "alice@test.com" + } + }, + "from": { + "emailAddress": { + "name": "Alice", + "address": "alice@test.com" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "Bob", + "address": "bob@test.com" + } + } + ], + "ccRecipients": [], + "bccRecipients": [], + "replyTo": [], + "flag": { + "flagStatus": "notFlagged" + } + } + }, + { + "id": "me/messages_msg2", + "status": 200, + "headers": { + "Cache-Control": "private", + "Content-Type": "application/json; odata.metadata=minimal; odata.streaming=true; IEEE754Compatible=false; charset=utf-8" + }, + "body": { + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('ae87552f-48fc-4dec-9322-65040bf9fdfd')/messages/$entity", + "@odata.etag": "W/\"msg2-etag\"", + "id": "msg2", + "createdDateTime": "2026-04-21T01:12:22Z", + "lastModifiedDateTime": "2026-04-21T01:12:22Z", + "changeKey": "msg2-changekey", + "categories": [], + "receivedDateTime": "2026-04-21T01:12:22Z", + "sentDateTime": "2026-04-21T01:12:22Z", + "hasAttachments": false, + "internetMessageId": "", + "subject": "Meeting", + "bodyPreview": "See you soon", + "importance": "normal", + "parentFolderId": "folder1", + "conversationId": "conv2", + "conversationIndex": "idx2", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": true, + "isDraft": true, + "webLink": "https://outlook.office365.com/owa/?ItemID=msg2", + "inferenceClassification": "focused", + "body": { + "contentType": "text", + "content": "See you soon" + }, + "sender": { + "emailAddress": { + "name": "Charlie", + "address": "charlie@test.com" + } + }, + "from": { + "emailAddress": { + "name": "Charlie", + "address": "charlie@test.com" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "David", + "address": "david@test.com" + } + } + ], + "ccRecipients": [], + "bccRecipients": [], + "replyTo": [], + "flag": { + "flagStatus": "notFlagged" + } + } + }, + { + "id": "me/messages_msg3", + "status": 200, + "headers": { + "Cache-Control": "private", + "Content-Type": "application/json; odata.metadata=minimal; odata.streaming=true; IEEE754Compatible=false; charset=utf-8" + }, + "body": { + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('ae87552f-48fc-4dec-9322-65040bf9fdfd')/messages/$entity", + "@odata.etag": "W/\"msg3-etag\"", + "id": "msg3", + "createdDateTime": "2026-04-21T01:12:23Z", + "lastModifiedDateTime": "2026-04-21T01:12:23Z", + "changeKey": "msg3-changekey", + "categories": [], + "receivedDateTime": "2026-04-21T01:12:23Z", + "sentDateTime": "2026-04-21T01:12:23Z", + "hasAttachments": false, + "internetMessageId": "", + "subject": "Lunch", + "bodyPreview": "Hungry?", + "importance": "normal", + "parentFolderId": "folder1", + "conversationId": "conv3", + "conversationIndex": "idx3", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": true, + "isDraft": true, + "webLink": "https://outlook.office365.com/owa/?ItemID=msg3", + "inferenceClassification": "focused", + "body": { + "contentType": "text", + "content": "Hungry?" + }, + "sender": { + "emailAddress": { + "name": "Eve", + "address": "eve@test.com" + } + }, + "from": { + "emailAddress": { + "name": "Eve", + "address": "eve@test.com" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "Frank", + "address": "frank@test.com" + } + } + ], + "ccRecipients": [], + "bccRecipients": [], + "replyTo": [], + "flag": { + "flagStatus": "notFlagged" + } + } + } + ] +} diff --git a/test/microsoft/connector.go b/test/microsoft/connector.go index b8be23e744..482dec48a7 100644 --- a/test/microsoft/connector.go +++ b/test/microsoft/connector.go @@ -1,4 +1,4 @@ -package dynamicscrm +package microsoft import ( "context" diff --git a/test/microsoft/outlook-mail/read-by-ids/messages/main.go b/test/microsoft/outlook-mail/read-by-ids/messages/main.go new file mode 100644 index 0000000000..c04c80e364 --- /dev/null +++ b/test/microsoft/outlook-mail/read-by-ids/messages/main.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "reflect" + "sort" + "syscall" + + "github.com/amp-labs/connectors/common" + "github.com/amp-labs/connectors/internal/datautils" + connTest "github.com/amp-labs/connectors/test/microsoft" + "github.com/amp-labs/connectors/test/utils" + "github.com/amp-labs/connectors/test/utils/testscenario" + "github.com/brianvoe/gofakeit/v6" + "github.com/go-test/deep" +) + +func main() { + if err := run(); err != nil { + utils.Fail("test failed", "error", err) + } +} + +func run() error { + // Handle Ctrl-C gracefully. + ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer done() + + conn := connTest.GetMicrosoftGraphConnector(ctx) + + messageIdentifiers := make([]string, 3) + + for index := range messageIdentifiers { + subject := gofakeit.Name() + fmt.Sprintf(" [%v]", index+1) + bodyData := gofakeit.Name() + from := gofakeit.Username() + to := gofakeit.Username() + message, cleanup, err := testscenario.SetupRecord(ctx, conn, "me/messages", + payload{ + Subject: subject, + Body: body{Content: bodyData, ContentType: TextContentType}, + From: &recipient{ + EmailAddress: address{Address: from + "@test.com", Name: from}, + }, + ToRecipients: []recipient{{ + EmailAddress: address{Address: to + "@test.com", Name: to}, + }}, + }, testscenario.RecordCreationRecipe{ + ReadFields: datautils.NewSet("id", "subject"), + SearchBy: testscenario.Property{ + Key: "subject", + Value: subject, + }, + RecordIdentifierKey: "id", + }) + if err != nil { + return err + } + + defer cleanup() + messageIdentifiers[index] = message.Id + } + + // https://learn.microsoft.com/en-us/graph/api/resources/message?view=graph-rest-1.0 + res, err := conn.GetRecordsByIds(ctx, + "me/messages", messageIdentifiers, + []string{"subject", "from", "toRecipients", "body"}, nil) + if err != nil { + return err + } + + displayResults(res, messageIdentifiers) + + return nil +} + +func displayResults(res []common.ReadResultRow, expectedIdentifiers []string) { + actualIdentifiers := datautils.ForEach(res, func(row common.ReadResultRow) string { + return row.Id + }) + + sort.Strings(expectedIdentifiers) + sort.Strings(actualIdentifiers) + + fmt.Println("========================") + if !reflect.DeepEqual(expectedIdentifiers, actualIdentifiers) { + diff := deep.Equal(expectedIdentifiers, actualIdentifiers) + fmt.Printf("Requested and returned record identifiers are mismatching\n\t%v\n", diff) + } else { + utils.DumpJSON(res, os.Stdout) + } + fmt.Println("========================") +} + +const TextContentType = "text" + +type payload struct { + Subject string `json:"subject,omitempty"` + Body body `json:"body,omitempty"` + From *recipient `json:"from,omitempty"` + ToRecipients []recipient `json:"toRecipients,omitempty"` +} + +type body struct { + Content string `json:"content"` + ContentType string `json:"contentType"` +} + +type recipient struct { + EmailAddress address `json:"emailAddress"` +} + +type address struct { + Address string `json:"address"` + Name string `json:"name"` +} diff --git a/test/utils/testroutines/comparator.go b/test/utils/testroutines/comparator.go index 1d17434e22..8c0cb71494 100644 --- a/test/utils/testroutines/comparator.go +++ b/test/utils/testroutines/comparator.go @@ -36,6 +36,21 @@ func ComparatorSubsetRead(serverURL string, actual, expected *common.ReadResult) return result } +// ComparatorSubsetReadByIds compares two slices of ReadResultRow as a subset, +// ignoring order and focusing only on relevant fields, raw data, associations, and identifiers. +func ComparatorSubsetReadByIds(serverURL string, actual, expected []common.ReadResultRow) *testutils.CompareResult { + return ComparatorSubsetRead(serverURL, + &common.ReadResult{ + Rows: int64(len(actual)), + Data: actual, + }, + &common.ReadResult{ + Rows: int64(len(expected)), + Data: expected, + }, + ) +} + // ComparatorPagination will check pagination related fields. // Note: you may use an alias for Mock-Server-URL which will be dynamically resolved at runtime. // Example: diff --git a/test/utils/testroutines/read-by-ids.go b/test/utils/testroutines/read-by-ids.go new file mode 100644 index 0000000000..62d5047492 --- /dev/null +++ b/test/utils/testroutines/read-by-ids.go @@ -0,0 +1,36 @@ +package testroutines + +import ( + "testing" + + "github.com/amp-labs/connectors" + "github.com/amp-labs/connectors/common" +) + +type ( + ReadByIdsType = TestCase[ReadByIdsParams, []common.ReadResultRow] + // ReadByIds is a test suite useful for testing connectors.BatchRecordReaderConnector interface. + ReadByIds ReadByIdsType +) + +type ReadByIdsParams struct { + ObjectName string + RecordIds []string + Fields []string + Associations []string +} + +// Run provides a procedure to test connectors.BatchRecordReaderConnector +func (r ReadByIds) Run(t *testing.T, builder ConnectorBuilder[connectors.BatchRecordReaderConnector]) { + t.Helper() + t.Cleanup(func() { + ReadByIdsType(r).Close() + }) + + conn := builder.Build(t, r.Name) + output, err := conn.GetRecordsByIds(t.Context(), + r.Input.ObjectName, r.Input.RecordIds, + r.Input.Fields, r.Input.Associations, + ) + ReadByIdsType(r).Validate(t, err, output) +}