Merge pull request #1347 from cuixq:client

PiperOrigin-RevId: 808791051

Author: copybara-github

clients/datasource/maven_registry.go

@@ -34,11 +34,15 @@ import (
 	"deps.dev/util/semver"
 	"github.com/google/osv-scalibr/log"
 	"golang.org/x/net/html/charset"
+	"golang.org/x/oauth2/google"
 )
 
 // mavenCentral holds the URL of Maven Central Repository.
 const mavenCentral = "https://repo.maven.apache.org/maven2"
 
+// artifactRegistryScheme defines the scheme for Google Artifact Registry.
+const artifactRegistryScheme = "artifactregistry"
+
 var errAPIFailed = errors.New("API query failed")
 
 // MavenRegistryAPIClient defines a client to fetch metadata from a Maven registry.
@@ -47,6 +51,7 @@ type MavenRegistryAPIClient struct {
 	registries      []MavenRegistry                // Additional registries specified to fetch projects
 	registryAuths   map[string]*HTTPAuthentication // Authentication for the registries keyed by registry ID. From settings.xml
 	localRegistry   string                         // The local directory that holds Maven manifests
+	googleClient    *http.Client                   // A client for authenticating with Google services, used for Artifact Registry.
 
 	// Cache fields
 	mu             *sync.Mutex
@@ -71,7 +76,7 @@ type MavenRegistry struct {
 }
 
 // NewMavenRegistryAPIClient returns a new MavenRegistryAPIClient.
-func NewMavenRegistryAPIClient(registry MavenRegistry, localRegistry string) (*MavenRegistryAPIClient, error) {
+func NewMavenRegistryAPIClient(ctx context.Context, registry MavenRegistry, localRegistry string) (*MavenRegistryAPIClient, error) {
 	if registry.URL == "" {
 		registry.URL = mavenCentral
 		registry.ID = "central"
@@ -90,14 +95,18 @@ func NewMavenRegistryAPIClient(registry MavenRegistry, localRegistry string) (*M
 	globalSettings := ParseMavenSettings(globalMavenSettingsFile())
 	userSettings := ParseMavenSettings(userMavenSettingsFile())
 
-	return &MavenRegistryAPIClient{
+	client := &MavenRegistryAPIClient{
 		// We assume only downloading releases is allowed on the default registry.
 		defaultRegistry: registry,
 		localRegistry:   localRegistry,
 		mu:              &sync.Mutex{},
 		responses:       NewRequestCache[string, response](),
 		registryAuths:   MakeMavenAuth(globalSettings, userSettings),
-	}, nil
+	}
+	if registry.Parsed.Scheme == artifactRegistryScheme {
+		client.createGoogleClient(ctx)
+	}
+	return client, nil
 }
 
 // SetLocalRegistry sets the local directory that stores the downloaded Maven manifests.
@@ -114,13 +123,14 @@ func (m *MavenRegistryAPIClient) WithoutRegistries() *MavenRegistryAPIClient {
 		cacheTimestamp:  m.cacheTimestamp,
 		responses:       m.responses,
 		registryAuths:   m.registryAuths,
+		googleClient:    m.googleClient,
 	}
 }
 
 // AddRegistry adds the given registry to the list of registries if it has not been added.
-func (m *MavenRegistryAPIClient) AddRegistry(registry MavenRegistry) error {
+func (m *MavenRegistryAPIClient) AddRegistry(ctx context.Context, registry MavenRegistry) error {
 	if registry.ID == m.defaultRegistry.ID {
-		return m.updateDefaultRegistry(registry)
+		return m.updateDefaultRegistry(ctx, registry)
 	}
 
 	for _, reg := range m.registries {
@@ -136,20 +146,42 @@ func (m *MavenRegistryAPIClient) AddRegistry(registry MavenRegistry) error {
 
 	registry.Parsed = u
 	m.registries = append(m.registries, registry)
+	if registry.Parsed.Scheme == artifactRegistryScheme {
+		m.createGoogleClient(ctx)
+	}
 
 	return nil
 }
 
-func (m *MavenRegistryAPIClient) updateDefaultRegistry(registry MavenRegistry) error {
+func (m *MavenRegistryAPIClient) updateDefaultRegistry(ctx context.Context, registry MavenRegistry) error {
 	u, err := url.Parse(registry.URL)
 	if err != nil {
 		return err
 	}
 	registry.Parsed = u
 	m.defaultRegistry = registry
+	if registry.Parsed.Scheme == artifactRegistryScheme {
+		m.createGoogleClient(ctx)
+	}
 	return nil
 }
 
+// createGoogleClient creates a client for authenticating with Google services.
+func (m *MavenRegistryAPIClient) createGoogleClient(ctx context.Context) {
+	if m.googleClient != nil {
+		return
+	}
+	// This is the scope that artifact-registry-go-tools uses.
+	// https://github.com/GoogleCloudPlatform/artifact-registry-go-tools/blob/main/pkg/auth/auth.go
+	client, err := google.DefaultClient(ctx, "https://www.googleapis.com/auth/cloud-platform")
+	if err != nil {
+		// We don't return an error here so that we can fall back to a regular http client.
+		log.Warnf("failed to create Google default client, Artifact Registry access will be unavailable: %v", err)
+		return
+	}
+	m.googleClient = client
+}
+
 // GetRegistries returns the registries added to this client.
 func (m *MavenRegistryAPIClient) GetRegistries() (registries []MavenRegistry) {
 	return m.registries
@@ -269,17 +301,28 @@ func (m *MavenRegistryAPIClient) get(ctx context.Context, auth *HTTPAuthenticati
 		}
 	}
 
-	u := registry.Parsed.JoinPath(paths...).String()
+	httpClient := http.DefaultClient
+	requestURL := *registry.Parsed
+	isArtifactRegistry := requestURL.Scheme == artifactRegistryScheme
+	if isArtifactRegistry {
+		requestURL.Scheme = "https"
+		// For Artifact Registry, use google.DefaultClient for ADC if available.
+		if m.googleClient != nil {
+			httpClient = m.googleClient
+		}
+	}
+
+	u := requestURL.JoinPath(paths...).String()
 	resp, err := m.responses.Get(u, func() (response, error) {
 		log.Infof("Fetching response from: %s", u)
-		resp, err := auth.Get(ctx, http.DefaultClient, u)
+		resp, err := auth.Get(ctx, httpClient, u)
 		if err != nil {
 			return response{}, fmt.Errorf("%w: Maven registry query failed: %w", errAPIFailed, err)
 		}
 		defer resp.Body.Close()
 
-		if !slices.Contains([]int{http.StatusOK, http.StatusNotFound, http.StatusUnauthorized}, resp.StatusCode) {
-			// Only cache responses with Status OK, NotFound, or Unauthorized
+		if !slices.Contains([]int{http.StatusOK, http.StatusNotFound, http.StatusUnauthorized, http.StatusForbidden}, resp.StatusCode) {
+			// Only cache responses with Status OK, NotFound, Unauthorized, or Forbidden
 			return response{}, fmt.Errorf("%w: Maven registry query status: %d", errAPIFailed, resp.StatusCode)
 		}
 
@@ -301,6 +344,10 @@ func (m *MavenRegistryAPIClient) get(ctx context.Context, auth *HTTPAuthenticati
 		return err
 	}
 
+	if resp.StatusCode == http.StatusForbidden && isArtifactRegistry {
+		return fmt.Errorf("%w: Maven registry query status: %d (Forbidden). Please check your Application Default Credentials (ADC) have permission to read from %s", errAPIFailed, resp.StatusCode, registry.URL)
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("%w: Maven registry query status: %d", errAPIFailed, resp.StatusCode)
 	}

clients/datasource/maven_registry_auth_test.go

@@ -28,7 +28,7 @@ func TestWithoutRegistriesMaintainsAuthData(t *testing.T) {
 	srv := clienttest.NewMockHTTPServer(t)
 
 	// Create original client with multiple registries
-	client, _ := NewMavenRegistryAPIClient(MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, "")
+	client, _ := NewMavenRegistryAPIClient(t.Context(), MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, "")
 	testRegistry1 := MavenRegistry{
 		URL:             "https://test1.maven.org/maven2/",
 		ID:              "test1",
@@ -39,10 +39,10 @@ func TestWithoutRegistriesMaintainsAuthData(t *testing.T) {
 		ID:               "test2",
 		SnapshotsEnabled: true,
 	}
-	if err := client.AddRegistry(testRegistry1); err != nil {
+	if err := client.AddRegistry(t.Context(), testRegistry1); err != nil {
 		t.Fatalf("failed to add registry %s: %v", testRegistry1.URL, err)
 	}
-	if err := client.AddRegistry(testRegistry2); err != nil {
+	if err := client.AddRegistry(t.Context(), testRegistry2); err != nil {
 		t.Fatalf("failed to add registry %s: %v", testRegistry2.URL, err)
 	}
 

clients/datasource/maven_registry_test.go

@@ -28,7 +28,7 @@ import (
 
 func TestGetProject(t *testing.T) {
 	srv := clienttest.NewMockHTTPServer(t)
-	client, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, "")
+	client, _ := datasource.NewMavenRegistryAPIClient(t.Context(), datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, "")
 	srv.SetResponse(t, "org/example/x.y.z/1.0.0/x.y.z-1.0.0.pom", []byte(`
 	<project>
 	  <groupId>org.example</groupId>
@@ -55,7 +55,7 @@ func TestGetProject(t *testing.T) {
 
 func TestGetProjectSnapshot(t *testing.T) {
 	srv := clienttest.NewMockHTTPServer(t)
-	client, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: srv.URL, SnapshotsEnabled: true}, "")
+	client, _ := datasource.NewMavenRegistryAPIClient(t.Context(), datasource.MavenRegistry{URL: srv.URL, SnapshotsEnabled: true}, "")
 	srv.SetResponse(t, "org/example/x.y.z/3.3.1-SNAPSHOT/maven-metadata.xml", []byte(`
 	<metadata>
 	  <groupId>org.example</groupId>
@@ -107,7 +107,7 @@ func TestGetProjectSnapshot(t *testing.T) {
 
 func TestMultipleRegistry(t *testing.T) {
 	dft := clienttest.NewMockHTTPServer(t)
-	client, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: dft.URL, ReleasesEnabled: true}, "")
+	client, _ := datasource.NewMavenRegistryAPIClient(t.Context(), datasource.MavenRegistry{URL: dft.URL, ReleasesEnabled: true}, "")
 	dft.SetResponse(t, "org/example/x.y.z/maven-metadata.xml", []byte(`
 	<metadata>
 	  <groupId>org.example</groupId>
@@ -138,7 +138,7 @@ func TestMultipleRegistry(t *testing.T) {
 	`))
 
 	srv := clienttest.NewMockHTTPServer(t)
-	if err := client.AddRegistry(datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}); err != nil {
+	if err := client.AddRegistry(t.Context(), datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}); err != nil {
 		t.Fatalf("failed to add registry %s: %v", srv.URL, err)
 	}
 	srv.SetResponse(t, "org/example/x.y.z/maven-metadata.xml", []byte(`
@@ -197,7 +197,7 @@ func TestMultipleRegistry(t *testing.T) {
 
 func TestUpdateDefaultRegistry(t *testing.T) {
 	dft := clienttest.NewMockHTTPServer(t)
-	client, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: dft.URL, ReleasesEnabled: true}, "")
+	client, _ := datasource.NewMavenRegistryAPIClient(t.Context(), datasource.MavenRegistry{URL: dft.URL, ReleasesEnabled: true}, "")
 	dft.SetResponse(t, "org/example/x.y.z/maven-metadata.xml", []byte(`
 	<metadata>
 	  <groupId>org.example</groupId>
@@ -222,7 +222,7 @@ func TestUpdateDefaultRegistry(t *testing.T) {
 	}
 
 	srv := clienttest.NewMockHTTPServer(t)
-	if err := client.AddRegistry(datasource.MavenRegistry{URL: srv.URL, ID: "default", ReleasesEnabled: true}); err != nil {
+	if err := client.AddRegistry(t.Context(), datasource.MavenRegistry{URL: srv.URL, ID: "default", ReleasesEnabled: true}); err != nil {
 		t.Fatalf("failed to add registry %s: %v", srv.URL, err)
 	}
 	srv.SetResponse(t, "org/example/x.y.z/maven-metadata.xml", []byte(`
@@ -252,7 +252,7 @@ func TestUpdateDefaultRegistry(t *testing.T) {
 func TestMavenLocalRegistry(t *testing.T) {
 	tempDir := t.TempDir()
 	srv := clienttest.NewMockHTTPServer(t)
-	client, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, tempDir)
+	client, _ := datasource.NewMavenRegistryAPIClient(t.Context(), datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, tempDir)
 	path := "org/example/x.y.z/1.0.0/x.y.z-1.0.0.pom"
 	resp := []byte(`
 	<project>

clients/resolution/client.go

@@ -16,14 +16,16 @@
 package resolution
 
 import (
+	"context"
+
 	"deps.dev/util/resolve"
 )
 
 // ClientWithRegistries is a resolve.Client that allows package registries to be added.
 type ClientWithRegistries interface {
 	resolve.Client
 	// AddRegistries adds the specified registries to fetch data.
-	AddRegistries(registries []Registry) error
+	AddRegistries(ctx context.Context, registries []Registry) error
 }
 
 // Registry is the interface of a registry to fetch data.

clients/resolution/combined_native_client.go

@@ -50,7 +50,7 @@ func NewCombinedNativeClient(opts CombinedNativeClientOptions) (*CombinedNativeC
 
 // Version returns metadata of a version specified by the VersionKey.
 func (c *CombinedNativeClient) Version(ctx context.Context, vk resolve.VersionKey) (resolve.Version, error) {
-	client, err := c.clientForSystem(vk.System)
+	client, err := c.clientForSystem(ctx, vk.System)
 	if err != nil {
 		return resolve.Version{}, err
 	}
@@ -59,7 +59,7 @@ func (c *CombinedNativeClient) Version(ctx context.Context, vk resolve.VersionKe
 
 // Versions returns all the available versions of the package specified by the given PackageKey.
 func (c *CombinedNativeClient) Versions(ctx context.Context, pk resolve.PackageKey) ([]resolve.Version, error) {
-	client, err := c.clientForSystem(pk.System)
+	client, err := c.clientForSystem(ctx, pk.System)
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +68,7 @@ func (c *CombinedNativeClient) Versions(ctx context.Context, pk resolve.PackageK
 
 // Requirements returns requirements of a version specified by the VersionKey.
 func (c *CombinedNativeClient) Requirements(ctx context.Context, vk resolve.VersionKey) ([]resolve.RequirementVersion, error) {
-	client, err := c.clientForSystem(vk.System)
+	client, err := c.clientForSystem(ctx, vk.System)
 	if err != nil {
 		return nil, err
 	}
@@ -77,20 +77,20 @@ func (c *CombinedNativeClient) Requirements(ctx context.Context, vk resolve.Vers
 
 // MatchingVersions returns versions matching the requirement specified by the VersionKey.
 func (c *CombinedNativeClient) MatchingVersions(ctx context.Context, vk resolve.VersionKey) ([]resolve.Version, error) {
-	client, err := c.clientForSystem(vk.System)
+	client, err := c.clientForSystem(ctx, vk.System)
 	if err != nil {
 		return nil, err
 	}
 	return client.MatchingVersions(ctx, vk)
 }
 
 // AddRegistries adds registries to the MavenRegistryClient.
-func (c *CombinedNativeClient) AddRegistries(registries []Registry) error {
+func (c *CombinedNativeClient) AddRegistries(ctx context.Context, registries []Registry) error {
 	// TODO(#541): Currently only MavenRegistryClient supports adding registries.
 	// We might need to add support for PyPIRegistryClient.
 	// But this AddRegistries method should take a system as input,
 	// so that we can add registries to the corresponding client.
-	client, err := c.clientForSystem(resolve.Maven)
+	client, err := c.clientForSystem(ctx, resolve.Maven)
 	if err != nil {
 		return err
 	}
@@ -99,10 +99,10 @@ func (c *CombinedNativeClient) AddRegistries(registries []Registry) error {
 		// Currently should not happen.
 		return nil
 	}
-	return regCl.AddRegistries(registries)
+	return regCl.AddRegistries(ctx, registries)
 }
 
-func (c *CombinedNativeClient) clientForSystem(sys resolve.System) (resolve.Client, error) {
+func (c *CombinedNativeClient) clientForSystem(ctx context.Context, sys resolve.System) (resolve.Client, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -114,7 +114,7 @@ func (c *CombinedNativeClient) clientForSystem(sys resolve.System) (resolve.Clie
 			if localRegistry != "" {
 				localRegistry = filepath.Join(c.opts.LocalRegistry, "maven")
 			}
-			c.mavenRegistryClient, err = NewMavenRegistryClient(c.opts.MavenRegistry, localRegistry)
+			c.mavenRegistryClient, err = NewMavenRegistryClient(ctx, c.opts.MavenRegistry, localRegistry)
 			if err != nil {
 				return nil, err
 			}

clients/resolution/maven_registry_client.go

@@ -33,8 +33,8 @@ type MavenRegistryClient struct {
 }
 
 // NewMavenRegistryClient makes a new MavenRegistryClient.
-func NewMavenRegistryClient(remote, local string) (*MavenRegistryClient, error) {
-	client, err := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: remote, ReleasesEnabled: true}, local)
+func NewMavenRegistryClient(ctx context.Context, remote, local string) (*MavenRegistryClient, error) {
+	client, err := datasource.NewMavenRegistryAPIClient(ctx, datasource.MavenRegistry{URL: remote, ReleasesEnabled: true}, local)
 	if err != nil {
 		return nil, err
 	}
@@ -174,13 +174,13 @@ func (c *MavenRegistryClient) MatchingVersions(ctx context.Context, vk resolve.V
 }
 
 // AddRegistries adds registries to the MavenRegistryClient.
-func (c *MavenRegistryClient) AddRegistries(registries []Registry) error {
+func (c *MavenRegistryClient) AddRegistries(ctx context.Context, registries []Registry) error {
 	for _, reg := range registries {
 		specific, ok := reg.(datasource.MavenRegistry)
 		if !ok {
 			return errors.New("invalid Maven registry information")
 		}
-		if err := c.api.AddRegistry(specific); err != nil {
+		if err := c.api.AddRegistry(ctx, specific); err != nil {
 			return err
 		}
 	}

extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go

@@ -58,7 +58,7 @@ type Config struct {
 // NewConfig returns the configuration given the URL of the Maven registry to fetch metadata.
 func NewConfig(remote, local string) Config {
 	// No need to check errors since we are using the default Maven Central URL.
-	mavenClient, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{
+	mavenClient, _ := datasource.NewMavenRegistryAPIClient(context.Background(), datasource.MavenRegistry{
 		URL:             remote,
 		ReleasesEnabled: true,
 	}, local)
@@ -117,7 +117,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) (in
 	// Clear the registries that may be from other extraction.
 	e.MavenClient = e.MavenClient.WithoutRegistries()
 	for _, repo := range project.Repositories {
-		if err := e.MavenClient.AddRegistry(datasource.MavenRegistry{
+		if err := e.MavenClient.AddRegistry(ctx, datasource.MavenRegistry{
 			URL:              string(repo.URL),
 			ID:               string(repo.ID),
 			ReleasesEnabled:  repo.Releases.Enabled.Boolean(),
@@ -150,7 +150,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) (in
 			clientRegs[i] = reg
 		}
 		if cl, ok := e.depClient.(resolution.ClientWithRegistries); ok {
-			if err := cl.AddRegistries(clientRegs); err != nil {
+			if err := cl.AddRegistries(ctx, clientRegs); err != nil {
 				return inventory.Inventory{}, err
 			}
 		}

extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go

@@ -478,7 +478,7 @@ func TestExtractor_Extract_WithMockServer(t *testing.T) {
 	</project>
 	`))
 
-	apiClient, err := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, "")
+	apiClient, err := datasource.NewMavenRegistryAPIClient(t.Context(), datasource.MavenRegistry{URL: srv.URL, ReleasesEnabled: true}, "")
 	if err != nil {
 		t.Fatalf("%v", err)
 	}