internal/stdlib: provide API for import graph of std library

This CL adds two functions for accessing the direct and
transitive imports of the packages of the standard library:

 func Imports(pkgs ...string) iter.Seq[string]
 func Dependencies(pkgs ...string) iter.Seq[string]

These are needed by modernizers so that they can avoid offering
fixes that add an import of, say, "slices" while analyzing a
package that is itself a dependency of "slices".

The compressed graph is generated from the current toolchain;
this may not exactly match the source code being analyzed by
the application, but we expect drift to be small.

Updates golang/go#70815

Change-Id: I2d7180bcff1d1c72ce61b8436a346b8921c02ba9
Reviewed-on: https://go-review.googlesource.com/c/tools/+/653356
Commit-Queue: Alan Donovan <[email protected]>
Reviewed-by: Robert Findley <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Auto-Submit: Alan Donovan <[email protected]>
Reviewed-by: Ian Lance Taylor <[email protected]>

Author: adonovan

internal/stdlib/deps.go

@@ -0,0 +1,36 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package stdlib_test
+
+import (
+	"iter"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/tools/internal/stdlib"
+)
+
+func TestImports(t *testing.T) { testDepsFunc(t, "testdata/nethttp.imports", stdlib.Imports) }
+func TestDeps(t *testing.T)    { testDepsFunc(t, "testdata/nethttp.deps", stdlib.Dependencies) }
+
+// testDepsFunc checks that the specified dependency function applied
+// to net/http returns the set of dependencies in the named file.
+func testDepsFunc(t *testing.T, filename string, depsFunc func(pkgs ...string) iter.Seq[string]) {
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := strings.Split(strings.TrimSpace(string(data)), "\n")
+	got := slices.Collect(depsFunc("net/http"))
+	sort.Strings(want)
+	sort.Strings(got)
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Fatalf("Deps mismatch (-want +got):\n%s", diff)
+	}
+}

internal/stdlib/deps_test.go

@@ -7,18 +7,26 @@
 // The generate command reads all the GOROOT/api/go1.*.txt files and
 // generates a single combined manifest.go file containing the Go
 // standard library API symbols along with versions.
+//
+// It also runs "go list -deps std" and records the import graph. This
+// information may be used, for example, to ensure that tools don't
+// suggest fixes that import package P when analyzing one of P's
+// dependencies.
 package main
 
 import (
 	"bytes"
 	"cmp"
+	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"go/format"
 	"go/types"
 	"io/fs"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -29,6 +37,13 @@ import (
 )
 
 func main() {
+	manifest()
+	deps()
+}
+
+// -- generate std manifest --
+
+func manifest() {
 	pkgs := make(map[string]map[string]symInfo) // package -> symbol -> info
 	symRE := regexp.MustCompile(`^pkg (\S+).*?, (var|func|type|const|method \([^)]*\)) ([\pL\p{Nd}_]+)(.*)`)
 
@@ -131,7 +146,7 @@ func main() {
 
 	// Write the combined manifest.
 	var buf bytes.Buffer
-	buf.WriteString(`// Copyright 2024 The Go Authors. All rights reserved.
+	buf.WriteString(`// Copyright 2025 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
@@ -157,7 +172,7 @@ var PackageSymbols = map[string][]Symbol{
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err := os.WriteFile("manifest.go", fmtbuf, 0666); err != nil {
+	if err := os.WriteFile("manifest.go", fmtbuf, 0o666); err != nil {
 		log.Fatal(err)
 	}
 }
@@ -223,3 +238,109 @@ func removeTypeParam(s string) string {
 	}
 	return s
 }
+
+// -- generate dependency graph --
+
+func deps() {
+	stdout := new(bytes.Buffer)
+	cmd := exec.Command("go", "list", "-deps", "-json", "std")
+	cmd.Stdout = stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatal(err)
+	}
+
+	type Package struct {
+		// go list JSON output
+		ImportPath string   // import path of package in dir
+		Imports    []string // import paths used by this package
+
+		// encoding
+		index int
+		deps  []int // indices of direct imports, sorted
+	}
+	pkgs := make(map[string]*Package)
+	var keys []string
+	for dec := json.NewDecoder(stdout); dec.More(); {
+		var pkg Package
+		if err := dec.Decode(&pkg); err != nil {
+			log.Fatal(err)
+		}
+		pkgs[pkg.ImportPath] = &pkg
+		keys = append(keys, pkg.ImportPath)
+	}
+
+	// Sort and number the packages.
+	// There are 344 as of Mar 2025.
+	slices.Sort(keys)
+	for i, name := range keys {
+		pkgs[name].index = i
+	}
+
+	// Encode the dependencies.
+	for _, pkg := range pkgs {
+		for _, imp := range pkg.Imports {
+			if imp == "C" {
+				continue
+			}
+			pkg.deps = append(pkg.deps, pkgs[imp].index)
+		}
+		slices.Sort(pkg.deps)
+	}
+
+	// Emit the table.
+	var buf bytes.Buffer
+	buf.WriteString(`// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by generate.go. DO NOT EDIT.
+
+package stdlib
+
+type pkginfo struct {
+	name string
+	deps string // list of indices of dependencies, as varint-encoded deltas
+}
+var deps = [...]pkginfo{
+`)
+	for _, name := range keys {
+		prev := 0
+		var deps []int
+		for _, v := range pkgs[name].deps {
+			deps = append(deps, v-prev) // delta
+			prev = v
+		}
+		var data []byte
+		for _, v := range deps {
+			data = binary.AppendUvarint(data, uint64(v))
+		}
+		fmt.Fprintf(&buf, "\t{%q, %q},\n", name, data)
+	}
+	fmt.Fprintln(&buf, "}")
+
+	fmtbuf, err := format.Source(buf.Bytes())
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := os.WriteFile("deps.go", fmtbuf, 0o666); err != nil {
+		log.Fatal(err)
+	}
+
+	// Also generate the data for the test.
+	for _, t := range [...]struct{ flag, filename string }{
+		{"-deps=true", "testdata/nethttp.deps"},
+		{`-f={{join .Imports "\n"}}`, "testdata/nethttp.imports"},
+	} {
+		stdout := new(bytes.Buffer)
+		cmd := exec.Command("go", "list", t.flag, "net/http")
+		cmd.Stdout = stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			log.Fatal(err)
+		}
+		if err := os.WriteFile(t.filename, stdout.Bytes(), 0666); err != nil {
+			log.Fatal(err)
+		}
+	}
+}

internal/stdlib/generate.go

@@ -0,0 +1,89 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package stdlib
+
+// This file provides the API for the import graph of the standard library.
+//
+// Be aware that the compiler-generated code for every package
+// implicitly depends on package "runtime" and a handful of others
+// (see runtimePkgs in GOROOT/src/cmd/internal/objabi/pkgspecial.go).
+
+import (
+	"encoding/binary"
+	"iter"
+	"slices"
+	"strings"
+)
+
+// Imports returns the sequence of packages directly imported by the
+// named standard packages, in name order.
+// The imports of an unknown package are the empty set.
+//
+// The graph is built into the application and may differ from the
+// graph in the Go source tree being analyzed by the application.
+func Imports(pkgs ...string) iter.Seq[string] {
+	return func(yield func(string) bool) {
+		for _, pkg := range pkgs {
+			if i, ok := find(pkg); ok {
+				var depIndex uint64
+				for data := []byte(deps[i].deps); len(data) > 0; {
+					delta, n := binary.Uvarint(data)
+					depIndex += delta
+					if !yield(deps[depIndex].name) {
+						return
+					}
+					data = data[n:]
+				}
+			}
+		}
+	}
+}
+
+// Dependencies returns the set of all dependencies of the named
+// standard packages, including the initial package,
+// in a deterministic topological order.
+// The dependencies of an unknown package are the empty set.
+//
+// The graph is built into the application and may differ from the
+// graph in the Go source tree being analyzed by the application.
+func Dependencies(pkgs ...string) iter.Seq[string] {
+	return func(yield func(string) bool) {
+		for _, pkg := range pkgs {
+			if i, ok := find(pkg); ok {
+				var seen [1 + len(deps)/8]byte // bit set of seen packages
+				var visit func(i int) bool
+				visit = func(i int) bool {
+					bit := byte(1) << (i % 8)
+					if seen[i/8]&bit == 0 {
+						seen[i/8] |= bit
+						var depIndex uint64
+						for data := []byte(deps[i].deps); len(data) > 0; {
+							delta, n := binary.Uvarint(data)
+							depIndex += delta
+							if !visit(int(depIndex)) {
+								return false
+							}
+							data = data[n:]
+						}
+						if !yield(deps[i].name) {
+							return false
+						}
+					}
+					return true
+				}
+				if !visit(i) {
+					return
+				}
+			}
+		}
+	}
+}
+
+// find returns the index of pkg in the deps table.
+func find(pkg string) (int, bool) {
+	return slices.BinarySearchFunc(deps[:], pkg, func(p pkginfo, n string) int {
+		return strings.Compare(p.name, n)
+	})
+}

internal/stdlib/import.go

@@ -6,7 +6,7 @@
 
 // Package stdlib provides a table of all exported symbols in the
 // standard library, along with the version at which they first
-// appeared.
+// appeared. It also provides the import graph of std packages.
 package stdlib
 
 import (