From 77ca5013816ffb827fe0f33565ebf126e52822b3 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Sun, 12 Jan 2025 07:14:06 +1300 Subject: [PATCH 01/11] feat: add support for `bun.lock` --- .../language/javascript/bunlock/bunlock.go | 146 ++++ .../javascript/bunlock/bunlock_test.go | 799 ++++++++++++++++++ .../javascript/bunlock/testdata/alias.json5 | 21 + .../bunlock/testdata/blog-sample.json5 | 13 + .../javascript/bunlock/testdata/commits.json5 | 58 ++ .../javascript/bunlock/testdata/empty.json5 | 1 + .../javascript/bunlock/testdata/files.json5 | 19 + .../testdata/nested-dependencies-dup.json5 | 30 + .../testdata/nested-dependencies.json5 | 34 + .../bunlock/testdata/no-packages.json5 | 6 + .../javascript/bunlock/testdata/not-json.txt | 1 + .../bunlock/testdata/one-package-dev.json5 | 14 + .../bunlock/testdata/one-package.json5 | 14 + .../bunlock/testdata/optional-package.json5 | 21 + .../testdata/peer-dependencies-explicit.json5 | 17 + .../testdata/peer-dependencies-implicit.json5 | 16 + .../same-package-different-groups.json5 | 19 + .../testdata/scoped-packages-mixed.json5 | 25 + .../bunlock/testdata/scoped-packages.json5 | 14 + .../bunlock/testdata/two-packages.json5 | 17 + extractor/filesystem/list/list.go | 2 + go.mod | 5 +- go.sum | 2 + 23 files changed, 1293 insertions(+), 1 deletion(-) create mode 100644 extractor/filesystem/language/javascript/bunlock/bunlock.go create mode 100644 extractor/filesystem/language/javascript/bunlock/bunlock_test.go create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/alias.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/blog-sample.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/commits.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/empty.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/files.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies-dup.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/no-packages.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/not-json.txt create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/one-package-dev.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/one-package.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/optional-package.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-explicit.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-implicit.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/same-package-different-groups.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages-mixed.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages.json5 create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/two-packages.json5 diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go new file mode 100644 index 00000000..bbc1f7d8 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -0,0 +1,146 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bunlock extracts bun.lock files +package bunlock + +import ( + "context" + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" + "golang.org/x/exp/maps" + + "github.com/tailscale/hujson" +) + +type bunLockfile struct { + Version int `json:"lockfileVersion"` + Packages map[string][]any `json:"packages"` +} + +// Extractor extracts npm packages from bun.lock files. +type Extractor struct{} + +// Name of the extractor. +func (e Extractor) Name() string { return "javascript/bunlock" } + +// Version of the extractor. +func (e Extractor) Version() int { return 0 } + +// Requirements of the extractor. +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +// FileRequired returns true if the specified file matches bun lockfile patterns. +func (e Extractor) FileRequired(api filesystem.FileAPI) bool { + return filepath.Base(api.Path()) == "bun.lock" +} + +// structurePackageDetails returns the name, version, and commit of a package +// specified as a tuple in a bun.lock +func structurePackageDetails(a []any) (string, string, string) { + str, ok := a[0].(string) + + // todo: should we return an error instead? + if !ok { + return "", "", "" + } + + str, isScoped := strings.CutPrefix(str, "@") + name, version, _ := strings.Cut(str, "@") + + if isScoped { + name = "@" + name + } + + version, commit, _ := strings.Cut(version, "#") + + // bun.lock does not track both the commit and version, + // so if we have a commit then we don't have a version + if commit != "" { + version = "" + } + + if strings.HasPrefix(version, "file:") { + version = "" + } + + return name, version, commit +} + +// Extract extracts packages from bun.lock files passed through the scan input. +func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + var parsedLockfile *bunLockfile + + b, err := io.ReadAll(input.Reader) + + if err != nil { + return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + } + + b, err = hujson.Standardize(b) + + if err != nil { + return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + } + + err = json.Unmarshal(b, &parsedLockfile) + + if err != nil { + return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + } + + packages := maps.Values(parsedLockfile.Packages) + inventories := make([]*extractor.Inventory, len(parsedLockfile.Packages)) + + for i, pkg := range packages { + name, version, commit := structurePackageDetails(pkg) + + inventories[i] = &extractor.Inventory{ + Name: name, + Version: version, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: commit, + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + Locations: []string{input.Path}, + } + } + + return inventories, nil +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(i *extractor.Inventory) *purl.PackageURL { + return &purl.PackageURL{ + Type: purl.TypeNPM, + Name: strings.ToLower(i.Name), + Version: i.Version, + } +} + +// Ecosystem returns the OSV ecosystem ('npm') of the software extracted by this extractor. +func (e Extractor) Ecosystem(_ *extractor.Inventory) string { return "npm" } diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go new file mode 100644 index 00000000..2083482b --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -0,0 +1,799 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bunlock_test + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/bunlock" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" + "github.com/google/osv-scalibr/testing/extracttest" +) + +func TestExtractor_FileRequired(t *testing.T) { + tests := []struct { + name string + inputPath string + want bool + }{ + { + name: "", + inputPath: "", + want: false, + }, + { + name: "", + inputPath: "bun.lock", + want: true, + }, + { + name: "", + inputPath: "path/to/my/bun.lock", + want: true, + }, + { + name: "", + inputPath: "path/to/my/bun.lock/file", + want: false, + }, + { + name: "", + inputPath: "path/to/my/bun.lock.file", + want: false, + }, + { + name: "", + inputPath: "path.to.my.bun.lock", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := bunlock.Extractor{} + got := e.FileRequired(simplefileapi.New(tt.inputPath, nil)) + if got != tt.want { + t.Errorf("FileRequired(%s, FileInfo) got = %v, want %v", tt.inputPath, got, tt.want) + } + }) + } +} + +func TestExtractor_Extract(t *testing.T) { + tests := []extracttest.TestTableEntry{ + { + Name: "invalid json", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/not-json.txt", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "empty", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/empty.json5", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "no packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/no-packages.json5", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "one package", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/one-package.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "wrappy", + Version: "1.0.2", + Locations: []string{"testdata/one-package.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "one package dev", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/one-package-dev.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "wrappy", + Version: "1.0.2", + Locations: []string{"testdata/one-package-dev.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "two packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/two-packages.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "has-flag", + Version: "4.0.0", + Locations: []string{"testdata/two-packages.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "wrappy", + Version: "1.0.2", + Locations: []string{"testdata/two-packages.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "same package in different groups", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/same-package-different-groups.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "has-flag", + Version: "3.0.0", + Locations: []string{"testdata/same-package-different-groups.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "supports-color", + Version: "5.5.0", + Locations: []string{"testdata/same-package-different-groups.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "scoped packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/scoped-packages.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "@typescript-eslint/types", + Version: "5.62.0", + Locations: []string{"testdata/scoped-packages.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "scoped packages mixed", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/scoped-packages-mixed.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "@babel/code-frame", + Version: "7.26.2", + Locations: []string{"testdata/scoped-packages-mixed.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "@babel/helper-validator-identifier", + Version: "7.25.9", + Locations: []string{"testdata/scoped-packages-mixed.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "js-tokens", + Version: "4.0.0", + Locations: []string{"testdata/scoped-packages-mixed.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "picocolors", + Version: "1.1.1", + Locations: []string{"testdata/scoped-packages-mixed.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "wrappy", + Version: "1.0.2", + Locations: []string{"testdata/scoped-packages-mixed.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "optional package", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/optional-package.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "acorn", + Version: "8.14.0", + Locations: []string{"testdata/optional-package.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "fsevents", + Version: "0.3.8", + Locations: []string{"testdata/optional-package.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "nan", + Version: "2.22.0", + Locations: []string{"testdata/optional-package.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "peer dependencies implicit", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/peer-dependencies-implicit.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "acorn-jsx", + Version: "5.3.2", + Locations: []string{"testdata/peer-dependencies-implicit.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "acorn", + Version: "8.14.0", + Locations: []string{"testdata/peer-dependencies-implicit.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "peer dependencies explicit", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/peer-dependencies-explicit.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "acorn-jsx", + Version: "5.3.2", + Locations: []string{"testdata/peer-dependencies-explicit.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "acorn", + Version: "8.14.0", + Locations: []string{"testdata/peer-dependencies-explicit.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "nested dependencies", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/nested-dependencies.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "ansi-styles", + Version: "4.3.0", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "chalk", + Version: "4.1.2", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "color-convert", + Version: "2.0.1", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "color-name", + Version: "1.1.4", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "has-flag", + Version: "2.0.0", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "supports-color", + Version: "5.5.0", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "supports-color", + Version: "7.2.0", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "has-flag", + Version: "3.0.0", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "has-flag", + Version: "4.0.0", + Locations: []string{"testdata/nested-dependencies.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "nested dependencies with duplicate versions", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/nested-dependencies-dup.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "ansi-styles", + Version: "4.3.0", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "chalk", + Version: "4.1.2", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "color-convert", + Version: "2.0.1", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "color-name", + Version: "1.1.4", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "has-flag", + Version: "2.0.0", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "supports-color", + Version: "7.2.0", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "has-flag", + Version: "4.0.0", + Locations: []string{"testdata/nested-dependencies-dup.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "alias", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/alias.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "has-flag", + Version: "4.0.0", + Locations: []string{"testdata/alias.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "supports-color", + Version: "7.2.0", + Locations: []string{"testdata/alias.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "supports-color", + Version: "6.1.0", + Locations: []string{"testdata/alias.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "has-flag", + Version: "3.0.0", + Locations: []string{"testdata/alias.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "commits", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/commits.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "@babel/helper-plugin-utils", + Version: "7.26.5", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "@babel/helper-string-parser", + Version: "7.25.9", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "@babel/helper-validator-identifier", + Version: "7.25.9", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "@babel/parser", + Version: "7.26.5", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "@babel/types", + Version: "7.26.5", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "@prettier/sync", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "527e8ce", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "babel-preset-php", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "1ae6dc1267500360b411ec711b8aeac8c68b2246", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "is-number", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "98e8ff1", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "is-number", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "d5ac058", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "is-number", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "b7aef34", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "jquery", + Version: "3.7.1", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "lodash", + Version: "1.3.1", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "make-synchronized", + Version: "0.2.9", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "php-parser", + Version: "2.2.0", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "prettier", + Version: "3.4.2", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "raven-js", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "91ef2d4", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "slick-carousel", + Version: "", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "fc6f7d8", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "stopwords", + Version: "0.0.1", + Locations: []string{"testdata/commits.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "files", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/files.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "etag", + Version: "", + Locations: []string{"testdata/files.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "lodash", + Version: "1.3.1", + Locations: []string{"testdata/files.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "sample from blog post", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/blog-sample.json5", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "uWebSockets.js", + Version: "", + Locations: []string{"testdata/blog-sample.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "6609a88", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + extr := bunlock.Extractor{} + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } + }) + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/alias.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/alias.json5 new file mode 100644 index 00000000..0e761cd8 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/alias.json5 @@ -0,0 +1,21 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "supports-color": "^7.0.0", + "supports-color-old": "npm:supports-color@^6.0.0", + }, + }, + }, + "packages": { + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + + "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "supports-color-old": ["supports-color@6.1.0", "", { "dependencies": { "has-flag": "^3.0.0" } }, "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ=="], + + "supports-color-old/has-flag": ["has-flag@3.0.0", "", {}, "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/blog-sample.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/blog-sample.json5 new file mode 100644 index 00000000..058c443b --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/blog-sample.json5 @@ -0,0 +1,13 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "dependencies": { + "uWebSocket.js": "uNetworking/uWebSockets.js#v20.51.0", + }, + }, + }, + "packages": { + "uWebSocket.js": ["uWebSockets.js@github:uNetworking/uWebSockets.js#6609a88", {}, "uNetworking-uWebSockets.js-6609a88"], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/commits.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/commits.json5 new file mode 100644 index 00000000..4e3c6c55 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/commits.json5 @@ -0,0 +1,58 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "lodash": "^1.2.1", + "stopwords": "0.0.1", + }, + "devDependencies": { + "@prettier/sync": "github:prettier/prettier-synchronized", + "babel-preset-php": "gitlab:kornelski/babel-preset-php#master", + "is-number-1": "https://github.com/jonschlinkert/is-number.git", + "is-number-2": "https://github.com/jonschlinkert/is-number.git#d5ac058", + "is-number-3": "https://github.com/jonschlinkert/is-number.git#2.0.0", + "raven-js": "getsentry/raven-js#3.23.1", + "slick-carousel": "git://github.com/brianfryer/slick", + }, + }, + }, + "packages": { + "@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.26.5", "", {}, "sha512-RS+jZcRdZdRFzMyr+wcsaqOmld1/EqTghfaBGQQd/WnRdzdlvSZ//kF7U8VQTxf1ynZ4cjUcYgjVGx13ewNPMg=="], + + "@babel/helper-string-parser": ["@babel/helper-string-parser@7.25.9", "", {}, "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA=="], + + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.25.9", "", {}, "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ=="], + + "@babel/parser": ["@babel/parser@7.26.5", "", { "dependencies": { "@babel/types": "^7.26.5" }, "bin": "./bin/babel-parser.js" }, "sha512-SRJ4jYmXRqV1/Xc+TIVG84WjHBXKlxO9sHQnA2Pf12QQEAp1LOh6kDzNHXcUnbH1QI0FDoPPVOt+vyUDucxpaw=="], + + "@babel/types": ["@babel/types@7.26.5", "", { "dependencies": { "@babel/helper-string-parser": "^7.25.9", "@babel/helper-validator-identifier": "^7.25.9" } }, "sha512-L6mZmwFDK6Cjh1nRCLXpa6no13ZIioJDz7mdkzHv399pThrTa/k0nUlNaenOeh2kWu/iaOQYElEpKPUswUa9Vg=="], + + "@prettier/sync": ["@prettier/sync@github:prettier/prettier-synchronized#527e8ce", { "dependencies": { "make-synchronized": "^0.2.8" }, "peerDependencies": { "prettier": "*" } }, "prettier-prettier-synchronized-527e8ce"], + + "babel-preset-php": ["babel-preset-php@git+ssh://gitlab:kornelski/babel-preset-php#1ae6dc1267500360b411ec711b8aeac8c68b2246", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.3", "@babel/parser": "^7.8.4", "php-parser": "^2.2.0" } }, "1ae6dc1267500360b411ec711b8aeac8c68b2246"], + + "is-number-1": ["is-number@github:jonschlinkert/is-number#98e8ff1", {}, "jonschlinkert-is-number-98e8ff1"], + + "is-number-2": ["is-number@github:jonschlinkert/is-number#d5ac058", {}, "jonschlinkert-is-number-d5ac058"], + + "is-number-3": ["is-number@github:jonschlinkert/is-number#b7aef34", {}, "jonschlinkert-is-number-b7aef34"], + + "jquery": ["jquery@3.7.1", "", {}, "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg=="], + + "lodash": ["lodash@1.3.1", "", {}, "sha512-F7AB8u+6d00CCgnbjWzq9fFLpzOMCgq6mPjOW4+8+dYbrnc0obRrC+IHctzfZ1KKTQxX0xo/punrlpOWcf4gpw=="], + + "make-synchronized": ["make-synchronized@0.2.9", "", {}, "sha512-4wczOs8SLuEdpEvp3vGo83wh8rjJ78UsIk7DIX5fxdfmfMJGog4bQzxfvOwq7Q3yCHLC4jp1urPHIxRS/A93gA=="], + + "php-parser": ["php-parser@2.2.0", "", {}, "sha512-zwPZ5fN7Bw+6Od1EMC8JEON/F/nRDEwIGxFoFsssU+c8aTIlu+13JjmnXsNt+Cnsqn8A7Lvy3O8MMGAlsMO8jA=="], + + "prettier": ["prettier@3.4.2", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ=="], + + "raven-js": ["raven-js@github:getsentry/raven-js#91ef2d4", {}, "getsentry-sentry-javascript-91ef2d4"], + + "slick-carousel": ["slick-carousel@github:brianfryer/slick#fc6f7d8", { "peerDependencies": { "jquery": ">=1.8.0" } }, "brianfryer-slick-fc6f7d8"], + + "stopwords": ["stopwords@0.0.1", "", {}, "sha512-7YdkaV8Z9eg5zEz6/z4B7Nm3e1nQuOxAW6V/k+mzdVAuxNfBo9PgfXTlQ1iwBPEwrfY0rI2cfVcNkBxhgj05ZA=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/empty.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/empty.json5 new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/empty.json5 @@ -0,0 +1 @@ +{} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/files.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/files.json5 new file mode 100644 index 00000000..b4dd016c --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/files.json5 @@ -0,0 +1,19 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "lodash": "^1.2.1", + }, + "devDependencies": { + "etag": "file:./deps/etag", + }, + }, + }, + "packages": { + "etag": ["etag@file:deps/etag", {}], + + "lodash": ["lodash@1.3.1", "", {}, "sha512-F7AB8u+6d00CCgnbjWzq9fFLpzOMCgq6mPjOW4+8+dYbrnc0obRrC+IHctzfZ1KKTQxX0xo/punrlpOWcf4gpw=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies-dup.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies-dup.json5 new file mode 100644 index 00000000..953d23ce --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies-dup.json5 @@ -0,0 +1,30 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "has-flag": "^2.0.0", + "supports-color": "^7.0.0", + }, + "devDependencies": { + "chalk": "^4.0.0", + }, + }, + }, + "packages": { + "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + + "has-flag": ["has-flag@2.0.0", "", {}, "sha512-P+1n3MnwjR/Epg9BBo1KT8qbye2g2Ou4sFumihwt6I4tsUX7jnLcX4BTOSKg/B1ZrIYMN9FcEnG4x5a7NB8Eng=="], + + "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "supports-color/has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies.json5 new file mode 100644 index 00000000..d2088f62 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/nested-dependencies.json5 @@ -0,0 +1,34 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "has-flag": "^2.0.0", + "supports-color": "^5.0.0", + }, + "devDependencies": { + "chalk": "^4.0.0", + }, + }, + }, + "packages": { + "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + + "has-flag": ["has-flag@2.0.0", "", {}, "sha512-P+1n3MnwjR/Epg9BBo1KT8qbye2g2Ou4sFumihwt6I4tsUX7jnLcX4BTOSKg/B1ZrIYMN9FcEnG4x5a7NB8Eng=="], + + "supports-color": ["supports-color@5.5.0", "", { "dependencies": { "has-flag": "^3.0.0" } }, "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow=="], + + "chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "supports-color/has-flag": ["has-flag@3.0.0", "", {}, "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="], + + "chalk/supports-color/has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/no-packages.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/no-packages.json5 new file mode 100644 index 00000000..cf6b5031 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/no-packages.json5 @@ -0,0 +1,6 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": {}, + }, +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/not-json.txt b/extractor/filesystem/language/javascript/bunlock/testdata/not-json.txt new file mode 100644 index 00000000..3ae3a213 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/not-json.txt @@ -0,0 +1 @@ +this is not json! diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/one-package-dev.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/one-package-dev.json5 new file mode 100644 index 00000000..a97cf3f8 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/one-package-dev.json5 @@ -0,0 +1,14 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "wrappy": "^1.0.0", + }, + }, + }, + "packages": { + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/one-package.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/one-package.json5 new file mode 100644 index 00000000..a97cf3f8 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/one-package.json5 @@ -0,0 +1,14 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "wrappy": "^1.0.0", + }, + }, + }, + "packages": { + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/optional-package.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/optional-package.json5 new file mode 100644 index 00000000..4f3e26f4 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/optional-package.json5 @@ -0,0 +1,21 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "acorn": "*", + }, + "optionalDependencies": { + "fsevents": "0.3.8", + }, + }, + }, + "packages": { + "acorn": ["acorn@8.14.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA=="], + + "fsevents": ["fsevents@0.3.8", "", { "dependencies": { "nan": "^2.0.2" }, "os": "darwin" }, "sha512-3vlmn1QaPoqSnhnorLFlp3+r3dUCZ8eZlaew+H8QhqB+0YBc9HSITh9wiZo76KYYExTC9DwG6otE/OzwbBLVIw=="], + + "nan": ["nan@2.22.0", "", {}, "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-explicit.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-explicit.json5 new file mode 100644 index 00000000..4a780282 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-explicit.json5 @@ -0,0 +1,17 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "acorn": "*", + "acorn-jsx": "*", + }, + }, + }, + "packages": { + "acorn": ["acorn@8.14.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA=="], + + "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-implicit.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-implicit.json5 new file mode 100644 index 00000000..6fd00ae0 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/peer-dependencies-implicit.json5 @@ -0,0 +1,16 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "acorn-jsx": "*", + }, + }, + }, + "packages": { + "acorn": ["acorn@8.14.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA=="], + + "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/same-package-different-groups.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/same-package-different-groups.json5 new file mode 100644 index 00000000..f98dc049 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/same-package-different-groups.json5 @@ -0,0 +1,19 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "supports-color": "^5.0.0", + }, + "devDependencies": { + "has-flag": "^3.0.0", + }, + }, + }, + "packages": { + "has-flag": ["has-flag@3.0.0", "", {}, "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="], + + "supports-color": ["supports-color@5.5.0", "", { "dependencies": { "has-flag": "^3.0.0" } }, "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages-mixed.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages-mixed.json5 new file mode 100644 index 00000000..438dbe83 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages-mixed.json5 @@ -0,0 +1,25 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "@babel/code-frame": "^7.0.0", + }, + "devDependencies": { + "wrappy": "^1.0.0", + }, + }, + }, + "packages": { + "@babel/code-frame": ["@babel/code-frame@7.26.2", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.25.9", "js-tokens": "^4.0.0", "picocolors": "^1.0.0" } }, "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ=="], + + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.25.9", "", {}, "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ=="], + + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages.json5 new file mode 100644 index 00000000..c2a8ea09 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/scoped-packages.json5 @@ -0,0 +1,14 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "@typescript-eslint/types": "^5.0.0", + }, + }, + }, + "packages": { + "@typescript-eslint/types": ["@typescript-eslint/types@5.62.0", "", {}, "sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ=="], + } +} diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/two-packages.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/two-packages.json5 new file mode 100644 index 00000000..1472508d --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/two-packages.json5 @@ -0,0 +1,17 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "has-flag": "*", + "wrappy": "^1.0.0", + }, + }, + }, + "packages": { + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + } +} diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index 4b754224..e1cb044d 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -25,6 +25,7 @@ import ( // SCALIBR internal extractors. "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/bunlock" "github.com/google/osv-scalibr/extractor/filesystem/containers/containerd" "github.com/google/osv-scalibr/extractor/filesystem/language/cpp/conanlock" @@ -97,6 +98,7 @@ var ( packagelockjson.New(packagelockjson.DefaultConfig()), &pnpmlock.Extractor{}, &yarnlock.Extractor{}, + &bunlock.Extractor{}, } // Python extractors. Python []filesystem.Extractor = []filesystem.Extractor{ diff --git a/go.mod b/go.mod index cb4deaec..383b5dcc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/google/osv-scalibr -go 1.22 +go 1.23 + +toolchain go1.23.4 require ( github.com/BurntSushi/toml v1.3.2 @@ -20,6 +22,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 github.com/package-url/packageurl-go v0.1.2 github.com/spdx/tools-golang v0.5.3 + github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b go.etcd.io/bbolt v1.3.10 go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.31.0 diff --git a/go.sum b/go.sum index 7f678a38..9d417862 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b h1:MNaGusDfB1qxEsl6iVb33Gbe777IKzPP5PDta0xGC8M= +github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b/go.mod h1:EbW0wDK/qEUYI0A5bqq0C2kF8JTQwWONmGDBbzsxxHo= github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= From 0096a65a1e1716579619446725124b0463c6d04a Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 14 Jan 2025 07:11:51 +1300 Subject: [PATCH 02/11] refactor: inline variable --- extractor/filesystem/language/javascript/bunlock/bunlock.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index bbc1f7d8..af6d7e43 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -111,10 +111,9 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) } - packages := maps.Values(parsedLockfile.Packages) inventories := make([]*extractor.Inventory, len(parsedLockfile.Packages)) - for i, pkg := range packages { + for i, pkg := range maps.Values(parsedLockfile.Packages) { name, version, commit := structurePackageDetails(pkg) inventories[i] = &extractor.Inventory{ From 79e177a26f0f41ce05ebafdd859c8716fc44224f Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 14 Jan 2025 07:26:35 +1300 Subject: [PATCH 03/11] fix: return an error if we cannot case the first tuple element to a string --- .../language/javascript/bunlock/bunlock.go | 21 +++++++++++-------- .../javascript/bunlock/bunlock_test.go | 8 +++++++ .../bunlock/testdata/bad-tuple.json5 | 15 +++++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index af6d7e43..8010c746 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -59,12 +59,11 @@ func (e Extractor) FileRequired(api filesystem.FileAPI) bool { // structurePackageDetails returns the name, version, and commit of a package // specified as a tuple in a bun.lock -func structurePackageDetails(a []any) (string, string, string) { +func structurePackageDetails(a []any) (string, string, string, error) { str, ok := a[0].(string) - // todo: should we return an error instead? if !ok { - return "", "", "" + return "", "", "", fmt.Errorf("first element of package tuple is not a string") } str, isScoped := strings.CutPrefix(str, "@") @@ -86,7 +85,7 @@ func structurePackageDetails(a []any) (string, string, string) { version = "" } - return name, version, commit + return name, version, commit, nil } // Extract extracts packages from bun.lock files passed through the scan input. @@ -111,12 +110,16 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) } - inventories := make([]*extractor.Inventory, len(parsedLockfile.Packages)) + inventories := make([]*extractor.Inventory, 0, len(parsedLockfile.Packages)) - for i, pkg := range maps.Values(parsedLockfile.Packages) { - name, version, commit := structurePackageDetails(pkg) + for _, pkg := range maps.Values(parsedLockfile.Packages) { + name, version, commit, err := structurePackageDetails(pkg) - inventories[i] = &extractor.Inventory{ + if err != nil { + return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + } + + inventories = append(inventories, &extractor.Inventory{ Name: name, Version: version, SourceCode: &extractor.SourceCodeIdentifier{ @@ -126,7 +129,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] DepGroupVals: []string{}, }, Locations: []string{input.Path}, - } + }) } return inventories, nil diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go index 2083482b..0271b945 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -133,6 +133,14 @@ func TestExtractor_Extract(t *testing.T) { }, }, }, + { + Name: "one package with bad tuple", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/bad-tuple.json5", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + WantInventory: []*extractor.Inventory{}, + }, { Name: "two packages", InputConfig: extracttest.ScanInputMockConfig{ diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 new file mode 100644 index 00000000..e1bf2a56 --- /dev/null +++ b/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 @@ -0,0 +1,15 @@ +{ + "lockfileVersion": 0, + "workspaces": { + "": { + "name": "bun-lockfile", + "dependencies": { + "wrappy": "^1.0.0", + }, + }, + }, + "packages": { + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "wrappy-bad": [123, "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + } +} From 008bd9adefcf47699028cabbd9a7874be93e791a Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 14 Jan 2025 07:27:34 +1300 Subject: [PATCH 04/11] chore: add comment about `file:` dependencies --- extractor/filesystem/language/javascript/bunlock/bunlock.go | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index 8010c746..2e57eb3b 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -81,6 +81,7 @@ func structurePackageDetails(a []any) (string, string, string, error) { version = "" } + // file dependencies do not have a semantic version recorded if strings.HasPrefix(version, "file:") { version = "" } From a3a77a50bb38003253364ee2389a59ce7a97d4eb Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 14 Jan 2025 07:48:32 +1300 Subject: [PATCH 05/11] feat: switch to `tidwall/jsonc` --- .../filesystem/language/javascript/bunlock/bunlock.go | 10 ++-------- go.mod | 6 ++---- go.sum | 4 ++-- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index 2e57eb3b..d5e61570 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -30,7 +30,7 @@ import ( "github.com/google/osv-scalibr/purl" "golang.org/x/exp/maps" - "github.com/tailscale/hujson" + "github.com/tidwall/jsonc" ) type bunLockfile struct { @@ -99,13 +99,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) } - b, err = hujson.Standardize(b) - - if err != nil { - return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) - } - - err = json.Unmarshal(b, &parsedLockfile) + err = json.Unmarshal(jsonc.ToJSON(b), &parsedLockfile) if err != nil { return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) diff --git a/go.mod b/go.mod index 383b5dcc..6682fa6d 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/google/osv-scalibr -go 1.23 - -toolchain go1.23.4 +go 1.22 require ( github.com/BurntSushi/toml v1.3.2 @@ -22,7 +20,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 github.com/package-url/packageurl-go v0.1.2 github.com/spdx/tools-golang v0.5.3 - github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b + github.com/tidwall/jsonc v0.3.2 go.etcd.io/bbolt v1.3.10 go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.31.0 diff --git a/go.sum b/go.sum index 9d417862..4be4786b 100644 --- a/go.sum +++ b/go.sum @@ -179,10 +179,10 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b h1:MNaGusDfB1qxEsl6iVb33Gbe777IKzPP5PDta0xGC8M= -github.com/tailscale/hujson v0.0.0-20241010212012-29efb4a0184b/go.mod h1:EbW0wDK/qEUYI0A5bqq0C2kF8JTQwWONmGDBbzsxxHo= github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= +github.com/tidwall/jsonc v0.3.2 h1:ZTKrmejRlAJYdn0kcaFqRAKlxxFIC21pYq8vLa4p2Wc= +github.com/tidwall/jsonc v0.3.2/go.mod h1:dw+3CIxqHi+t8eFSpzzMlcVYxKp08UP5CD8/uSFCyJE= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= From 3d329ccad662bf571ab84a9c14c650c330f4cd31 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 14 Jan 2025 07:48:43 +1300 Subject: [PATCH 06/11] refactor: sort imports --- extractor/filesystem/language/javascript/bunlock/bunlock.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index d5e61570..5ea459b3 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -28,9 +28,8 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/osv" "github.com/google/osv-scalibr/plugin" "github.com/google/osv-scalibr/purl" - "golang.org/x/exp/maps" - "github.com/tidwall/jsonc" + "golang.org/x/exp/maps" ) type bunLockfile struct { From 36d6f2e42849bb6ea649b46710840f63d9c84e3a Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 16 Jan 2025 07:08:07 +1300 Subject: [PATCH 07/11] fix: return `nil` slice when erroring --- extractor/filesystem/language/javascript/bunlock/bunlock.go | 6 +++--- .../filesystem/language/javascript/bunlock/bunlock_test.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index 5ea459b3..ce39057e 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -95,13 +95,13 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] b, err := io.ReadAll(input.Reader) if err != nil { - return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + return nil, fmt.Errorf("could not extract from %q: %w", input.Path, err) } err = json.Unmarshal(jsonc.ToJSON(b), &parsedLockfile) if err != nil { - return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + return nil, fmt.Errorf("could not extract from %q: %w", input.Path, err) } inventories := make([]*extractor.Inventory, 0, len(parsedLockfile.Packages)) @@ -110,7 +110,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] name, version, commit, err := structurePackageDetails(pkg) if err != nil { - return []*extractor.Inventory{}, fmt.Errorf("could not extract from %q: %w", input.Path, err) + return nil, fmt.Errorf("could not extract from %q: %w", input.Path, err) } inventories = append(inventories, &extractor.Inventory{ diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go index 0271b945..6d32cff1 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -83,7 +83,7 @@ func TestExtractor_Extract(t *testing.T) { Path: "testdata/not-json.txt", }, WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, - WantInventory: []*extractor.Inventory{}, + WantInventory: nil, }, { Name: "empty", @@ -139,7 +139,7 @@ func TestExtractor_Extract(t *testing.T) { Path: "testdata/bad-tuple.json5", }, WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, - WantInventory: []*extractor.Inventory{}, + WantInventory: nil, }, { Name: "two packages", From 57dee08f44c3684a72bd38cf79e0a91e667ed843 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 16 Jan 2025 07:13:12 +1300 Subject: [PATCH 08/11] feat: don't stop extracting if a tuple is bad --- .../language/javascript/bunlock/bunlock.go | 9 +++++++-- .../language/javascript/bunlock/bunlock_test.go | 12 +++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index ce39057e..6b1f8a89 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -18,6 +18,7 @@ package bunlock import ( "context" "encoding/json" + "errors" "fmt" "io" "path/filepath" @@ -106,11 +107,15 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] inventories := make([]*extractor.Inventory, 0, len(parsedLockfile.Packages)) + var errs []error + for _, pkg := range maps.Values(parsedLockfile.Packages) { name, version, commit, err := structurePackageDetails(pkg) if err != nil { - return nil, fmt.Errorf("could not extract from %q: %w", input.Path, err) + errs = append(errs, fmt.Errorf("could not extract from %q: %w", input.Path, err)) + + continue } inventories = append(inventories, &extractor.Inventory{ @@ -126,7 +131,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] }) } - return inventories, nil + return inventories, errors.Join(errs...) } // ToPURL converts an inventory created by this extractor into a PURL. diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go index 6d32cff1..fbfffb8c 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -139,7 +139,17 @@ func TestExtractor_Extract(t *testing.T) { Path: "testdata/bad-tuple.json5", }, WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, - WantInventory: nil, + WantInventory: []*extractor.Inventory{ + { + Name: "wrappy", + Version: "1.0.2", + Locations: []string{"testdata/bad-tuple.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, }, { Name: "two packages", From 75e03c19a35bc18fd276e87c3374a64289af44b8 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 16 Jan 2025 07:17:02 +1300 Subject: [PATCH 09/11] feat: include the property key whose tuple was bad --- extractor/filesystem/language/javascript/bunlock/bunlock.go | 5 ++--- .../filesystem/language/javascript/bunlock/bunlock_test.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock.go b/extractor/filesystem/language/javascript/bunlock/bunlock.go index 6b1f8a89..9536d6b9 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock.go @@ -30,7 +30,6 @@ import ( "github.com/google/osv-scalibr/plugin" "github.com/google/osv-scalibr/purl" "github.com/tidwall/jsonc" - "golang.org/x/exp/maps" ) type bunLockfile struct { @@ -109,11 +108,11 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] var errs []error - for _, pkg := range maps.Values(parsedLockfile.Packages) { + for key, pkg := range parsedLockfile.Packages { name, version, commit, err := structurePackageDetails(pkg) if err != nil { - errs = append(errs, fmt.Errorf("could not extract from %q: %w", input.Path, err)) + errs = append(errs, fmt.Errorf("could not extract '%s' from %q: %w", key, input.Path, err)) continue } diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go index fbfffb8c..d611c90d 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -138,7 +138,7 @@ func TestExtractor_Extract(t *testing.T) { InputConfig: extracttest.ScanInputMockConfig{ Path: "testdata/bad-tuple.json5", }, - WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad' from"}, WantInventory: []*extractor.Inventory{ { Name: "wrappy", From 9eb2a2d83039e6214e06a5a31360e6ee5b5e4650 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 16 Jan 2025 07:17:17 +1300 Subject: [PATCH 10/11] test: ensure that multiple errors are chained as expected --- .../javascript/bunlock/bunlock_test.go | 22 +++++++++++++++++-- .../bunlock/testdata/bad-tuple.json5 | 3 ++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go index d611c90d..01cf2633 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -134,11 +134,29 @@ func TestExtractor_Extract(t *testing.T) { }, }, { - Name: "one package with bad tuple", + Name: "one package with bad tuple (first error)", InputConfig: extracttest.ScanInputMockConfig{ Path: "testdata/bad-tuple.json5", }, - WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad' from"}, + WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad1' from"}, + WantInventory: []*extractor.Inventory{ + { + Name: "wrappy", + Version: "1.0.2", + Locations: []string{"testdata/bad-tuple.json5"}, + SourceCode: &extractor.SourceCodeIdentifier{}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "one package with bad tuple (second error)", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/bad-tuple.json5", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad2' from"}, WantInventory: []*extractor.Inventory{ { Name: "wrappy", diff --git a/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 b/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 index e1bf2a56..0edddabb 100644 --- a/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 +++ b/extractor/filesystem/language/javascript/bunlock/testdata/bad-tuple.json5 @@ -9,7 +9,8 @@ }, }, "packages": { + "wrappy-bad1": [123, "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], - "wrappy-bad": [123, "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "wrappy-bad2": [{}, "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], } } From ec806a162ead073331e2480c8087064f34d9ee5d Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 16 Jan 2025 07:18:55 +1300 Subject: [PATCH 11/11] refactor: format file --- .../filesystem/language/javascript/bunlock/bunlock_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go index 01cf2633..ecdb926e 100644 --- a/extractor/filesystem/language/javascript/bunlock/bunlock_test.go +++ b/extractor/filesystem/language/javascript/bunlock/bunlock_test.go @@ -138,7 +138,7 @@ func TestExtractor_Extract(t *testing.T) { InputConfig: extracttest.ScanInputMockConfig{ Path: "testdata/bad-tuple.json5", }, - WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad1' from"}, + WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad1' from"}, WantInventory: []*extractor.Inventory{ { Name: "wrappy", @@ -156,7 +156,7 @@ func TestExtractor_Extract(t *testing.T) { InputConfig: extracttest.ScanInputMockConfig{ Path: "testdata/bad-tuple.json5", }, - WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad2' from"}, + WantErr: extracttest.ContainsErrStr{Str: "could not extract 'wrappy-bad2' from"}, WantInventory: []*extractor.Inventory{ { Name: "wrappy",