-
Notifications
You must be signed in to change notification settings - Fork 371
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: Unify where clients and interfaces are initialized, and how…
… extractors are defined (#1500) After some discussions, this PR properly groups where clients, interfaces, and extractors so that they are all defined in sensible places that's easy to extend and modify in the future, and improve readability of the code. The scanning process is now: - Given CLI flags -> enable the clients + matchers we are able to. <sub>(We assume that initialising a client is not an expensive operation, nor does it make any network requests, so e.g. even if we don't scan a pomxml file, we can still initialise the Maven client)</sub> - Given list of clients -> We build a list of extractors that can successfully function with the available clients - Given list of extractors -> Perform the commanded scanning operations with the list of extractors to return an inventory list - Given list of inventories -> Perform matching and enrichment with the list of available matchers. - Build result into JSON format - Return error if vuln found. This PR also removes deprecated errors, renames errors to the proper format with Err at the front to remove the nolint flags.
- Loading branch information
1 parent
17e7a15
commit a6d4ec8
Showing
10 changed files
with
309 additions
and
234 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| package scanners | ||
|
|
||
| import ( | ||
| "github.com/google/osv-scalibr/extractor/filesystem" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/cpp/conanlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/dotnet/packageslockjson" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/erlang/mixlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gobinary" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/golang/gomod" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradlelockfile" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradleverificationmetadataxml" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/yarnlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/php/composerlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/python/pdmlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/language/rust/cargolock" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/os/apk" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/os/dpkg" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx" | ||
| "github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx" | ||
| "github.com/google/osv-scanner/internal/osvdev" | ||
| "github.com/google/osv-scanner/internal/resolution/client" | ||
| "github.com/google/osv-scanner/internal/resolution/datasource" | ||
| "github.com/google/osv-scanner/internal/scalibrextract/filesystem/vendored" | ||
| "github.com/google/osv-scanner/internal/scalibrextract/language/java/pomxmlnet" | ||
| "github.com/google/osv-scanner/internal/scalibrextract/language/javascript/nodemodules" | ||
| "github.com/google/osv-scanner/internal/scalibrextract/vcs/gitrepo" | ||
| "github.com/ossf/osv-schema/bindings/go/osvschema" | ||
| ) | ||
|
|
||
| var sbomExtractors = []filesystem.Extractor{ | ||
| spdx.Extractor{}, | ||
| cdx.Extractor{}, | ||
| } | ||
|
|
||
| var lockfileExtractors = []filesystem.Extractor{ | ||
| conanlock.Extractor{}, | ||
| packageslockjson.Extractor{}, | ||
| mixlock.Extractor{}, | ||
| pubspec.Extractor{}, | ||
| gomod.Extractor{}, | ||
| gradlelockfile.Extractor{}, | ||
| gradleverificationmetadataxml.Extractor{}, | ||
| packagelockjson.Extractor{}, | ||
| pnpmlock.Extractor{}, | ||
| yarnlock.Extractor{}, | ||
| composerlock.Extractor{}, | ||
| pipfilelock.Extractor{}, | ||
| pdmlock.Extractor{}, | ||
| poetrylock.Extractor{}, | ||
| requirements.Extractor{}, | ||
| renvlock.Extractor{}, | ||
| gemfilelock.Extractor{}, | ||
| cargolock.Extractor{}, | ||
| } | ||
|
|
||
| // BuildLockfileExtractors returns all relevant extractors for lockfile scanning given the required clients | ||
| // All clients can be nil, and if nil the extractors requiring those clients will not be returned. | ||
| func BuildLockfileExtractors(dependencyClients map[osvschema.Ecosystem]client.DependencyClient, mavenAPIClient *datasource.MavenRegistryAPIClient) []filesystem.Extractor { | ||
| extractorsToUse := lockfileExtractors | ||
|
|
||
| if dependencyClients[osvschema.EcosystemMaven] != nil && mavenAPIClient != nil { | ||
| extractorsToUse = append(extractorsToUse, pomxmlnet.Extractor{ | ||
| DependencyClient: dependencyClients[osvschema.EcosystemMaven], | ||
| MavenRegistryAPIClient: mavenAPIClient, | ||
| }) | ||
| } else { | ||
| extractorsToUse = append(extractorsToUse, pomxml.Extractor{}) | ||
| } | ||
|
|
||
| return extractorsToUse | ||
| } | ||
|
|
||
| // BuildSBOMExtractors returns extractors relevant to SBOM extraction | ||
| func BuildSBOMExtractors() []filesystem.Extractor { | ||
| return sbomExtractors | ||
| } | ||
|
|
||
| // BuildWalkerExtractors returns all relevant extractors for directory scanning given the required clients | ||
| // All clients can be nil, and if nil the extractors requiring those clients will not be returned. | ||
| func BuildWalkerExtractors( | ||
| skipGit bool, | ||
| osvdevClient *osvdev.OSVClient, | ||
| dependencyClients map[osvschema.Ecosystem]client.DependencyClient, | ||
| mavenAPIClient *datasource.MavenRegistryAPIClient) []filesystem.Extractor { | ||
| relevantExtractors := []filesystem.Extractor{} | ||
|
|
||
| if !skipGit { | ||
| relevantExtractors = append(relevantExtractors, gitrepo.Extractor{}) | ||
| } | ||
| relevantExtractors = append(relevantExtractors, lockfileExtractors...) | ||
| relevantExtractors = append(relevantExtractors, sbomExtractors...) | ||
|
|
||
| if osvdevClient != nil { | ||
| relevantExtractors = append(relevantExtractors, vendored.Extractor{ | ||
| ScanGitDir: skipGit, | ||
| OSVClient: osvdevClient, | ||
| }) | ||
| } | ||
|
|
||
| if dependencyClients[osvschema.EcosystemMaven] != nil && mavenAPIClient != nil { | ||
| relevantExtractors = append(relevantExtractors, pomxmlnet.Extractor{ | ||
| DependencyClient: dependencyClients[osvschema.EcosystemMaven], | ||
| MavenRegistryAPIClient: mavenAPIClient, | ||
| }) | ||
| } else { | ||
| relevantExtractors = append(relevantExtractors, pomxml.Extractor{}) | ||
| } | ||
|
|
||
| return relevantExtractors | ||
| } | ||
|
|
||
| // BuildArtifactExtractors returns all relevant extractors for artifact scanning given the required clients | ||
| // All clients can be nil, and if nil the extractors requiring those clients will not be returned. | ||
| func BuildArtifactExtractors() []filesystem.Extractor { | ||
| extractorsToUse := []filesystem.Extractor{ | ||
| nodemodules.Extractor{}, | ||
| apk.New(apk.DefaultConfig()), | ||
| gobinary.New(gobinary.DefaultConfig()), | ||
| // TODO: Add tests for debian containers | ||
| dpkg.New(dpkg.DefaultConfig()), | ||
| } | ||
|
|
||
| return extractorsToUse | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.