diff --git a/README.md b/README.md index 3226a3b9..2e65f4aa 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,8 @@ typically a few hours ahead of the offline databases, and supports commits; however it currently can produce false negatives for some ecosystems. > While the API supports commits, the detector currently has limited support for -> extracting them - only the `composer.lock` parser includes commit details +> extracting them - only the `composer.lock` & `yarn.lock` parsers include +> commit details You cannot use the API in `--offline` mode, but you can use both the offline databases and the API together; the detector will remove any duplicate results. diff --git a/pkg/lockfile/fixtures/yarn/commits.v1.lock b/pkg/lockfile/fixtures/yarn/commits.v1.lock new file mode 100644 index 00000000..092a290f --- /dev/null +++ b/pkg/lockfile/fixtures/yarn/commits.v1.lock @@ -0,0 +1,84 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"mine1@git+ssh://git@github.com:G-Rath/npm-git-repo-2#0a2d2506c1": + version "1.0.0-alpha.37" + resolved "git+ssh://git@github.com:G-Rath/npm-git-repo-2#0a2d2506c1fe299691fc5db53a2097db3bd615bc" + dependencies: + mine G-Rath/npm-git-repo-1 + +mine2@G-Rath/npm-git-repo-2#0a2d2506c1: + version "0.0.1" + resolved "https://codeload.github.com/G-Rath/npm-git-repo-2/tar.gz/0a2d2506c1fe299691fc5db53a2097db3bd615bc" + dependencies: + mine G-Rath/npm-git-repo-1 + +mine3@G-Rath/npm-git-repo-1: + version "1.2.3" + resolved "https://codeload.github.com/G-Rath/npm-git-repo-1/tar.gz/094e581aaf927d010e4b61d706ba584551dac502" + +"mine4@git+ssh://git@github.com:G-Rath/npm-git-repo-2#another-branch", "mine4@git+ssh://git@github.com:G-Rath/npm-git-repo-2#aa3bdfcb": + version "0.0.2" + resolved "git+ssh://git@github.com:G-Rath/npm-git-repo-2#aa3bdfcb1d845c79f14abb66f60d35b8a3ee5998" + +mine4@G-Rath/npm-git-repo-2#another-branch: + version "0.0.4" + resolved "https://codeload.github.com/G-Rath/npm-git-repo-2/tar.gz/aa3bdfcb1d845c79f14abb66f60d35b8a3ee5998" + +"my-package@git+https://git@github.com/my-org/my-package.git#v1.8.3": + version "1.8.3" + resolved "git+https://git@github.com/my-org/my-package.git#b3bd3f1b3dad036e671251f5258beaae398f983a" + +"@bower_components/angular-animate@git://github.com/angular/bower-angular-animate.git#~1.4.0": + version "1.4.14" + resolved "git://github.com/angular/bower-angular-animate.git#e7f778fc054a086ba3326d898a00fa1bc78650a8" + +"@bower_components/alertify@fabien-d/alertify.js-shim#^0.3.10": + version "0.0.0" + resolved "https://codeload.github.com/fabien-d/alertify.js-shim/tar.gz/e7b6c46d76604d297c389d830817b611c9a8f17c" + +minimist@0.0.8, "minimist@ssh://github.com/substack/minimist.git#0.0.8": + version "0.0.8" + resolved "ssh://github.com/substack/minimist.git#3754568bfd43a841d2d72d7fb54598635aea8fa4" + +"bats-assert@https://github.com/bats-core/bats-assert": + version "2.0.0" + resolved "https://github.com/bats-core/bats-assert#4bdd58d3fbcdce3209033d44d884e87add1d8405" + +"bats-support@https://github.com/bats-core/bats-support": + version "0.3.0" + resolved "https://github.com/bats-core/bats-support#d140a65044b2d6810381935ae7f0c94c7023c8c3" + +"bats@https://github.com/bats-core/bats-core#master": + version "1.5.0" + resolved "https://github.com/bats-core/bats-core#172580d2ce19ee33780b5f1df817bbddced43789" + +"vue@https://github.com/vuejs/vue.git#v2.6.12": + version "2.6.12" + resolved "https://github.com/vuejs/vue.git#bb253db0b3e17124b6d1fe93fbf2db35470a1347" + +"kit@git+https://bitbucket.org/kettlelogic/kit.git": + version "1.0.0" + resolved "git+https://bitbucket.org/kettlelogic/kit.git#5b6830c0252eb73c6024d40a8ff5106d3023a2a6" + dependencies: + ramda "^0.23.0" + source-map-support "^0.4.8" + +"casadistance@git+ssh://git@bitbucket.org/casasoftag/casadistance.git": + version "1.0.0" + resolved "git+ssh://git@bitbucket.org/casasoftag/casadistance.git#f0308391f0c50104182bfb2332a53e4e523a4603" + +"babel-preset-php@gitlab:kornelski/babel-preset-php#master": + version "1.1.1" + resolved "https://gitlab.com/kornelski/babel-preset-php/repository/archive.tar.gz?ref=c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced" + dependencies: + php-parser "^2.0.6" + +"is-number@github:jonschlinkert/is-number#master": + version "2.0.0" + resolved "https://codeload.github.com/jonschlinkert/is-number/tar.gz/d5ac0584ee9ae7bd9288220a39780f155b9ad4c8" + +"is-number@https://dummy-token@github.com/jonschlinkert/is-number.git#master": + version "5.0.0" + resolved "https://dummy-token@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5" diff --git a/pkg/lockfile/fixtures/yarn/commits.v2.lock b/pkg/lockfile/fixtures/yarn/commits.v2.lock new file mode 100644 index 00000000..cae9bc7f --- /dev/null +++ b/pkg/lockfile/fixtures/yarn/commits.v2.lock @@ -0,0 +1,63 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 5 + cacheKey: 8 + +"@my-scope/my-first-package@my-scope/my-first-package#commit=0b824c650d3a03444dbcf2b27a5f3566f6e41358": + version: 0.0.6 + resolution: "@my-scope/my-first-package@https://github.com/my-org/my-first-package.git#commit=0b824c650d3a03444dbcf2b27a5f3566f6e41358" + +"my-second-package@my-org/my-second-package#commit=59e2127b9f9d4fda5f928c4204213b3502cd5bb0": + version: 0.2.2 + resolution: "my-second-package@https://github.com/my-org/my-second-package.git#commit=59e2127b9f9d4fda5f928c4204213b3502cd5bb0" + peerDependencies: + "@my-scope/my-first-package": "*" + checksum: 709b31cf9d0b7deb3ed3d591dbb1f1af3f199fd88c63a93eb02aedacc074902353f5a9776c7895e5fb87a4d72f5218a844bfc4d7422927e11403484c8ba9f4a0 + languageName: node + linkType: hard + +"@typegoose/typegoose@https://github.com/typegoose/typegoose.git#main": + version: 7.2.0 + resolution: "@typegoose/typegoose@https://github.com/typegoose/typegoose.git#commit:3ed06e5097ab929f69755676fee419318aaec73a" + dependencies: + lodash: ^4.17.15 + loglevel: ^1.6.8 + reflect-metadata: ^0.1.13 + semver: ^7.3.2 + tslib: ^2.0.0 + peerDependencies: + "@types/mongoose": ^5.7.21 + mongoose: ^5.9.17 + checksum: f02168420eba060a09b6b3771e17e2ebc224997b5e417e81e0b108128bf4fc5020a1816d15ce2a857ef2a8f33fbb79925a76020e3fc80250b998ad5328458900 + languageName: node + linkType: hard + +"vuejs@https://github.com/vuejs/vue.git": + version: 2.5.0 + resolution: "vuejs@https://github.com/vuejs/vue.git#commit=0948d999f2fddf9f90991956493f976273c5da1f" + checksum: 34739eba9e89a7368f833e52e8d0219c1adb7f9e63ab52b862534b8cafcee8f70e2053e28fe6065a66d5ea78b6e852347d377dfde85083f2ecf1ea642e7a3ad4 + languageName: node + linkType: hard + +# version 4 +"my-third-package@https://github.com/my-org/my-third-package#everything": + version: 0.16.1-dev + resolution: "my-third-package@https://github.com/my-org/my-third-package.git#commit=5675a0aed98e067ff6ecccc5ac674fe8995960e0" + +# version 4 +"my-node-sdk@git+https://github.com/my-org/my-node-sdk.git#v1.1.0": + version: 1.1.0 + resolution: "my-node-sdk@https://github.com/my-org/my-node-sdk.git#commit=053dea9e0b8af442d8f867c8e690d2fb0ceb1bf5" + checksum: 0c5f2734e71e8284d97e2c3dd89c366c8e1603b0934006aed6b4a6247f1448033e8a71aff5c67c9b96307b76e911cbac9961d9865f760013c4f0b27e3906f1c4 + languageName: node + linkType: hard + +# version 5 +"is-really-great@ssh://git@github.com:my-org/is-really-great.git": + version: 1.0.0 + resolution: "is-really-great@ssh://git@github.com:my-org/is-really-great.git#commit=191eeef50c584714e1fb8927d17ee72b3b8c97c4" + checksum: bf868a7451379998b2e2b777ab96e3c13c496f100cb7bf92ce44a425c91c8f1a2d5aed9a77a75bb62a9967c435e51e762b93afcc3f9c4650ca1042f5994756d1 + languageName: node + linkType: hard diff --git a/pkg/lockfile/helpers_test.go b/pkg/lockfile/helpers_test.go index 0e8a6c77..b11a228d 100644 --- a/pkg/lockfile/helpers_test.go +++ b/pkg/lockfile/helpers_test.go @@ -20,7 +20,13 @@ func expectErrContaining(t *testing.T, err error, str string) { } func packageToString(pkg lockfile.PackageDetails) string { - return fmt.Sprintf("%s@%s (%s)", pkg.Name, pkg.Version, pkg.Ecosystem) + commit := pkg.Commit + + if commit == "" { + commit = "" + } + + return fmt.Sprintf("%s@%s (%s, %s)", pkg.Name, pkg.Version, pkg.Ecosystem, commit) } func hasPackage(packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) bool { diff --git a/pkg/lockfile/parse-yarn-lock-v1_test.go b/pkg/lockfile/parse-yarn-lock-v1_test.go index 8f747d68..f3c6d09b 100644 --- a/pkg/lockfile/parse-yarn-lock-v1_test.go +++ b/pkg/lockfile/parse-yarn-lock-v1_test.go @@ -183,3 +183,124 @@ func TestYarnLock_v1_VersionsWithBuildString(t *testing.T) { }, }) } + +func TestYarnLock_v1_Commits(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseYarnLock("fixtures/yarn/commits.v1.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "mine1", + Version: "1.0.0-alpha.37", + Ecosystem: lockfile.YarnEcosystem, + Commit: "0a2d2506c1fe299691fc5db53a2097db3bd615bc", + }, + { + Name: "mine2", + Version: "0.0.1", + Ecosystem: lockfile.YarnEcosystem, + Commit: "0a2d2506c1fe299691fc5db53a2097db3bd615bc", + }, + { + Name: "mine3", + Version: "1.2.3", + Ecosystem: lockfile.YarnEcosystem, + Commit: "094e581aaf927d010e4b61d706ba584551dac502", + }, + { + Name: "mine4", + Version: "0.0.2", + Ecosystem: lockfile.YarnEcosystem, + Commit: "aa3bdfcb1d845c79f14abb66f60d35b8a3ee5998", + }, + { + Name: "mine4", + Version: "0.0.4", + Ecosystem: lockfile.YarnEcosystem, + Commit: "aa3bdfcb1d845c79f14abb66f60d35b8a3ee5998", + }, + { + Name: "my-package", + Version: "1.8.3", + Ecosystem: lockfile.YarnEcosystem, + Commit: "b3bd3f1b3dad036e671251f5258beaae398f983a", + }, + { + Name: "@bower_components/angular-animate", + Version: "1.4.14", + Ecosystem: lockfile.YarnEcosystem, + Commit: "e7f778fc054a086ba3326d898a00fa1bc78650a8", + }, + { + Name: "@bower_components/alertify", + Version: "0.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "e7b6c46d76604d297c389d830817b611c9a8f17c", + }, + { + Name: "minimist", + Version: "0.0.8", + Ecosystem: lockfile.YarnEcosystem, + Commit: "3754568bfd43a841d2d72d7fb54598635aea8fa4", + }, + { + Name: "bats-assert", + Version: "2.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "4bdd58d3fbcdce3209033d44d884e87add1d8405", + }, + { + Name: "bats-support", + Version: "0.3.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "d140a65044b2d6810381935ae7f0c94c7023c8c3", + }, + { + Name: "bats", + Version: "1.5.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "172580d2ce19ee33780b5f1df817bbddced43789", + }, + { + Name: "vue", + Version: "2.6.12", + Ecosystem: lockfile.YarnEcosystem, + Commit: "bb253db0b3e17124b6d1fe93fbf2db35470a1347", + }, + { + Name: "kit", + Version: "1.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "5b6830c0252eb73c6024d40a8ff5106d3023a2a6", + }, + { + Name: "casadistance", + Version: "1.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "f0308391f0c50104182bfb2332a53e4e523a4603", + }, + { + Name: "babel-preset-php", + Version: "1.1.1", + Ecosystem: lockfile.YarnEcosystem, + Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + }, + { + Name: "is-number", + Version: "2.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + }, + { + Name: "is-number", + Version: "5.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + }) +} diff --git a/pkg/lockfile/parse-yarn-lock-v2_test.go b/pkg/lockfile/parse-yarn-lock-v2_test.go index 99b8a7b8..1dd13868 100644 --- a/pkg/lockfile/parse-yarn-lock-v2_test.go +++ b/pkg/lockfile/parse-yarn-lock-v2_test.go @@ -150,3 +150,58 @@ func TestYarnLock_v2_VersionsWithBuildString(t *testing.T) { }, }) } + +func TestYarnLock_v2_Commits(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseYarnLock("fixtures/yarn/commits.v2.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "@my-scope/my-first-package", + Version: "0.0.6", + Ecosystem: lockfile.YarnEcosystem, + Commit: "0b824c650d3a03444dbcf2b27a5f3566f6e41358", + }, + { + Name: "my-second-package", + Version: "0.2.2", + Ecosystem: lockfile.YarnEcosystem, + Commit: "59e2127b9f9d4fda5f928c4204213b3502cd5bb0", + }, + { + Name: "@typegoose/typegoose", + Version: "7.2.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "3ed06e5097ab929f69755676fee419318aaec73a", + }, + { + Name: "vuejs", + Version: "2.5.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "0948d999f2fddf9f90991956493f976273c5da1f", + }, + { + Name: "my-third-package", + Version: "0.16.1-dev", + Ecosystem: lockfile.YarnEcosystem, + Commit: "5675a0aed98e067ff6ecccc5ac674fe8995960e0", + }, + { + Name: "my-node-sdk", + Version: "1.1.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "053dea9e0b8af442d8f867c8e690d2fb0ceb1bf5", + }, + { + Name: "is-really-great", + Version: "1.0.0", + Ecosystem: lockfile.YarnEcosystem, + Commit: "191eeef50c584714e1fb8927d17ee72b3b8c97c4", + }, + }) +} diff --git a/pkg/lockfile/parse-yarn-lock.go b/pkg/lockfile/parse-yarn-lock.go index 7b690900..93abdc17 100644 --- a/pkg/lockfile/parse-yarn-lock.go +++ b/pkg/lockfile/parse-yarn-lock.go @@ -3,6 +3,7 @@ package lockfile import ( "bufio" "fmt" + "net/url" "os" "regexp" "strings" @@ -76,9 +77,77 @@ func determineYarnPackageVersion(group []string) string { return "" } +func determineYarnPackageResolution(group []string) string { + re := regexp.MustCompile(`^ {2}(?:resolution:|resolved) "([^ '"]+)"$`) + + for _, s := range group { + matched := re.FindStringSubmatch(s) + + if matched != nil { + return matched[1] + } + } + + // todo: decide what to do here - maybe panic...? + return "" +} + +func tryExtractCommit(resolution string) string { + // language=GoRegExp + matchers := []string{ + // ssh://... + // git://... + // git+ssh://... + // git+https://... + `(?:^|.+@)(?:git(?:\+(?:ssh|https))?|ssh)://.+#(\w+)$`, + // https://....git/... + `(?:^|.+@)https://.+\.git#(\w+)$`, + `https://codeload\.github\.com(?:/[\w-.]+){2}/tar\.gz/(\w+)$`, + `.+#commit[:=](\w+)$`, + } + + for _, matcher := range matchers { + re := regexp.MustCompile(matcher) + matched := re.FindStringSubmatch(resolution) + + if matched != nil { + return matched[1] + } + } + + u, err := url.Parse(resolution) + + if err == nil { + gitRepoHosts := []string{ + "bitbucket.org", + "github.com", + "gitlab.com", + } + + for _, host := range gitRepoHosts { + if u.Host != host { + continue + } + + if u.RawQuery != "" { + queries := u.Query() + + if queries.Has("ref") { + return queries.Get("ref") + } + } + + return u.Fragment + } + } + + return "" +} + func parsePackageGroup(group []string) PackageDetails { name := extractYarnPackageName(group[0]) version := determineYarnPackageVersion(group) + resolution := determineYarnPackageResolution(group) if version == "" { _, _ = fmt.Fprintf( @@ -92,6 +161,7 @@ func parsePackageGroup(group []string) PackageDetails { Name: name, Version: version, Ecosystem: YarnEcosystem, + Commit: tryExtractCommit(resolution), } }