diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13996d764..e6666cd11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,6 +78,7 @@ If you come up with something useful, consider posting it to the Google Group To successfully run the test suite, you will need the following installed: - NPM (requires Node) - Yarn (requires Node) +- PNPM (requires Node) - Bower (requires Node and NPM) - Maven (requires Java) - Gradle (requires Java) diff --git a/Dockerfile b/Dockerfile index 994cf04e9..e2188432c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,15 +15,15 @@ ENV COMPOSER_ALLOW_SUPERUSER 1 # programs needed for building RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - sudo \ - unzip \ - wget \ - gnupg2 \ - apt-utils \ - software-properties-common \ - bzr + build-essential \ + curl \ + sudo \ + unzip \ + wget \ + gnupg2 \ + apt-utils \ + software-properties-common \ + bzr RUN add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git @@ -33,14 +33,18 @@ RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ # install yarn RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - && \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list && \ - apt-get update && \ - apt-get install yarn + echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list && \ + apt-get update && \ + apt-get install yarn # install bower RUN npm install -g bower && \ echo '{ "allow_root": true }' > /root/.bowerrc +# install pnpm +RUN npm install -g pnpm && \ + pnpm version + # install jdk 12 RUN curl -L -o openjdk12.tar.gz https://download.java.net/java/GA/jdk12.0.2/e482c34c86bd4bf8b56c0b35558996b9/10/GPL/openjdk-12.0.2_linux-x64_bin.tar.gz && \ tar xvf openjdk12.tar.gz && \ @@ -97,10 +101,10 @@ ENV GOPATH=/gopath ENV PATH=$PATH:$GOPATH/bin RUN mkdir /gopath && \ - go install github.com/tools/godep@latest && \ - go install github.com/FiloSottile/gvt@latest && \ - go install github.com/kardianos/govendor@latest && \ - go clean -cache + go install github.com/tools/godep@latest && \ + go install github.com/FiloSottile/gvt@latest && \ + go install github.com/kardianos/govendor@latest && \ + go clean -cache #install rvm and glide and godep RUN apt-add-repository -y ppa:rael-gc/rvm && \ @@ -145,10 +149,10 @@ WORKDIR / # install conan RUN apt-get install -y python-dev && \ - pip install --no-cache-dir --ignore-installed six --ignore-installed colorama \ - --ignore-installed requests --ignore-installed chardet \ - --ignore-installed urllib3 \ - --upgrade setuptools && \ + pip install --no-cache-dir --ignore-installed six --ignore-installed colorama \ + --ignore-installed requests --ignore-installed chardet \ + --ignore-installed urllib3 \ + --upgrade setuptools && \ pip3 install --no-cache-dir -Iv conan==1.51.3 && \ conan config install https://github.com/conan-io/conanclientcert.git @@ -156,18 +160,18 @@ RUN apt-get install -y python-dev && \ # install NuGet (w. mono) # https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools#macoslinux RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF &&\ - echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list &&\ - apt-get update &&\ - apt-get install -y mono-complete &&\ - curl -o "/usr/local/bin/nuget.exe" "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" &&\ - curl -o "/usr/local/bin/nugetv3.5.0.exe" "https://dist.nuget.org/win-x86-commandline/v3.5.0/nuget.exe" + echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list &&\ + apt-get update &&\ + apt-get install -y mono-complete &&\ + curl -o "/usr/local/bin/nuget.exe" "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" &&\ + curl -o "/usr/local/bin/nugetv3.5.0.exe" "https://dist.nuget.org/win-x86-commandline/v3.5.0/nuget.exe" # install dotnet core RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb &&\ - sudo dpkg -i packages-microsoft-prod.deb &&\ - rm packages-microsoft-prod.deb &&\ - sudo apt-get update &&\ - sudo apt-get install -y dotnet-runtime-2.1 dotnet-sdk-2.1 dotnet-sdk-2.2 dotnet-sdk-3.0 dotnet-sdk-3.1 + sudo dpkg -i packages-microsoft-prod.deb &&\ + rm packages-microsoft-prod.deb &&\ + sudo apt-get update &&\ + sudo apt-get install -y dotnet-runtime-2.1 dotnet-sdk-2.1 dotnet-sdk-2.2 dotnet-sdk-3.0 dotnet-sdk-3.1 # install Composer # The ARG and ENV are for installing tzdata which is part of this installaion. @@ -190,12 +194,12 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 4F4EA0AAE5 # See https://docs.conda.io/en/latest/miniconda_hashes.html # for latest versions and SHAs. RUN \ - conda_installer=Miniconda3-py38_4.9.2-Linux-x86_64.sh &&\ - ref='1314b90489f154602fd794accfc90446111514a5a72fe1f71ab83e07de9504a7' &&\ - wget -q https://repo.anaconda.com/miniconda/${conda_installer} &&\ - sha=`openssl sha256 "${conda_installer}" | cut -d' ' -f2` &&\ - ([ "$sha" = "${ref}" ] || (echo "Verification failed: ${sha} != ${ref}"; false)) &&\ - (echo; echo "yes") | sh "${conda_installer}" + conda_installer=Miniconda3-py38_4.9.2-Linux-x86_64.sh &&\ + ref='1314b90489f154602fd794accfc90446111514a5a72fe1f71ab83e07de9504a7' &&\ + wget -q https://repo.anaconda.com/miniconda/${conda_installer} &&\ + sha=`openssl sha256 "${conda_installer}" | cut -d' ' -f2` &&\ + ([ "$sha" = "${ref}" ] || (echo "Verification failed: ${sha} != ${ref}"; false)) &&\ + (echo; echo "yes") | sh "${conda_installer}" # install Swift Package Manager # Based on https://github.com/apple/swift-docker/blob/main/5.3/ubuntu/18.04/Dockerfile @@ -225,7 +229,7 @@ RUN curl -o flutter_linux_2.8.1-stable.tar.xz https://storage.googleapis.com/flu && tar xf flutter_linux_2.8.1-stable.tar.xz \ && mv flutter ${FLUTTER_HOME} \ && rm flutter_linux_2.8.1-stable.tar.xz - + ENV PATH=$PATH:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin RUN flutter doctor -v \ && flutter update-packages \ diff --git a/features/features/package_managers/pnpm_spec.rb b/features/features/package_managers/pnpm_spec.rb new file mode 100644 index 000000000..f52636d16 --- /dev/null +++ b/features/features/package_managers/pnpm_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative '../../support/feature_helper' + +describe 'PNPM Dependencies' do + # As a Node developer + # I want to be able to manage PNPM dependencies + + let(:node_developer) { LicenseFinder::TestingDSL::User.new } + + specify 'are shown in reports' do + LicenseFinder::TestingDSL::PNPMProject.create + node_developer.run_license_finder + expect(node_developer).to be_seeing_line 'http-server, 0.11.1, MIT' + end +end diff --git a/features/support/testing_dsl.rb b/features/support/testing_dsl.rb index 50c785951..4bfb9a03b 100644 --- a/features/support/testing_dsl.rb +++ b/features/support/testing_dsl.rb @@ -223,6 +223,16 @@ def add_dep end end + class PNPMProject < Project + def add_dep + add_to_file('package.json', '{"dependencies" : {"http-server": "0.11.1"}}') + end + + def install + shell_out('pnpm install 2>/dev/null') + end + end + class MavenProject < Project def add_dep install_fixture('pom.xml') diff --git a/lib/license_finder/cli/base.rb b/lib/license_finder/cli/base.rb index 06e302806..915fd3422 100644 --- a/lib/license_finder/cli/base.rb +++ b/lib/license_finder/cli/base.rb @@ -47,6 +47,7 @@ def license_finder_config :maven_options, :npm_options, :yarn_options, + :pnpm_options, :pip_requirements_path, :python_version, :rebar_command, diff --git a/lib/license_finder/cli/main.rb b/lib/license_finder/cli/main.rb index d440b4d62..00a6834be 100644 --- a/lib/license_finder/cli/main.rb +++ b/lib/license_finder/cli/main.rb @@ -33,6 +33,7 @@ class Main < Base class_option :maven_options, desc: 'Maven options to append to command. Defaults to empty.' class_option :npm_options, desc: 'npm options to append to command. Defaults to empty.' class_option :yarn_options, desc: 'yarn options to append to command. Defaults to empty.' + class_option :pnpm_options, desc: 'pnpm options to append to command. Defaults to empty.' class_option :pip_requirements_path, desc: 'Path to python requirements file. Defaults to requirements.txt.' class_option :python_version, desc: 'Python version to invoke pip with. Valid versions: 2 or 3. Default: 2' class_option :rebar_command, desc: "Command to use when fetching rebar packages. Only meaningful if used with a Erlang/rebar project. Defaults to 'rebar'." diff --git a/lib/license_finder/configuration.rb b/lib/license_finder/configuration.rb index f0160552f..eccde110b 100644 --- a/lib/license_finder/configuration.rb +++ b/lib/license_finder/configuration.rb @@ -101,6 +101,10 @@ def yarn_options get(:yarn_options) end + def pnpm_options + get(:pnpm_options) + end + def pip_requirements_path get(:pip_requirements_path) end diff --git a/lib/license_finder/core.rb b/lib/license_finder/core.rb index c7e1a916e..38defd3a9 100644 --- a/lib/license_finder/core.rb +++ b/lib/license_finder/core.rb @@ -102,6 +102,7 @@ def options # rubocop:disable Metrics/AbcSize maven_options: config.maven_options, npm_options: config.npm_options, yarn_options: config.yarn_options, + pnpm_options: config.pnpm_options, pip_requirements_path: config.pip_requirements_path, python_version: config.python_version, rebar_command: config.rebar_command, diff --git a/lib/license_finder/package.rb b/lib/license_finder/package.rb index 04a8370a7..c10096b5d 100644 --- a/lib/license_finder/package.rb +++ b/lib/license_finder/package.rb @@ -187,6 +187,7 @@ def log_activation(activation) require 'license_finder/packages/nuget_package' require 'license_finder/packages/conan_package' require 'license_finder/packages/yarn_package' +require 'license_finder/packages/pnpm_package' require 'license_finder/packages/sbt_package' require 'license_finder/packages/cargo_package' require 'license_finder/packages/composer_package' diff --git a/lib/license_finder/package_manager.rb b/lib/license_finder/package_manager.rb index 95c9efcd8..1ab041e6b 100644 --- a/lib/license_finder/package_manager.rb +++ b/lib/license_finder/package_manager.rb @@ -158,6 +158,7 @@ def log_to_file(prep_cmd, contents) require 'license_finder/package_managers/trash' require 'license_finder/package_managers/bundler' require 'license_finder/package_managers/npm' +require 'license_finder/package_managers/pnpm' require 'license_finder/package_managers/yarn' require 'license_finder/package_managers/pip' require 'license_finder/package_managers/pipenv' diff --git a/lib/license_finder/package_managers/pnpm.rb b/lib/license_finder/package_managers/pnpm.rb new file mode 100644 index 000000000..61c402aed --- /dev/null +++ b/lib/license_finder/package_managers/pnpm.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require 'json' +require 'tempfile' + +module LicenseFinder + class PNPM < PackageManager + def initialize(options = {}) + super + @pnpm_options = options[:pnpm_options] + end + + SHELL_COMMAND = 'pnpm licenses list --json --long' + + def possible_package_paths + [project_path.join('pnpm-lock.yaml')] + end + + def self.takes_priority_over + NPM + end + + def current_packages + # check if the minimum version of PNPM is met + raise 'The minimum PNPM version is not met, requires 7.17.0 or later' unless supported_pnpm? + + # check if the project directory has workspace file + cmd = PNPM::SHELL_COMMAND.to_s + cmd += ' --no-color' + cmd += ' --recursive' unless project_has_workspaces == false + cmd += " --dir #{project_path}" unless project_path.nil? + cmd += " #{@pnpm_options}" unless @pnpm_options.nil? + + stdout, stderr, status = Cmd.run(cmd) + raise "Command '#{cmd}' failed to execute: #{stderr}" unless status.success? + + json_objects = JSON.parse(stdout) + get_pnpm_packages(json_objects) + end + + def get_pnpm_packages(json_objects) + packages = [] + incompatible_packages = [] + + json_objects.map do |_, value| + value.each do |pkg| + name = pkg['name'] + version = pkg['version'] + license = pkg['license'] + homepage = pkg['vendorUrl'] + author = pkg['vendorName'] + module_path = pkg['path'] + + package = PNPMPackage.new( + name, + version, + spec_licenses: [license], + homepage: homepage, + authors: author, + install_path: module_path + ) + packages << package + end + end + + packages + incompatible_packages.uniq + end + + def package_management_command + 'pnpm' + end + + def prepare_command + 'pnpm install --no-lockfile --ignore-scripts' + end + + def prepare + prep_cmd = "#{prepare_command}#{production_flag}" + _stdout, stderr, status = Dir.chdir(project_path) { Cmd.run(prep_cmd) } + + return if status.success? + + log_errors stderr + raise "Prepare command '#{prep_cmd}' failed" unless @prepare_no_fail + end + + private + + def project_has_workspaces + Dir.chdir(project_path) do + return File.file?('pnpm-workspace.yaml') + end + end + + # PNPM introduced the licenses command in 7.17.0 + def supported_pnpm? + Dir.chdir(project_path) do + version_string, stderr_str, status = Cmd.run('pnpm --version') + raise "Command 'pnpm -v' failed to execute: #{stderr_str}" unless status.success? + + version = version_string.split('.').map(&:to_i) + major = version[0] + minor = version[1] + patch = version[1] + + return true if major > 7 + return true if major == 7 && minor > 17 + return true if major == 7 && minor == 17 && patch >= 0 + + return false + end + end + + def production_flag + return '' if @ignored_groups.nil? + + @ignored_groups.include?('devDependencies') ? ' --prod' : '' + end + end +end diff --git a/lib/license_finder/packages/pnpm_package.rb b/lib/license_finder/packages/pnpm_package.rb new file mode 100644 index 000000000..6264f26c6 --- /dev/null +++ b/lib/license_finder/packages/pnpm_package.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module LicenseFinder + class PNPMPackage < Package + def package_manager + 'PNPM' + end + + def package_url + "https://www.npmjs.com/package/#{CGI.escape(name)}/v/#{CGI.escape(version)}" + end + end +end diff --git a/lib/license_finder/scanner.rb b/lib/license_finder/scanner.rb index a0fbc196f..0d5f78b67 100644 --- a/lib/license_finder/scanner.rb +++ b/lib/license_finder/scanner.rb @@ -3,7 +3,7 @@ module LicenseFinder class Scanner PACKAGE_MANAGERS = [ - GoModules, GoDep, GoWorkspace, Go15VendorExperiment, Glide, Gvt, Govendor, Trash, Dep, Bundler, NPM, Pip, + GoModules, GoDep, GoWorkspace, Go15VendorExperiment, Glide, Gvt, Govendor, Trash, Dep, Bundler, NPM, PNPM, Pip, Yarn, Bower, Maven, Gradle, CocoaPods, Rebar, Erlangmk, Nuget, Carthage, Mix, Conan, Sbt, Cargo, Dotnet, Composer, Pipenv, Conda, Spm, Pub ].freeze diff --git a/spec/fixtures/all_pms/pnpm-lock.yaml b/spec/fixtures/all_pms/pnpm-lock.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/spec/lib/license_finder/package_managers/pnpm_spec.rb b/spec/lib/license_finder/package_managers/pnpm_spec.rb new file mode 100644 index 000000000..772a4acea --- /dev/null +++ b/spec/lib/license_finder/package_managers/pnpm_spec.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'fakefs/spec_helpers' +require 'json' + +module LicenseFinder + describe PNPM do + let(:root) { '/fake-node-project' } + it_behaves_like 'a PackageManager' + + let(:pnpm_shell_command_output) do + { + 'MIT': [ + { + 'name': 'yn', + 'version': '2.0.0', + 'path': Pathname(root).join('node_modules', 'yn'), + 'license': 'MIT', + 'vendorUrl': 'sindresorhus.com', + 'vendorName': 'Sindre Sorhus' + } + ] + }.to_json + end + + let(:pnpm_incompatible_packages_shell_output) do + { + 'MIT': [ + { + 'name': 'fsevents', + 'version': '1.1.1', + 'path': Pathname(root).join('node_modules', 'fsevents'), + 'license': 'Unknown', + 'vendorUrl': 'github.com/fsevents/fsevents' + } + ] + }.to_json + end + + describe '.prepare' do + subject { PNPM.new(project_path: Pathname(root), logger: double(:logger, active: nil)) } + + include FakeFS::SpecHelpers + before do + FileUtils.mkdir_p(Dir.tmpdir) + FileUtils.mkdir_p(root) + end + + context 'when using PNPM 6 throws error' do + before do + allow(SharedHelpers::Cmd).to receive(:run).with('pnpm --version').and_return(['6.1.4', '', cmd_success]) + end + + context 'when the shell command fails' do + it 'an error is raised' do + allow(SharedHelpers::Cmd).to receive(:run).with(PNPM::SHELL_COMMAND + " #{Pathname(root)}").and_return([nil, 'error', cmd_failure]) + + expect { subject.current_packages }.to raise_error(/The minimum PNPM version is not met, requires 7.17.0 or later/) + end + end + end + + context 'when using PNPM projects' do + before do + allow(SharedHelpers::Cmd).to receive(:run).with('pnpm --version').and_return(['7.17.0', '', cmd_success]) + end + + it 'should call pnpm install with no cli parameters' do + expect(SharedHelpers::Cmd).to receive(:run).with('pnpm install --no-lockfile --ignore-scripts').and_return([pnpm_shell_command_output, '', cmd_success]) + subject.prepare + end + + context 'ignored_groups contains devDependencies' do + subject { PNPM.new(project_path: Pathname(root), ignored_groups: 'devDependencies') } + + it 'should include a production flag' do + expect(SharedHelpers::Cmd).to receive(:run).with('pnpm install --no-lockfile --ignore-scripts --prod').and_return([pnpm_shell_command_output, '', cmd_success]) + subject.prepare + end + end + end + end + + describe '.prepare_command' do + include FakeFS::SpecHelpers + before do + FileUtils.mkdir_p(Dir.tmpdir) + FileUtils.mkdir_p(root) + end + + context 'when in a PNPM project' do + before do + allow(SharedHelpers::Cmd).to receive(:run).with('pnpm --version').and_return(['7.17.0', '', cmd_success]) + end + + subject { PNPM.new(project_path: Pathname(root), logger: double(:logger, active: nil)) } + it 'returns the correct prepare method' do + expect(subject.prepare_command).to eq('pnpm install --no-lockfile --ignore-scripts') + end + end + end + + describe '#current_packages' do + subject { PNPM.new(project_path: Pathname(root), logger: double(:logger, active: nil)) } + + include FakeFS::SpecHelpers + before do + FileUtils.mkdir_p(Dir.tmpdir) + FileUtils.mkdir_p(root) + allow(SharedHelpers::Cmd).to receive(:run).with('pnpm --version').and_return(['7.17.0', '', cmd_success]) + end + + context 'when using PNPM 7.17.0+' do + before do + allow(SharedHelpers::Cmd).to receive(:run).with('pnpm --version').and_return(['7.17.0', '', cmd_success]) + end + end + + it 'displays packages as returned from "pnpm licenses list"' do + allow(SharedHelpers::Cmd).to receive(:run).with(PNPM::SHELL_COMMAND + " --no-color --dir #{Pathname(root)}") do + [pnpm_shell_command_output, '', cmd_success] + end + + expect(subject.current_packages.length).to eq 1 + expect(subject.current_packages.first.name).to eq 'yn' + expect(subject.current_packages.first.version).to eq '2.0.0' + expect(subject.current_packages.first.license_names_from_spec).to eq ['MIT'] + expect(subject.current_packages.first.homepage).to eq 'sindresorhus.com' + expect(subject.current_packages.first.authors).to eq 'Sindre Sorhus' + expect(subject.current_packages.first.install_path).to eq Pathname(root).join('node_modules', 'yn').to_s + end + + it 'displays incompatible packages with license type unknown' do + allow(SharedHelpers::Cmd).to receive(:run).with(PNPM::SHELL_COMMAND + " --no-color --dir #{Pathname(root)}") do + [pnpm_incompatible_packages_shell_output, '', cmd_success] + end + + expect(subject.current_packages.length).to eq 1 + expect(subject.current_packages.last.name).to eq 'fsevents' + expect(subject.current_packages.last.version).to eq '1.1.1' + expect(subject.current_packages.last.license_names_from_spec).to eq ['Unknown'] + end + + context 'ignored_groups contains devDependencies' do + subject { PNPM.new(project_path: Pathname(root), ignored_groups: 'devDependencies') } + it 'should include a production flag' do + expect(SharedHelpers::Cmd).to receive(:run).with("#{PNPM::SHELL_COMMAND} --no-color --dir #{Pathname(root)}") + .and_return([pnpm_shell_command_output, '', cmd_success]) + subject.current_packages + end + end + + context 'when the shell command fails' do + it 'an error is raised' do + allow(SharedHelpers::Cmd).to receive(:run).with(PNPM::SHELL_COMMAND + " --no-color --dir #{Pathname(root)}").and_return([nil, 'error', cmd_failure]) + + expect { subject.current_packages }.to raise_error(/Command 'pnpm licenses list --json --long --no-color --dir #{Pathname(root)}' failed to execute: error/) + end + end + end + + describe '.package_management_command' do + it 'returns the correct package management command' do + expect(subject.package_management_command).to eq('pnpm') + end + end + end +end diff --git a/spec/lib/license_finder/packages/pnpm_package_spec.rb b/spec/lib/license_finder/packages/pnpm_package_spec.rb new file mode 100644 index 000000000..45d101df0 --- /dev/null +++ b/spec/lib/license_finder/packages/pnpm_package_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module LicenseFinder + describe PNPMPackage do + subject { described_class.new('a package', '1.1.1') } + + its(:package_url) { should == 'https://www.npmjs.com/package/a+package/v/1.1.1' } + end +end