From c71f38e880254e9b1fb9001e5ddf22c408144a6c Mon Sep 17 00:00:00 2001 From: Wout Mertens Date: Thu, 14 Jul 2022 02:38:21 +0200 Subject: [PATCH] nodejs: add package-lock v2 translator - discovers peer dependencies correctly - adds metadata like os, dev and hasInstallScripts to sources --- flake.nix | 1 + .../nodejs/discoverers/default/default.nix | 43 +++-- .../translators/package-json/default.nix | 9 +- .../translators/package-lock-v2/default.nix | 174 ++++++++++++++++++ .../translators/package-lock-v2/v2-parse.nix | 106 +++++++++++ src/subsystems/translators.nix | 1 - 6 files changed, 313 insertions(+), 21 deletions(-) create mode 100644 src/subsystems/nodejs/translators/package-lock-v2/default.nix create mode 100644 src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix diff --git a/flake.nix b/flake.nix index 21cdde3a5..67f2bc3a1 100644 --- a/flake.nix +++ b/flake.nix @@ -226,6 +226,7 @@ # a dev shell for working on dream2nix # use via 'nix develop . -c $SHELL' + # TODO only for the current system? devShells = forAllSystems (system: pkgs: let makeDevshell = import "${inp.devshell}/modules" pkgs; mkShell = config: diff --git a/src/subsystems/nodejs/discoverers/default/default.nix b/src/subsystems/nodejs/discoverers/default/default.nix index a2fe5bc90..589a9bddc 100644 --- a/src/subsystems/nodejs/discoverers/default/default.nix +++ b/src/subsystems/nodejs/discoverers/default/default.nix @@ -35,9 +35,10 @@ in childrenRemoved; - getTranslatorNames = path: let - nodes = l.readDir path; - packageJson = l.fromJSON (l.readFile "${path}/package.json"); + getTranslatorNames = subTree: let + # TODO use nodejsUtils.getWorkspaceLockFile + packageJson = subTree.files."package.json".jsonContent; + lockJson = subTree.files."package-lock.json".jsonContent or null; translators = # if the package has no dependencies we use the # package-lock translator with `packageLock = null` @@ -46,17 +47,21 @@ && (packageJson.devDependencies or {} == {}) && (packageJson.workspaces or [] == []) then ["package-lock"] + else if lockJson != null + then let + lockVersion = lockJson.lockfileVersion or 0; + in + if lockVersion == 1 + then ["package-lock"] + else if lockVersion == 2 || lockVersion == 3 + then ["package-lock-v2"] + else ["package-json"] else - l.optionals (nodes ? "package-lock.json") ["package-lock"] - ++ l.optionals (nodes ? "yarn.lock") ["yarn-lock"] + l.optionals (subTree.files ? "yarn.lock") ["yarn-lock"] ++ ["package-json"]; in translators; - # returns the parsed package.json of a given directory - getPackageJson = dirPath: - l.fromJSON (l.readFile "${dirPath}/package.json"); - # returns all relative paths to workspaces defined by a glob getWorkspacePaths = glob: tree: if l.hasSuffix "*" glob @@ -121,15 +126,13 @@ makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo: dlib.construct.discoveredProject { inherit subsystem; - name = - (getPackageJson "${tree.fullPath}/${wsRelPath}").name - or "${parentInfo.name}/${wsRelPath}"; + name = (tree.getNodeFromPath wsRelPath).files."package.json".jsonContent.name or "${parentInfo.name}/${wsRelPath}"; relPath = dlib.sanitizeRelativePath "${tree.relPath}/${wsRelPath}"; translators = l.unique ( - (lib.filter (trans: l.elem trans ["package-lock" "yarn-lock"]) parentInfo.translators) - ++ (getTranslatorNames "${tree.fullPath}/${wsRelPath}") + (lib.filter (trans: l.elem trans ["package-lock" "package-lock-v2" "yarn-lock"]) parentInfo.translators) + ++ (getTranslatorNames (tree.getNodeFromPath wsRelPath)) ); subsystemInfo = { workspaceParent = tree.relPath; @@ -152,7 +155,7 @@ }) (tree.directories or {})); in - # skip if not a nodajs project + # skip if not a nodejs project if alreadyDiscovered ? "${tree.relPath}" @@ -165,8 +168,14 @@ currentProjectInfo = dlib.construct.discoveredProject { inherit subsystem; inherit (tree) relPath; - name = tree.files."package.json".jsonContent.name or tree.relPath; - translators = getTranslatorNames tree.fullPath; + name = + tree.files."package.json".jsonContent.name + or ( + if tree.relPath == "" + then "noname" + else tree.relPath + ); + translators = getTranslatorNames tree; subsystemInfo = l.optionalAttrs (workspaces != []) { workspaces = l.map diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index d8e1eaae2..cf0d83c06 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -31,7 +31,7 @@ openssh ] '' - # accroding to the spec, the translator reads the input from a json file + # according to the spec, the translator reads the input from a json file jsonInput=$1 # read the json input @@ -40,6 +40,8 @@ relPath=$(jq '.project.relPath' -c -r $jsonInput) npmArgs=$(jq '.project.subsystemInfo.npmArgs' -c -r $jsonInput) + # TODO: Do we really need to copy everything? Just package.json + .npmrc + # is enough, no? And then pass the lock file to translate separately? cp -r $source/* ./ chmod -R +w ./ newSource=$(pwd) @@ -47,7 +49,8 @@ cd ./$relPath rm -rf package-lock.json yarn.lock - echo "translating in temp dir: $(pwd)" + echo "Translating with npm in temp dir: $(pwd)" + echo "You can avoid this by adding your own package-lock.json file" if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then echo "excluding dev dependencies" @@ -61,7 +64,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput cd $WORKDIR - ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput + ${subsystems.nodejs.translators.package-lock-v2.translateBin} $TMPDIR/newJsonInput ''; # inherit options from package-lock translator diff --git a/src/subsystems/nodejs/translators/package-lock-v2/default.nix b/src/subsystems/nodejs/translators/package-lock-v2/default.nix new file mode 100644 index 000000000..48c34bab0 --- /dev/null +++ b/src/subsystems/nodejs/translators/package-lock-v2/default.nix @@ -0,0 +1,174 @@ +# TODO use translate2 +# TODO use package.json for v1 lock files +{ + dlib, + lib, + ... +}: let + b = builtins; + l = lib // builtins; + nodejsUtils = import ../utils.nix {inherit lib;}; + + translate = { + translatorName, + utils, + pkgs, + ... + }: { + project, + source, + tree, + # translator args + # name + # nodejs + ... + } @ args: let + b = builtins; + + name = + if (args.name or "{automatic}") != "{automatic}" + then args.name + else project.name; + tree = args.tree.getNodeFromPath project.relPath; + relPath = project.relPath; + source = "${args.source}/${relPath}"; + workspaces = project.subsystemInfo.workspaces or []; + + getResolved = tree: project: let + lock = + (nodejsUtils.getWorkspaceLockFile tree project "package-lock.json").jsonContent; + resolved = import ./v2-parse.nix {inherit lib lock source;}; + in + resolved; + + resolved = getResolved args.tree project; + + packageVersion = resolved.self.version or "unknown"; + + rootDependencies = resolved.self.deps; + + identifyGitSource = dep: + # TODO: when integrity is there, and git url is github then use tarball instead + # ! (dep ? integrity) && + dlib.identifyGitUrl dep.url; + + getVersion = dep: dep.version; + + getPath = dep: + lib.removePrefix "file:" dep.url; + + getSource = { + url, + hash, + ... + }: {inherit url hash;}; + + # TODO check that this works with workspaces + extraInfo = b.foldl' (acc: dep: + if dep.extra != {} + then l.recursiveUpdate acc {${dep.pname}.${dep.version} = dep.extra;} + else acc) {} + resolved.allDeps; + + # TODO workspaces + hasBuildScript = let + pkgJson = + (nodejsUtils.getWorkspaceLockFile tree project "package.json").jsonContent; + in + (pkgJson.scripts or {}) ? build; + in + utils.simpleTranslate + ({ + getDepByNameVer, + dependenciesByOriginalID, + ... + }: rec { + inherit translatorName; + location = relPath; + + # values + inputData = resolved.allDeps; + + defaultPackage = name; + + packages = + {"${defaultPackage}" = packageVersion;} + // (nodejsUtils.getWorkspacePackages tree workspaces); + + mainPackageDependencies = resolved.self.deps; + + subsystemName = "nodejs"; + + subsystemAttrs = { + inherit extraInfo hasBuildScript; + nodejsVersion = args.nodejs; + }; + + # functions + serializePackages = inputData: inputData; + + getName = dep: dep.pname; + + inherit getVersion; + + # TODO handle npm link maybe? not sure what it looks like in lock + getSourceType = dep: + if lib.hasPrefix "file:" dep.url + then "path" + else if identifyGitSource dep + then "git" + else "http"; + + sourceConstructors = { + git = dep: + (getSource dep) + // (dlib.parseGitUrl dep.url); + + http = dep: (getSource dep); + + path = dep: (dlib.construct.pathSource { + path = getPath dep; + rootName = project.name; + rootVersion = packageVersion; + }); + }; + + getDependencies = dep: dep.deps; + }); +in rec { + version = 2; + + type = "pure"; + + inherit translate; + + extraArgs = { + name = { + description = "The name of the main package"; + examples = [ + "react" + "@babel/code-frame" + ]; + default = "{automatic}"; + type = "argument"; + }; + + transitiveBinaries = { + description = "Should all the binaries from all modules be available, or only those from dependencies"; + default = false; + type = "boolean"; + }; + + # TODO: this should either be removed or only used to select + # the nodejs version for translating, not for building. + nodejs = { + description = "nodejs version to use for building"; + default = "16"; + examples = [ + "14" + "16" + ]; + type = "argument"; + }; + }; +} diff --git a/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix new file mode 100644 index 000000000..27a2a9f17 --- /dev/null +++ b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix @@ -0,0 +1,106 @@ +# This parses a v2 package-lock.json file. This format includes all information +# to get correct dependencies, including peer dependencies and multiple +# versions. lock.packages is a set that includes the path of each dep, and +# this function teases it apart to know exactly which dep is being resolved. +# The format of the lockfile is documented at +# https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json/ +{ + lib, + lock, + source, +}: +assert (lib.elem lock.lockfileVersion [2 3]); let + b = builtins; + # { "node_modules/@foo/bar/node_modules/meep": pkg; ... } + pkgs = lock.packages; + lockName = lock.name or "unnamed"; + lockVersion = lock.version or "unknown"; + + # First part is always "" and path doesn't start with / + toPath = parts: let + joined = b.concatStringsSep "/node_modules/" parts; + len = b.stringLength joined; + sliced = b.substring 1 len joined; + in + sliced; + toParts = path: b.filter b.isString (b.split "/?node_modules/" path); + + getDep = name: parts: + if b.length parts == 0 + then null + else pkgs.${toPath (parts ++ [name])} or (getDep name (lib.init parts)); + resolveDep = name: parts: isOptional: let + dep = getDep name parts; + in + if dep == null + then + if !isOptional + then b.abort "Cannot resolve dependency ${name} from ${parts}" + else null + else { + inherit name; + inherit (dep) version; + }; + resolveDeps = nameSet: parts: isOptional: + if nameSet == null + then [] + else let + depNames = b.attrNames nameSet; + resolved = b.map (n: resolveDep n parts isOptional) depNames; + in + b.filter (d: d != null) resolved; + + mapPkg = path: let + parts = toParts path; + pname = let + n = lib.last parts; + in + if n == "" + then lockName + else n; + + extraAttrs = { + # platforms this package works on + os = 1; + # this is a dev dependency + dev = 1; + # this is an optional dependency + optional = 1; + # this is an optional dev dependency + devOptional = 1; + # set of binary scripts { name = relativePath } + bin = 1; # pkg needs to run install scripts + hasInstallScript = 1; + }; + getExtra = pkg: b.intersectAttrs extraAttrs pkg; + in + { + version ? "unknown", + # URL to content - only main package is not defined + resolved ? "file://${source}", + # hash for content + integrity ? null, + dependencies ? null, + devDependencies ? null, + peerDependencies ? null, + optionalDependencies ? null, + peerDependenciesMeta ? null, + ... + } @ pkg: let + deps = + lib.unique + ((resolveDeps dependencies parts false) + ++ (resolveDeps devDependencies parts true) + ++ (resolveDeps optionalDependencies parts true) + ++ (resolveDeps peerDependencies parts true) + ++ (resolveDeps peerDependenciesMeta parts true)); + in { + inherit pname version deps; + url = resolved; + hash = integrity; + extra = getExtra pkg; + }; + + allDeps = lib.mapAttrsToList mapPkg pkgs; + self = lib.findFirst (d: d.pname == lockName && d.version == lockVersion) (b.abort "Could not find main package") allDeps; +in {inherit allDeps self;} diff --git a/src/subsystems/translators.nix b/src/subsystems/translators.nix index af191bca3..3b3930f81 100644 --- a/src/subsystems/translators.nix +++ b/src/subsystems/translators.nix @@ -54,7 +54,6 @@ dreamLock = dreamLock'.result or dreamLock'; in dream2nix.utils.dreamLock.toJSON - # don't use nix to detect cycles, this will be more efficient in python (dreamLock // { _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; })