nodejs: add package-lock v2 translator

- discovers peer dependencies correctly
- adds metadata like os, dev and hasInstallScripts to sources

Author: wmertens

flake.nix

@@ -226,6 +226,7 @@
 
     # a dev shell for working on dream2nix
     # use via 'nix develop . -c $SHELL'
+    # TODO only for the current system?
     devShells = forAllSystems (system: pkgs: let
       makeDevshell = import "${inp.devshell}/modules" pkgs;
       mkShell = config:

src/subsystems/nodejs/discoverers/default/default.nix

@@ -35,9 +35,10 @@
   in
     childrenRemoved;
 
-  getTranslatorNames = path: let
-    nodes = l.readDir path;
-    packageJson = l.fromJSON (l.readFile "${path}/package.json");
+  getTranslatorNames = subTree: let
+    # TODO use nodejsUtils.getWorkspaceLockFile
+    packageJson = subTree.files."package.json".jsonContent;
+    lockJson = subTree.files."package-lock.json".jsonContent or null;
     translators =
       # if the package has no dependencies we use the
       # package-lock translator with `packageLock = null`
@@ -46,17 +47,21 @@
         && (packageJson.devDependencies or {} == {})
         && (packageJson.workspaces or [] == [])
       then ["package-lock"]
+      else if lockJson != null
+      then let
+        lockVersion = lockJson.lockfileVersion or 0;
+      in
+        if lockVersion == 1
+        then ["package-lock"]
+        else if lockVersion == 2 || lockVersion == 3
+        then ["package-lock-v2"]
+        else ["package-json"]
       else
-        l.optionals (nodes ? "package-lock.json") ["package-lock"]
-        ++ l.optionals (nodes ? "yarn.lock") ["yarn-lock"]
+        l.optionals (subTree.files ? "yarn.lock") ["yarn-lock"]
         ++ ["package-json"];
   in
     translators;
 
-  # returns the parsed package.json of a given directory
-  getPackageJson = dirPath:
-    l.fromJSON (l.readFile "${dirPath}/package.json");
-
   # returns all relative paths to workspaces defined by a glob
   getWorkspacePaths = glob: tree:
     if l.hasSuffix "*" glob
@@ -121,15 +126,13 @@
   makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo:
     dlib.construct.discoveredProject {
       inherit subsystem;
-      name =
-        (getPackageJson "${tree.fullPath}/${wsRelPath}").name
-        or "${parentInfo.name}/${wsRelPath}";
+      name = (tree.getNodeFromPath wsRelPath).files."package.json".jsonContent.name or "${parentInfo.name}/${wsRelPath}";
       relPath = dlib.sanitizeRelativePath "${tree.relPath}/${wsRelPath}";
       translators =
         l.unique
         (
-          (lib.filter (trans: l.elem trans ["package-lock" "yarn-lock"]) parentInfo.translators)
-          ++ (getTranslatorNames "${tree.fullPath}/${wsRelPath}")
+          (lib.filter (trans: l.elem trans ["package-lock" "package-lock-v2" "yarn-lock"]) parentInfo.translators)
+          ++ (getTranslatorNames (tree.getNodeFromPath wsRelPath))
         );
       subsystemInfo = {
         workspaceParent = tree.relPath;
@@ -152,7 +155,7 @@
           })
         (tree.directories or {}));
   in
-    # skip if not a nodajs project
+    # skip if not a nodejs project
     if
       alreadyDiscovered
       ? "${tree.relPath}"
@@ -165,8 +168,14 @@
       currentProjectInfo = dlib.construct.discoveredProject {
         inherit subsystem;
         inherit (tree) relPath;
-        name = tree.files."package.json".jsonContent.name or tree.relPath;
-        translators = getTranslatorNames tree.fullPath;
+        name =
+          tree.files."package.json".jsonContent.name
+          or (
+            if tree.relPath == ""
+            then "noname"
+            else tree.relPath
+          );
+        translators = getTranslatorNames tree;
         subsystemInfo = l.optionalAttrs (workspaces != []) {
           workspaces =
             l.map

src/subsystems/nodejs/translators/package-json/default.nix

@@ -31,7 +31,7 @@
       openssh
     ]
     ''
-      # accroding to the spec, the translator reads the input from a json file
+      # according to the spec, the translator reads the input from a json file
       jsonInput=$1
 
       # read the json input
@@ -40,14 +40,17 @@
       relPath=$(jq '.project.relPath' -c -r $jsonInput)
       npmArgs=$(jq '.project.subsystemInfo.npmArgs' -c -r $jsonInput)
 
+      # TODO: Do we really need to copy everything? Just package.json + .npmrc
+      # is enough, no? And then pass the lock file to translate separately?
       cp -r $source/* ./
       chmod -R +w ./
       newSource=$(pwd)
 
       cd ./$relPath
       rm -rf package-lock.json yarn.lock
 
-      echo "translating in temp dir: $(pwd)"
+      echo "Translating with npm in temp dir: $(pwd)"
+      echo "You can avoid this by adding your own package-lock.json file"
 
       if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then
         echo "excluding dev dependencies"
@@ -61,7 +64,7 @@
       jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
 
       cd $WORKDIR
-      ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput
+      ${subsystems.nodejs.translators.package-lock-v2.translateBin} $TMPDIR/newJsonInput
     '';
 
   # inherit options from package-lock translator

src/subsystems/nodejs/translators/package-lock-v2/default.nix

@@ -0,0 +1,174 @@
+# TODO use translate2
+# TODO use package.json for v1 lock files
+{
+  dlib,
+  lib,
+  ...
+}: let
+  b = builtins;
+  l = lib // builtins;
+  nodejsUtils = import ../utils.nix {inherit lib;};
+
+  translate = {
+    translatorName,
+    utils,
+    pkgs,
+    ...
+  }: {
+    project,
+    source,
+    tree,
+    # translator args
+    # name
+    # nodejs
+    ...
+  } @ args: let
+    b = builtins;
+
+    name =
+      if (args.name or "{automatic}") != "{automatic}"
+      then args.name
+      else project.name;
+    tree = args.tree.getNodeFromPath project.relPath;
+    relPath = project.relPath;
+    source = "${args.source}/${relPath}";
+    workspaces = project.subsystemInfo.workspaces or [];
+
+    getResolved = tree: project: let
+      lock =
+        (nodejsUtils.getWorkspaceLockFile tree project "package-lock.json").jsonContent;
+      resolved = import ./v2-parse.nix {inherit lib lock source;};
+    in
+      resolved;
+
+    resolved = getResolved args.tree project;
+
+    packageVersion = resolved.self.version or "unknown";
+
+    rootDependencies = resolved.self.deps;
+
+    identifyGitSource = dep:
+    # TODO: when integrity is there, and git url is github then use tarball instead
+    # ! (dep ? integrity) &&
+      dlib.identifyGitUrl dep.url;
+
+    getVersion = dep: dep.version;
+
+    getPath = dep:
+      lib.removePrefix "file:" dep.url;
+
+    getSource = {
+      url,
+      hash,
+      ...
+    }: {inherit url hash;};
+
+    # TODO check that this works with workspaces
+    extraInfo = b.foldl' (acc: dep:
+      if dep.extra != {}
+      then l.recursiveUpdate acc {${dep.pname}.${dep.version} = dep.extra;}
+      else acc) {}
+    resolved.allDeps;
+
+    # TODO workspaces
+    hasBuildScript = let
+      pkgJson =
+        (nodejsUtils.getWorkspaceLockFile tree project "package.json").jsonContent;
+    in
+      (pkgJson.scripts or {}) ? build;
+  in
+    utils.simpleTranslate
+    ({
+      getDepByNameVer,
+      dependenciesByOriginalID,
+      ...
+    }: rec {
+      inherit translatorName;
+      location = relPath;
+
+      # values
+      inputData = resolved.allDeps;
+
+      defaultPackage = name;
+
+      packages =
+        {"${defaultPackage}" = packageVersion;}
+        // (nodejsUtils.getWorkspacePackages tree workspaces);
+
+      mainPackageDependencies = resolved.self.deps;
+
+      subsystemName = "nodejs";
+
+      subsystemAttrs = {
+        inherit extraInfo hasBuildScript;
+        nodejsVersion = args.nodejs;
+      };
+
+      # functions
+      serializePackages = inputData: inputData;
+
+      getName = dep: dep.pname;
+
+      inherit getVersion;
+
+      # TODO handle npm link maybe? not sure what it looks like in lock
+      getSourceType = dep:
+        if lib.hasPrefix "file:" dep.url
+        then "path"
+        else if identifyGitSource dep
+        then "git"
+        else "http";
+
+      sourceConstructors = {
+        git = dep:
+          (getSource dep)
+          // (dlib.parseGitUrl dep.url);
+
+        http = dep: (getSource dep);
+
+        path = dep: (dlib.construct.pathSource {
+          path = getPath dep;
+          rootName = project.name;
+          rootVersion = packageVersion;
+        });
+      };
+
+      getDependencies = dep: dep.deps;
+    });
+in rec {
+  version = 2;
+
+  type = "pure";
+
+  inherit translate;
+
+  extraArgs = {
+    name = {
+      description = "The name of the main package";
+      examples = [
+        "react"
+        "@babel/code-frame"
+      ];
+      default = "{automatic}";
+      type = "argument";
+    };
+
+    transitiveBinaries = {
+      description = "Should all the binaries from all modules be available, or only those from dependencies";
+      default = false;
+      type = "boolean";
+    };
+
+    # TODO: this should either be removed or only used to select
+    # the nodejs version for translating, not for building.
+    nodejs = {
+      description = "nodejs version to use for building";
+      default = "16";
+      examples = [
+        "14"
+        "16"
+      ];
+      type = "argument";
+    };
+  };
+}