Skip to content

Commit 54cf131

Browse files
wmertenswmertens
committed
nodejs builder: implement tree-of-symlinks
- works out-of-the-box, no node|tsc settings necessary - optimal use of storage, composable - handles cyclic dependencies by co-locating cycles Changes: - remove now-unnecessary code - store binaries in bin/ NixOS standard location, and .bin should only be used for dependencies, not the main package
1 parent 0723fec commit 54cf131

File tree

3 files changed

+197
-294
lines changed

3 files changed

+197
-294
lines changed

‎src/subsystems/nodejs/builders/granular/default.nix‎

Lines changed: 189 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,25 @@
1313
...
1414
}: {
1515
# Funcs
16-
# AttrSet -> Bool) -> AttrSet -> [x]
17-
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
18-
getDependencies, # name: version: -> [ {name=; version=; } ]
19-
getSource, # name: version: -> store-path
16+
# [{name; version;}] -> [{name; version;}]
17+
replaceCyclees,
18+
# name: version: -> [ {name=; version=; } ]
19+
getCyclicDependencies,
20+
# name: version: -> [ {name=; version=; } ]
21+
getDependencies,
22+
# name: version: -> store-path
23+
getSource,
24+
# name: version: -> {type="git"; url=""; hash="";} + extra values from npm packages
25+
getSourceSpec,
2026
# Attributes
21-
subsystemAttrs, # attrset
22-
defaultPackageName, # string
23-
defaultPackageVersion, # string
24-
packages, # list
27+
# attrset
28+
subsystemAttrs,
29+
# string
30+
defaultPackageName,
31+
# string
32+
defaultPackageVersion,
33+
# list
34+
packages,
2535
# attrset of pname -> versions,
2636
# where versions is a list of version strings
2737
packageVersions,
@@ -158,51 +168,128 @@
158168

159169
# Generates a derivation for a specific package name + version
160170
makePackage = name: version: let
161-
pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
162-
163-
deps = getDependencies name version;
164-
165-
nodeDeps =
166-
lib.forEach
167-
deps
168-
(dep: allPackages."${dep.name}"."${dep.version}");
171+
pname = name;
172+
173+
rawDeps = getDependencies name version;
174+
inherit (getCyclicDependencies name version) cycleeDeps cyclicParent isCyclee isThisCycleeFor;
175+
176+
# cycles
177+
# for node, we need to copy any cycles into a single package together
178+
# getCyclicDependencies already cut the cycles for us, into one cyclic (e.g. eslint) and many cyclee (e.g. eslint-util)
179+
# when a package is cyclic:
180+
# - the cyclee deps should not be in the node_modules folder
181+
# - the cyclee deps need to be copied into the package so node can find them all together
182+
# when a package is cyclee:
183+
# - the cyclic dep should not be in the node_modules folder
184+
# when a dep is cyclee:
185+
# - the dep should be replaced with the cyclic parent
186+
187+
# Keep only the deps we can install, assume it all works out
188+
deps = let
189+
myOS = with stdenv.targetPlatform;
190+
if isLinux
191+
then "linux"
192+
else if isDarwin
193+
then "darwin"
194+
else "";
195+
in
196+
replaceCyclees (lib.filter (
197+
dep: let
198+
p = allPackages."${dep.name}"."${dep.version}";
199+
s = p.sourceInfo;
200+
in
201+
# this dep is a cyclee
202+
!(isCyclee dep.name dep.version)
203+
&& (!(s ? os) || lib.any (o: o == myOS) s.os)
204+
# this package is a cyclee
205+
&& !(isThisCycleeFor dep.name dep.version)
206+
)
207+
rawDeps);
208+
209+
nodePkgs =
210+
l.map
211+
(dep: allPackages."${dep.name}"."${dep.version}")
212+
deps;
213+
cycleePkgs =
214+
l.map
215+
(dep: allPackages."${dep.name}"."${dep.version}")
216+
cycleeDeps;
169217

170218
# Derivation building the ./node_modules directory in isolation.
171-
# This is used for the devShell of the current package.
172-
# We do not want to build the full package for the devShell.
173-
nodeModulesDir = pkgs.runCommand "node_modules-${pname}" {} ''
174-
# symlink direct dependencies to ./node_modules
175-
mkdir $out
176-
${l.concatStringsSep "\n" (
177-
l.forEach nodeDeps
178-
(pkg: ''
179-
for dir in $(ls ${pkg}/lib/node_modules/); do
180-
if [[ $dir == @* ]]; then
181-
mkdir -p $out/$dir
182-
ln -s ${pkg}/lib/node_modules/$dir/* $out/$dir/
183-
else
184-
ln -s ${pkg}/lib/node_modules/$dir $out/
219+
makeModules = {
220+
withDev ? false,
221+
withOptionals ? true,
222+
}: let
223+
isMain = isMainPackage name version;
224+
# These flags will only be present if true. Also, dev deps are required for non-main packages
225+
myDeps = lib.filter (dep: let
226+
s = dep.sourceInfo;
227+
in
228+
(withOptionals || !(s ? optional))
229+
&& (!isMain || (withDev || !(s ? dev))))
230+
nodePkgs;
231+
in
232+
if lib.length myDeps == 0
233+
then null
234+
else
235+
pkgs.runCommandLocal "node_modules-${pname}" {} ''
236+
shopt -s nullglob
237+
set -e
238+
239+
mkdir $out
240+
241+
function doLink() {
242+
local n=$(basename $1)
243+
local t="$2/$n"
244+
if [ -e "$t" ]; then
245+
local tl=$(readlink $t)
246+
if [ "$tl" = $1 ]; then
247+
# cyclic dep, all ok
248+
return
249+
fi
250+
echo "Cannot overwrite $tl with $1 - incorrect cycle! Versions issue?" >&2
251+
exit 1
252+
fi
253+
ln -s $1 $t
254+
}
255+
256+
for pkg in ${l.toString myDeps}; do
257+
if [ -d $pkg/lib/node_modules/ ]; then
258+
cd $pkg/lib/node_modules/
259+
for dir in *; do
260+
# special case for namespaced modules
261+
if [[ $dir == @* ]]; then
262+
mkdir -p $out/$dir
263+
for sub in $pkg/lib/node_modules/$dir/*; do
264+
doLink $sub $out/$dir
265+
done
266+
else
267+
doLink $pkg/lib/node_modules/$dir $out
268+
fi
269+
done
185270
fi
186271
done
187-
'')
188-
)}
189-
190-
# symlink transitive executables to ./node_modules/.bin
191-
mkdir $out/.bin
192-
for dep in ${l.toString nodeDeps}; do
193-
for binDir in $(ls -d $dep/lib/node_modules/.bin 2>/dev/null ||:); do
194-
ln -sf $binDir/* $out/.bin/
195-
done
196-
done
197-
'';
198272
199-
passthruDeps =
200-
l.listToAttrs
201-
(l.forEach deps
202-
(dep:
203-
l.nameValuePair
204-
dep.name
205-
allPackages."${dep.name}"."${dep.version}"));
273+
# symlink module executables to ./node_modules/.bin
274+
mkdir $out/.bin
275+
for dep in ${l.toString myDeps}; do
276+
for b in $dep/bin/*; do
277+
# these are all relative symlinks, make absolute; Nix post build will make relative
278+
# last one wins (-f)
279+
ln -sf $dep/bin/$(readlink $b) $out/.bin/$(basename $b)
280+
done
281+
done
282+
'';
283+
prodModules = makeModules {withDev = false;};
284+
devModules = makeModules {withDev = true;};
285+
286+
# passthruDeps =
287+
# l.listToAttrs
288+
# (l.forEach deps
289+
# (dep:
290+
# l.nameValuePair
291+
# dep.name
292+
# allPackages."${dep.name}"."${dep.version}"));
206293

207294
dependenciesJson =
208295
b.toJSON
@@ -232,7 +319,6 @@
232319
inherit
233320
dependenciesJson
234321
electronHeaders
235-
nodeDeps
236322
nodeSources
237323
version
238324
;
@@ -241,26 +327,25 @@
241327

242328
inherit pname;
243329

244-
passthru.dependencies = passthruDeps;
330+
# passthru.dependencies = passthruDeps;
245331

246332
passthru.devShell = pkgs.mkShell {
247-
path = [
248-
nodejs
249-
];
250-
buildInputs = [
251-
nodejs
252-
];
253-
shellHook = ''
254-
# create the ./node_modules directory
255-
if [ -e ./node_modules ] && [ ! -L ./node_modules ]; then
256-
echo -e "\nFailed creating the ./node_modules symlink to ${nodeModulesDir}"
257-
echo -e "\n./node_modules already exists and is a directory, which means it is managed by anaother program. Please delete ./node_modules first and re-enter the dev shell."
258-
else
259-
rm -f ./node_modules
260-
ln -s ${nodeModulesDir} ./node_modules
261-
export PATH="$PATH:$(realpath ./node_modules)/.bin"
262-
fi
263-
'';
333+
path = [nodejs];
334+
buildInputs = [nodejs];
335+
shellHook =
336+
if devModules != null
337+
then ''
338+
# create the ./node_modules directory
339+
if [ -e ./node_modules ] && [ ! -L ./node_modules ]; then
340+
echo -e "\nFailed creating the ./node_modules symlink to '${devModules}'"
341+
echo -e "\n./node_modules already exists and is a directory, which means it is managed by another program. Please delete ./node_modules first and re-enter the dev shell."
342+
else
343+
rm -f ./node_modules
344+
ln -s ${devModules} ./node_modules
345+
export PATH="$PATH:$(realpath ./node_modules)/.bin"
346+
fi
347+
''
348+
else "";
264349
};
265350

266351
installMethod = "symlink";
@@ -277,7 +362,7 @@
277362
buildInputs = [jq nodejs python3];
278363

279364
# prevents running into ulimits
280-
passAsFile = ["dependenciesJson" "nodeDeps"];
365+
passAsFile = ["dependenciesJson"];
281366

282367
preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
283368

@@ -289,9 +374,6 @@
289374
# (see comments below on d2nPatchPhase)
290375
fixPackage = "${./fix-package.py}";
291376

292-
# script to install (symlink or copy) dependencies.
293-
installDeps = "${./install-deps.py}";
294-
295377
# costs performance and doesn't seem beneficial in most scenarios
296378
dontStrip = true;
297379

@@ -374,11 +456,11 @@
374456
'';
375457

376458
# The python script wich is executed in this phase:
377-
# - ensures that the package is compatible to the current system
459+
# - ensures that the package is compatible to the current system (but already filtered above with os)
378460
# - ensures the main version in package.json matches the expected
379461
# - pins dependency versions in package.json
380462
# (some npm commands might otherwise trigger networking)
381-
# - creates symlinks for executables declared in package.json
463+
# - creates symlinks for executables declared in package.json in $out/bin
382464
# Apart from that:
383465
# - Any usage of 'link:' in package.json is replaced with 'file:'
384466
# - If package-lock.json exists, it is deleted, as it might conflict
@@ -388,9 +470,7 @@
388470
rm -f package-lock.json
389471
390472
# repair 'link:' -> 'file:'
391-
mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
392-
cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
393-
rm $nodeModules/$packageName/package.json.old
473+
sed -i 's!link:!file\:!g' $nodeModules/$packageName/package.json
394474
395475
# run python script (see comment above):
396476
python $fixPackage \
@@ -405,33 +485,43 @@
405485
exit 1
406486
fi
407487
408-
# configure typescript
409-
if [ -f ./tsconfig.json ] \
410-
&& node -e 'require("typescript")' &>/dev/null; then
411-
node ${./tsconfig-to-json.js}
412-
${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
413-
| ${pkgs.moreutils}/bin/sponge tsconfig.json
414-
fi
488+
# configure typescript to resolve symlinks locally
489+
# disabled since it should just work
490+
# if [ -f ./tsconfig.json ]; then
491+
# node ${./tsconfig-to-json.js}
492+
# fi
415493
'';
416494

417-
# - installs dependencies into the node_modules directory
418-
# - adds executables of direct node module dependencies to PATH
419-
# - adds the current node module to NODE_PATH
495+
# - links dependencies into the node_modules directory + adds bin to PATH
420496
# - sets HOME=$TMPDIR, as this is required by some npm scripts
421-
# TODO: don't install dev dependencies. Load into NODE_PATH instead
422497
configurePhase = ''
423498
runHook preConfigure
424499
425-
# symlink sub dependencies as well as this imitates npm better
426-
python $installDeps
427-
428-
# add bin path entries collected by python script
429-
if [ -e $TMP/ADD_BIN_PATH ]; then
430-
export PATH="$PATH:$(cat $TMP/ADD_BIN_PATH)"
431-
fi
432-
433-
# add dependencies to NODE_PATH
434-
export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
500+
${
501+
if prodModules != null
502+
then ''
503+
if [ -L $sourceRoot/node_modules ] || [ -e $sourceRoot/node_modules ]; then
504+
echo Warning: The source $sourceRoot includes a node_modules directory. Replacing. >&2
505+
rm -rf $sourceRoot/node_modules
506+
fi
507+
ln -s ${prodModules} $sourceRoot/node_modules
508+
export PATH="$PATH:$sourceRoot/node_modules/.bin"
509+
''
510+
else ""
511+
}
512+
${
513+
# Here we copy cyclee deps into the parent node_modules
514+
# so the cyclic and cyclee can find each other
515+
if cycleePkgs != []
516+
then ''
517+
for dep in ${l.toString cycleePkgs}; do
518+
# TODO handle .bin
519+
# TODO handle @namespace
520+
cp -a $dep/lib/node_modules/* $nodeModules
521+
done
522+
''
523+
else ""
524+
}
435525
436526
export HOME=$TMPDIR
437527
@@ -474,13 +564,6 @@
474564
installPhase = ''
475565
runHook preInstall
476566
477-
echo "Symlinking exectuables to /bin"
478-
if [ -d "$nodeModules/.bin" ]
479-
then
480-
chmod +x $nodeModules/.bin/*
481-
ln -s $nodeModules/.bin $out/bin
482-
fi
483-
484567
echo "Symlinking manual pages"
485568
if [ -d "$nodeModules/$packageName/man" ]
486569
then
@@ -505,7 +588,8 @@
505588
'';
506589
});
507590
in
508-
pkg;
591+
pkg
592+
// {sourceInfo = getSourceSpec name version;};
509593
in
510594
outputs;
511595
}

0 commit comments

Comments
 (0)