From 0b408887659990a1c4e07d96adc6ca62c8b26bcf Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 6 Mar 2020 16:48:19 +0800 Subject: [PATCH 1/5] example - add files that trigger dedupe --- example/files/a1.js | 1 + example/files/a2.js | 1 + example/files/x.js | 3 ++- example/files/y.js | 3 ++- 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 example/files/a1.js create mode 100644 example/files/a2.js diff --git a/example/files/a1.js b/example/files/a1.js new file mode 100644 index 0000000..0f251a3 --- /dev/null +++ b/example/files/a1.js @@ -0,0 +1 @@ +module.exports = 42 \ No newline at end of file diff --git a/example/files/a2.js b/example/files/a2.js new file mode 100644 index 0000000..0f251a3 --- /dev/null +++ b/example/files/a2.js @@ -0,0 +1 @@ +module.exports = 42 \ No newline at end of file diff --git a/example/files/x.js b/example/files/x.js index 9bde3e9..95685c6 100644 --- a/example/files/x.js +++ b/example/files/x.js @@ -1,3 +1,4 @@ +var a = require('./a1.js'); var z = require('./z.js'); var w = require('./w.js'); -console.log(z(5) * w(2)); +console.log(a * z(5) * w(2)); diff --git a/example/files/y.js b/example/files/y.js index be021ec..398d08b 100644 --- a/example/files/y.js +++ b/example/files/y.js @@ -1,2 +1,3 @@ +var a = require('./a2.js'); var z = require('./z.js'); -console.log(z(2) + 111); +console.log(a + z(2) + 111); From c8195efe6d0433f1cb0e4a1e15fe7213de28bf31 Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 6 Mar 2020 16:50:21 +0800 Subject: [PATCH 2/5] update browserify and add dedupe: true option to example --- example/api.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example/api.js b/example/api.js index 9d88f64..e591eed 100644 --- a/example/api.js +++ b/example/api.js @@ -2,6 +2,6 @@ var browserify = require('browserify'); var fs = require('fs'); var files = [ './files/x.js', './files/y.js' ]; -var b = browserify(files); +var b = browserify(files, { dedupe: false }); b.plugin('../', { o: [ 'bundle/x.js', 'bundle/y.js' ] }); b.bundle().pipe(fs.createWriteStream('bundle/common.js')); diff --git a/package.json b/package.json index edc82a3..9a831cb 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "devDependencies": { "browser-unpack": "^1.1.1", - "browserify": "^11.0.1", + "browserify": "^16.5.0", "concat-stream": "^1.4.6", "mkdirp": "~0.5.0", "module-deps": "^3.9.0", From dd34b23089142840e80b23e59e694a10e44385b5 Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 6 Mar 2020 16:58:44 +0800 Subject: [PATCH 3/5] add comments and improve readability --- index.js | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/index.js b/index.js index 7367e97..d13dbd6 100644 --- a/index.js +++ b/index.js @@ -26,9 +26,9 @@ module.exports = function f (b, opts) { .concat(opts._).filter(Boolean); var needRecords = !files.length; - + var outopt = defined(opts.outputs, opts.output, opts.o); - + opts.objectMode = true; opts.raw = true; opts.rmap = {}; @@ -44,6 +44,7 @@ module.exports = function f (b, opts) { function addHooks () { b.pipeline.get('record').push(through.obj(function(row, enc, next) { + // record entrypoints if (row.file && needRecords) { files.push(row.file); } @@ -65,6 +66,7 @@ module.exports = function f (b, opts) { outputs = []; } + // utility for creating output streams function moreOutputs (file) { if (isarray(outopt)) return []; if (!outopt) return []; @@ -72,33 +74,43 @@ module.exports = function f (b, opts) { return [ outpipe(outopt, xopts) ]; } - var pipelines = files.reduce(function (acc, x, ix) { + // create a pipeline (labeled-stream-splicer) for every entry file + // with only a packer + var pipelines = files.reduce(function (acc, file, index) { var pipeline = splicer.obj([ 'pack', [ pack(packOpts) ], 'wrap', [] ]); - if (ix >= outputs.length) { - outputs.push.apply(outputs, moreOutputs(x)); + // if not enough specified outputs, possibly add additional outputs (?) + if (index >= outputs.length) { + outputs.push.apply(outputs, moreOutputs(file)); } - if (outputs[ix]) pipeline.pipe(outputs[ix]); - - acc[path.resolve(cwd, x)] = pipeline; + // pipe to output if one exists + if (outputs[index]) pipeline.pipe(outputs[index]); + + acc[path.resolve(cwd, file)] = pipeline; return acc; }, {}); // Force browser-pack to wrap the common bundle b._bpack.hasExports = true; + // for tests/debugging Object.keys(pipelines).forEach(function (id) { b.emit('factor.pipeline', id, pipelines[id]); }); + // create factor stream var s = createStream(files, opts); + + // connect each factor groups module stream to bundle pipeline s.on('stream', function (bundle) { + // each group's output stream bundle.pipe(pipelines[bundle.file]); }); + // add factor stream before pack b.pipeline.get('pack').unshift(s); if (needRecords) files = []; @@ -106,6 +118,7 @@ module.exports = function f (b, opts) { next(); })); + // capture module's relative path. used for recovering entry point pathnames b.pipeline.get('label').push(through.obj(function(row, enc, next) { opts.rmap[row.id] = path.resolve(cwd, row.file); next(null, row); @@ -118,10 +131,10 @@ module.exports = function f (b, opts) { function createStream (files, opts) { if (!opts) opts = {}; - + var fr = new Factor(files, opts); var parse, dup; - + if (opts.objectMode) { dup = combine(depsTopoSort(), reverse(), fr); } @@ -135,7 +148,7 @@ function createStream (files, opts) { ; parse.on('error', function (err) { dup.emit('error', err) }); } - + fr.on('error', function (err) { dup.emit('error', err) }); fr.on('stream', function (s) { if (opts.raw) dup.emit('stream', s) @@ -150,21 +163,21 @@ function Factor (files, opts) { var self = this; if (!(this instanceof Factor)) return new Factor(files, opts); Transform.call(this, { objectMode: true }); - + if (!opts) opts = {}; this.basedir = defined(opts.basedir, process.cwd()); - + this._streams = {}; this._groups = {}; this._buffered = {}; - + this._ensureCommon = {}; this._files = files.reduce(function (acc, file) { acc[path.resolve(self.basedir, file)] = true; return acc; }, {}); this._rmap = opts.rmap || {}; - + this._thresholdVal = typeof opts.threshold === "number" ? opts.threshold : 1 ; @@ -200,9 +213,9 @@ Factor.prototype._transform = function (row, enc, next) { self._streams[id].push(row); }); } - + next(); - + function addGroups (gid) { Object.keys(row.deps || {}).forEach(function (key) { var file = row.deps[key]; @@ -215,7 +228,7 @@ Factor.prototype._transform = function (row, enc, next) { Factor.prototype._flush = function () { var self = this; - + Object.keys(self._streams).forEach(function (key) { self._streams[key].push(null); }); From c5f0f4000b03debfad6af593102c0b3df8ec03a8 Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 6 Mar 2020 17:08:28 +0800 Subject: [PATCH 4/5] doc - update readme about dedupe option --- readme.markdown | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/readme.markdown b/readme.markdown index c8119ed..783c75e 100644 --- a/readme.markdown +++ b/readme.markdown @@ -37,7 +37,7 @@ module.exports = function (n) { return n * 50 } Now run factor-bundle as a plugin (new in browserify 3.28.0): ``` sh -browserify x.js y.js -p [ factor-bundle -o bundle/x.js -o bundle/y.js ] \ +browserify --no-dedupe x.js y.js -p [ factor-bundle -o bundle/x.js -o bundle/y.js ] \ -o bundle/common.js ``` @@ -54,7 +54,7 @@ $ module-deps x.js y.js | factor-bundle \ or factor out an existing bundle already compiled by browserify: ``` sh -$ browserify x.js y.js > bundle.js +$ browserify --no-dedupe x.js y.js > bundle.js $ browser-unpack < bundle.js | factor-bundle \ x.js -o bundle/x.js \ y.js -o bundle/y.js \ @@ -84,13 +84,18 @@ $ cat bundle/common.js bundle/y.js | node 333 ``` +## usage note + +You must disable browserify's dedupe feature in order to use this module. +It can be disabled by providing `{ dedupe: false }` in the js api, or by the `--no-dedup` cli argument. + ## command-line outpipe example We can pipe each output file through some other processes. Here we'll do minification with uglify compression with gzip: ``` sh -browserify files/*.js \ +browserify --no-dedupe files/*.js \ -p [ factor-bundle -o 'uglifyjs -cm | gzip > bundle/`basename $FILE`.gz' ] \ | uglifyjs -cm | gzip > bundle/common.js.gz ``` @@ -104,7 +109,7 @@ var browserify = require('browserify'); var fs = require('fs'); var files = [ './files/x.js', './files/y.js' ]; -var b = browserify(files); +var b = browserify(files, { dedupe: true }); b.plugin('factor-bundle', { outputs: [ 'bundle/x.js', 'bundle/y.js' ] }); b.bundle().pipe(fs.createWriteStream('bundle/common.js')); ``` @@ -116,7 +121,7 @@ var browserify = require('browserify'); var concat = require('concat-stream'); var files = [ './files/x.js', './files/y.js' ]; -var b = browserify(files); +var b = browserify(files, { dedupe: true }); b.plugin('factor-bundle', { outputs: [ write('x'), write('y') ] }); b.bundle().pipe(write('common')); @@ -141,7 +146,7 @@ where OPTIONS are: -o Output FILE or CMD that maps to a corresponding entry file at the same index. CMDs are executed with $FILE set to the corresponding input file. Optionally specify a function that returns a valid value for this argument. - + -e Entry file to use, overriding the entry files listed in the original bundle. @@ -159,10 +164,10 @@ the corresponding output file (-o). Write output to FILE or CMD. CMD is distinguished from FILE by the presence of one or more `>` or `|` characters. For example, use: - + factor-bundle browser/a.js browser/b.js \ -o bundle/a.js -o bundle/b.js - + to write to a FILE. And do something like: factor-bundle browser/*.js \ @@ -203,10 +208,10 @@ this information is gathered from browserify itself. The files held in common among `> opts.threshold` (default: 1) bundles will be output on the `fr` stream itself. The entry-specific bundles are diverted into each `'stream'` event's output. `opts.threshold` can be a number or a function -`opts.threshold(row, groups)` where `row` is a -[module-deps](https://github.com/substack/module-deps) object and `groups` is -an array of bundles which depend on the row. If the threshold function returns -`true`, that row and all its dependencies will go to the `common` bundle. If +`opts.threshold(row, groups)` where `row` is a +[module-deps](https://github.com/substack/module-deps) object and `groups` is +an array of bundles which depend on the row. If the threshold function returns +`true`, that row and all its dependencies will go to the `common` bundle. If false, the row (but not its dependencies) will go to each bundle in `groups`. For example: From 85a54a66623ae978e2e826d4e87f7881a3938cc9 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 12 Mar 2020 11:37:05 +0800 Subject: [PATCH 5/5] Update readme.markdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Renée Kooi --- readme.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.markdown b/readme.markdown index 783c75e..7ee04fe 100644 --- a/readme.markdown +++ b/readme.markdown @@ -109,7 +109,7 @@ var browserify = require('browserify'); var fs = require('fs'); var files = [ './files/x.js', './files/y.js' ]; -var b = browserify(files, { dedupe: true }); +var b = browserify(files, { dedupe: false }); b.plugin('factor-bundle', { outputs: [ 'bundle/x.js', 'bundle/y.js' ] }); b.bundle().pipe(fs.createWriteStream('bundle/common.js')); ```