diff --git a/pipeline/compilers/coffee.py b/pipeline/compilers/coffee.py index 8260ede6..77d78399 100644 --- a/pipeline/compilers/coffee.py +++ b/pipeline/compilers/coffee.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import os + from pipeline.conf import settings from pipeline.compilers import SubProcessCompiler @@ -13,10 +15,16 @@ def match_file(self, path): def compile_file(self, infile, outfile, outdated=False, force=False): if not outdated and not force: return # File doesn't need to be recompiled + + args = list(settings.COFFEE_SCRIPT_ARGUMENTS) + if settings.OUTPUT_SOURCEMAPS and not(set(args) & set(['-m', '--map'])): + args.append('--map') + command = ( settings.COFFEE_SCRIPT_BINARY, - "-cp", - settings.COFFEE_SCRIPT_ARGUMENTS, + "-c", + "-o", os.path.dirname(outfile), + args, infile, ) - return self.execute_command(command, stdout_captured=outfile) + return self.execute_command(command, cwd=os.path.dirname(outfile)) diff --git a/pipeline/compilers/es6.py b/pipeline/compilers/es6.py index a5cedd77..e14c4dde 100644 --- a/pipeline/compilers/es6.py +++ b/pipeline/compilers/es6.py @@ -13,9 +13,16 @@ def match_file(self, path): def compile_file(self, infile, outfile, outdated=False, force=False): if not outdated and not force: return # File doesn't need to be recompiled + + args = list(settings.BABEL_ARGUMENTS) + + sourcemap_flags = set(['-s', '--source-maps']) + if settings.OUTPUT_SOURCEMAPS and not(set(args) & sourcemap_flags): + args += ['--source-maps', 'true'] + command = ( settings.BABEL_BINARY, - settings.BABEL_ARGUMENTS, + args, infile, "-o", outfile diff --git a/pipeline/compilers/less.py b/pipeline/compilers/less.py index fa81747b..4e09950a 100644 --- a/pipeline/compilers/less.py +++ b/pipeline/compilers/less.py @@ -14,9 +14,15 @@ def match_file(self, filename): def compile_file(self, infile, outfile, outdated=False, force=False): # Pipe to file rather than provide outfile arg due to a bug in lessc + args = list(settings.LESS_ARGUMENTS) + + if settings.OUTPUT_SOURCEMAPS and '--source-map' not in args: + args += ['--source-map'] + command = ( settings.LESS_BINARY, - settings.LESS_ARGUMENTS, + args, infile, + outfile, ) - return self.execute_command(command, cwd=dirname(infile), stdout_captured=outfile) + return self.execute_command(command, cwd=dirname(infile)) diff --git a/pipeline/compilers/livescript.py b/pipeline/compilers/livescript.py index f72f896b..7f5ab4fb 100644 --- a/pipeline/compilers/livescript.py +++ b/pipeline/compilers/livescript.py @@ -1,5 +1,8 @@ from __future__ import unicode_literals +from os.path import dirname, basename +import json + from pipeline.conf import settings from pipeline.compilers import SubProcessCompiler @@ -13,10 +16,25 @@ def match_file(self, path): def compile_file(self, infile, outfile, outdated=False, force=False): if not outdated and not force: return # File doesn't need to be recompiled + + args = list(settings.LIVE_SCRIPT_ARGUMENTS) + if settings.OUTPUT_SOURCEMAPS and not(set(args) & set(['-m', '--map'])): + args += ['--map', 'linked'] + command = ( settings.LIVE_SCRIPT_BINARY, - "-cp", - settings.LIVE_SCRIPT_ARGUMENTS, + "-c", + "-o", dirname(outfile), + args, infile, ) - return self.execute_command(command, stdout_captured=outfile) + ret = self.execute_command(command, cwd=dirname(outfile)) + + if settings.OUTPUT_SOURCEMAPS: + with open("%s.map" % outfile) as f: + source_map = json.loads(f.read()) + source_map['sources'] = map(basename, source_map['sources']) + with open("%s.map" % outfile, mode='w') as f: + f.write(json.dumps(source_map)) + + return ret diff --git a/pipeline/compilers/sass.py b/pipeline/compilers/sass.py index d05c87ec..0de3e73f 100644 --- a/pipeline/compilers/sass.py +++ b/pipeline/compilers/sass.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import re from os.path import dirname from pipeline.conf import settings @@ -9,14 +10,36 @@ class SASSCompiler(SubProcessCompiler): output_extension = 'css' + _sass_types = {} + + @property + def sass_type(self): + bin = " ".join(settings.SASS_BINARY) + if bin not in self._sass_types: + if re.search(r'node\-sass', bin): + self._sass_types[bin] = 'node' + elif re.search(r'sassc', bin): + self._sass_types[bin] = 'libsass' + else: + self._sass_types[bin] = 'ruby' + return self._sass_types[bin] + def match_file(self, filename): return filename.endswith(('.scss', '.sass')) def compile_file(self, infile, outfile, outdated=False, force=False): - command = ( - settings.SASS_BINARY, - settings.SASS_ARGUMENTS, - infile, - outfile - ) + args = list(settings.SASS_ARGUMENTS) + + if settings.OUTPUT_SOURCEMAPS: + if self.sass_type == 'node': + if '--source-map' not in args: + args += ['--source-map', 'true'] + elif self.sass_type == 'libsass': + if not(set(args) & set(['-m', 'g', '--sourcemap'])): + args += ['--sourcemap'] + else: + if not any([re.search(r'^\-\-sourcemap', a) for a in args]): + args += ['--sourcemap=auto'] + + command = (settings.SASS_BINARY, args, infile, outfile) return self.execute_command(command, cwd=dirname(infile)) diff --git a/pipeline/compilers/stylus.py b/pipeline/compilers/stylus.py index 320efd9e..7f762b5a 100644 --- a/pipeline/compilers/stylus.py +++ b/pipeline/compilers/stylus.py @@ -13,9 +13,15 @@ def match_file(self, filename): return filename.endswith('.styl') def compile_file(self, infile, outfile, outdated=False, force=False): + args = list(settings.STYLUS_ARGUMENTS) + + sourcemap_flags = set(['-s', '--sourcemap']) + if settings.OUTPUT_SOURCEMAPS and not(set(args) & sourcemap_flags): + args += ['--sourcemap'] + command = ( settings.STYLUS_BINARY, - settings.STYLUS_ARGUMENTS, + args, infile ) return self.execute_command(command, cwd=dirname(infile)) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 6043a242..c9e2b9fc 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -5,6 +5,7 @@ import posixpath import re import subprocess +import warnings from itertools import takewhile @@ -57,6 +58,15 @@ def css_compressor(self): def compress_js(self, paths, templates=None, **kwargs): """Concatenate and compress JS files""" + compressor = self.js_compressor + + if settings.OUTPUT_SOURCEMAPS: + if hasattr(compressor, 'compress_js_with_source_map'): + if templates: + warnings.warn("Source maps are not supported with javascript templates") + else: + return compressor(verbose=self.verbose).compress_js_with_source_map(paths) + js = self.concatenate(paths) if templates: js = js + self.compile_templates(templates) @@ -64,22 +74,30 @@ def compress_js(self, paths, templates=None, **kwargs): if not settings.DISABLE_WRAPPER: js = "(function() {\n%s\n}).call(this);" % js - compressor = self.js_compressor if compressor: js = getattr(compressor(verbose=self.verbose), 'compress_js')(js) - return js + return js, None def compress_css(self, paths, output_filename, variant=None, **kwargs): """Concatenate and compress CSS files""" - css = self.concatenate_and_rewrite(paths, output_filename, variant) compressor = self.css_compressor + + if settings.OUTPUT_SOURCEMAPS: + if hasattr(compressor, 'compress_css_with_source_map'): + if variant == "datauri": + warnings.warn("Source maps are not supported with datauri variant") + else: + return (compressor(verbose=self.verbose) + .compress_css_with_source_map(paths, output_filename)) + + css = self.concatenate_and_rewrite(paths, output_filename, variant) if compressor: css = getattr(compressor(verbose=self.verbose), 'compress_css')(css) if not variant: - return css + return css, None elif variant == "datauri": - return self.with_data_uri(css) + return self.with_data_uri(css), None else: raise CompressorError("\"%s\" is not a valid variant" % variant) @@ -235,16 +253,17 @@ def filter_js(self, js): class SubProcessCompressor(CompressorBase): - def execute_command(self, command, content): + def execute_command(self, command, content=None, **kwargs): argument_list = [] for flattening_arg in command: if isinstance(flattening_arg, string_types): argument_list.append(flattening_arg) else: argument_list.extend(flattening_arg) + stdin = subprocess.PIPE if content else None pipe = subprocess.Popen(argument_list, stdout=subprocess.PIPE, - stdin=subprocess.PIPE, stderr=subprocess.PIPE) + stdin=stdin, stderr=subprocess.PIPE, **kwargs) if content: content = smart_bytes(content) stdout, stderr = pipe.communicate(content) diff --git a/pipeline/compressors/cleancss.py b/pipeline/compressors/cleancss.py index fadb464a..45f6ab0e 100644 --- a/pipeline/compressors/cleancss.py +++ b/pipeline/compressors/cleancss.py @@ -1,7 +1,14 @@ from __future__ import unicode_literals +import codecs +import json +import os + +from django.contrib.staticfiles.storage import staticfiles_storage + from pipeline.conf import settings from pipeline.compressors import SubProcessCompressor +from pipeline.utils import source_map_re, relurl class CleanCSSCompressor(SubProcessCompressor): @@ -9,3 +16,51 @@ class CleanCSSCompressor(SubProcessCompressor): def compress_css(self, css): args = [settings.CLEANCSS_BINARY, settings.CLEANCSS_ARGUMENTS] return self.execute_command(args, css) + + def compress_css_with_source_map(self, paths, output_filename): + output_path = staticfiles_storage.path(output_filename) + output_dir = os.path.dirname(output_path) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + args = [settings.CLEANCSS_BINARY] + args += ['--source-map'] + if settings.CLEANCSS_ARGUMENTS: + args += [settings.CLEANCSS_ARGUMENTS] + else: + # At present, without these arguments, cleancss does not + # generate accurate source maps + args += [ + '--skip-advanced', '--skip-media-merging', + '--skip-restructuring', '--skip-shorthand-compacting', + '--keep-line-breaks'] + args += ['--output', output_path] + args += [staticfiles_storage.path(p) for p in paths] + + self.execute_command(args, cwd=output_dir) + + source_map_file = "%s.map" % output_path + + with codecs.open(output_path, encoding='utf-8') as f: + css = f.read() + with codecs.open(source_map_file, encoding='utf-8') as f: + source_map = f.read() + + # Strip out existing source map comment (it will be re-added with packaging) + css = source_map_re.sub('', css) + + output_url = "%s/%s" % ( + staticfiles_storage.url(os.path.dirname(output_filename)), + os.path.basename(output_path)) + + # Grab urls from staticfiles storage (in case filenames are hashed) + source_map_data = json.loads(source_map) + for i, source in enumerate(source_map_data['sources']): + source_abs_path = os.path.join(output_dir, source) + source_rel_path = os.path.relpath( + source_abs_path, staticfiles_storage.base_location) + source_url = staticfiles_storage.url(source_rel_path) + source_map_data['sources'][i] = relurl(source_url, output_url) + source_map = json.dumps(source_map_data) + + return css, source_map diff --git a/pipeline/compressors/closure.py b/pipeline/compressors/closure.py index 1ee22392..bfa0d411 100644 --- a/pipeline/compressors/closure.py +++ b/pipeline/compressors/closure.py @@ -1,10 +1,56 @@ from __future__ import unicode_literals +import os +import tempfile + +from django.contrib.staticfiles.storage import staticfiles_storage + from pipeline.conf import settings from pipeline.compressors import SubProcessCompressor +from pipeline.utils import source_map_re class ClosureCompressor(SubProcessCompressor): + def compress_js(self, js): command = (settings.CLOSURE_BINARY, settings.CLOSURE_ARGUMENTS) return self.execute_command(command, js) + + def compress_js_with_source_map(self, paths): + args = [settings.CLOSURE_BINARY, settings.CLOSURE_ARGUMENTS] + + location_maps = set([]) + + abs_paths = [] + for path in paths: + abs_path = staticfiles_storage.path(path) + location_maps.add("%s|%s" % ( + os.path.dirname(abs_path), + staticfiles_storage.url(os.path.dirname(path)))) + abs_paths.append(abs_path) + with open(abs_path) as f: + content = f.read() + matches = source_map_re.search(content) + if matches: + input_source_map = filter(None, matches.groups())[0] + input_source_map_file = os.path.join(os.path.dirname(abs_path), input_source_map) + args += [ + '--source_map_input', + "%s|%s" % (abs_path, input_source_map_file)] + for location_map in location_maps: + args += ['--source_map_location_mapping', location_map] + + temp_file = tempfile.NamedTemporaryFile() + + args += ["--create_source_map", temp_file.name] + for path in abs_paths: + args += ["--js", path] + + js = self.execute_command(args, None) + + with open(temp_file.name) as f: + source_map = f.read() + + temp_file.close() + + return js, source_map diff --git a/pipeline/compressors/uglifyjs.py b/pipeline/compressors/uglifyjs.py index 78733844..00d646b7 100644 --- a/pipeline/compressors/uglifyjs.py +++ b/pipeline/compressors/uglifyjs.py @@ -1,12 +1,45 @@ from __future__ import unicode_literals +import codecs +import tempfile + +from django.contrib.staticfiles.storage import staticfiles_storage + from pipeline.conf import settings from pipeline.compressors import SubProcessCompressor +from pipeline.utils import source_map_re, path_depth class UglifyJSCompressor(SubProcessCompressor): + def compress_js(self, js): - command = (settings.UGLIFYJS_BINARY, settings.UGLIFYJS_ARGUMENTS) + command = [settings.UGLIFYJS_BINARY, settings.UGLIFYJS_ARGUMENTS] if self.verbose: - command += ' --verbose' + command.append(' --verbose') return self.execute_command(command, js) + + def compress_js_with_source_map(self, paths): + source_map_file = tempfile.NamedTemporaryFile() + + args = [settings.UGLIFYJS_BINARY] + args += [staticfiles_storage.path(p) for p in paths] + args += ["--source-map", source_map_file.name] + args += ["--source-map-root", staticfiles_storage.base_url] + args += ["--prefix", "%s" % path_depth(staticfiles_storage.base_location)] + + args += settings.UGLIFYJS_ARGUMENTS + + if self.verbose: + args.append('--verbose') + + js = self.execute_command(args) + + with codecs.open(source_map_file.name, encoding='utf-8') as f: + source_map = f.read() + + source_map_file.close() + + # Strip out existing source map comment (it will be re-added with packaging) + js = source_map_re.sub('', js) + + return js, source_map diff --git a/pipeline/conf.py b/pipeline/conf.py index 235900df..42bdde61 100644 --- a/pipeline/conf.py +++ b/pipeline/conf.py @@ -37,6 +37,8 @@ 'DISABLE_WRAPPER': False, + 'OUTPUT_SOURCEMAPS': False, + 'CSSTIDY_BINARY': '/usr/bin/env csstidy', 'CSSTIDY_ARGUMENTS': '--template=highest', diff --git a/pipeline/packager.py b/pipeline/packager.py index eed7e6a1..c838672b 100644 --- a/pipeline/packager.py +++ b/pipeline/packager.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import os.path from django.contrib.staticfiles.storage import staticfiles_storage from django.contrib.staticfiles.finders import find @@ -92,24 +93,38 @@ def individual_url(self, filename): def pack_stylesheets(self, package, **kwargs): return self.pack(package, self.compressor.compress_css, css_compressed, + compress_type='css', output_filename=package.output_filename, variant=package.variant, **kwargs) def compile(self, paths, force=False): return self.compiler.compile(paths, force=force) - def pack(self, package, compress, signal, **kwargs): + def pack(self, package, compress, signal, compress_type, **kwargs): output_filename = package.output_filename if self.verbose: print("Saving: %s" % output_filename) paths = self.compile(package.paths, force=True) - content = compress(paths, **kwargs) + content, source_map = compress(paths, **kwargs) + if source_map is not None: + source_map_output_filename = output_filename + '.map' + if self.verbose: + print("Saving: %s" % source_map_output_filename) + self.save_file(source_map_output_filename, source_map) + source_map_comment = "sourceMappingURL=%s" % ( + os.path.basename(staticfiles_storage.url(source_map_output_filename))) + if compress_type == 'js': + content += "\n//# %s" % source_map_comment + else: + content += "\n/*# %s */" % source_map_comment + yield source_map_output_filename self.save_file(output_filename, content) signal.send(sender=self, package=package, **kwargs) - return output_filename + yield output_filename def pack_javascripts(self, package, **kwargs): - return self.pack(package, self.compressor.compress_js, js_compressed, templates=package.templates, **kwargs) + return self.pack(package, self.compressor.compress_js, js_compressed, + compress_type='js', templates=package.templates, **kwargs) def pack_templates(self, package): return self.compressor.compile_templates(package.templates) diff --git a/pipeline/storage.py b/pipeline/storage.py index c75d0ebd..f84e99df 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -21,18 +21,22 @@ def post_process(self, paths, dry_run=False, **options): packager = Packager(storage=self) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) - output_file = package.output_filename if self.packing: - packager.pack_stylesheets(package) - paths[output_file] = (self, output_file) - yield output_file, output_file, True + output_files = packager.pack_stylesheets(package) + else: + output_files = [package.output_filename] + for output_file in output_files: + paths[output_file] = (self, output_file) + yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) - output_file = package.output_filename if self.packing: - packager.pack_javascripts(package) - paths[output_file] = (self, output_file) - yield output_file, output_file, True + output_files = packager.pack_javascripts(package) + else: + output_files = [package.output_filename] + for output_file in output_files: + paths[output_file] = (self, output_file) + yield output_file, output_file, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): diff --git a/pipeline/utils.py b/pipeline/utils.py index 729667f6..2c211132 100644 --- a/pipeline/utils.py +++ b/pipeline/utils.py @@ -10,18 +10,28 @@ import mimetypes import posixpath import os +import re import sys -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - from django.utils.encoding import smart_text +from django.utils.six.moves.urllib.parse import urlparse, quote from pipeline.conf import settings +source_map_re = re.compile(( + "(?:" + "/\\*" + "(?:\\s*\r?\n(?://)?)?" + "(?:%(inner)s)" + "\\s*" + "\\*/" + "|" + "//(?:%(inner)s)" + ")" + "\\s*$") % {'inner': r"""[#@] sourceMappingURL=([^\s'"]*)"""}) + + def to_class(class_str): if not class_str: return None @@ -64,6 +74,16 @@ def relpath(path, start=posixpath.curdir): return posixpath.join(*rel_list) +def relurl(path, start): + base = urlparse(start) + target = urlparse(path) + if base.netloc != target.netloc: + raise ValueError('target and base netlocs do not match') + base_dir = '.' + posixpath.dirname(base.path) + target = '.' + target.path + return posixpath.relpath(target, start=base_dir) + + def set_std_streams_blocking(): """ Set stdout and stderr to be blocking. @@ -78,3 +98,17 @@ def set_std_streams_blocking(): fileno = f.fileno() flags = fcntl.fcntl(fileno, fcntl.F_GETFL) fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) + + +def path_depth(path): + """Cross-platform compatible path depth count""" + import os + if hasattr(os.path, 'splitunc'): + _, path = os.path.splitunc(path) + path = os.path.normpath(path) + parent = os.path.dirname(path) + count = 0 + while path != parent: + path, parent = parent, os.path.dirname(parent) + count += 1 + return count diff --git a/tests/tests/test_compressor.py b/tests/tests/test_compressor.py index b56a54cc..737fffc9 100644 --- a/tests/tests/test_compressor.py +++ b/tests/tests/test_compressor.py @@ -212,10 +212,10 @@ def _test_compressor(self, compressor_cls, compress_type, expected_file): } with pipeline_settings(**override_settings): if compress_type == 'js': - result = self.compressor.compress_js( + result, source_map = self.compressor.compress_js( [_('pipeline/js/first.js'), _('pipeline/js/second.js')]) else: - result = self.compressor.compress_css( + result, source_map = self.compressor.compress_css( [_('pipeline/css/first.css'), _('pipeline/css/second.css')], os.path.join('pipeline', 'css', os.path.basename(expected_file))) with self.compressor.storage.open(expected_file) as f: