From 02745e4e8728de285be6fb1bf25955f51592c93a Mon Sep 17 00:00:00 2001 From: Frankie Dintino Date: Sun, 16 Aug 2015 18:13:32 -0400 Subject: [PATCH] Add sourcemap support --- pipeline/compressors/__init__.py | 35 +++++++++++++----- pipeline/compressors/closure.py | 61 ++++++++++++++++++++++++++++++++ pipeline/conf.py | 2 ++ pipeline/packager.py | 23 +++++++++--- pipeline/storage.py | 20 ++++++----- 5 files changed, 121 insertions(+), 20 deletions(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index ef9974e4..35cdcb55 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -4,6 +4,7 @@ import os import posixpath import re +import warnings from itertools import takewhile @@ -55,6 +56,15 @@ def css_compressor(self): def compress_js(self, paths, templates=None, **kwargs): """Concatenate and compress JS files""" + compressor = self.js_compressor + + if settings.PIPELINE_OUTPUT_SOURCEMAPS: + if hasattr(compressor, 'compress_js_with_source_map'): + if templates: + warnings.warn("Source maps are not supported with javascript templates") + else: + return compressor(verbose=self.verbose).compress_js_with_source_map(paths) + js = self.concatenate(paths) if templates: js = js + self.compile_templates(templates) @@ -62,22 +72,30 @@ def compress_js(self, paths, templates=None, **kwargs): if not settings.PIPELINE_DISABLE_WRAPPER: js = "(function() {\n%s\n}).call(this);" % js - compressor = self.js_compressor if compressor: js = getattr(compressor(verbose=self.verbose), 'compress_js')(js) - return js + return js, None def compress_css(self, paths, output_filename, variant=None, **kwargs): """Concatenate and compress CSS files""" - css = self.concatenate_and_rewrite(paths, output_filename, variant) compressor = self.css_compressor + + if settings.PIPELINE_OUTPUT_SOURCEMAPS: + if hasattr(compressor, 'compress_css_with_source_map'): + if variant == "datauri": + warnings.warn("Source maps are not supported with datauri variant") + else: + return (compressor(verbose=self.verbose) + .compress_css_with_source_map(paths, output_filename)) + + css = self.concatenate_and_rewrite(paths, output_filename, variant) if compressor: css = getattr(compressor(verbose=self.verbose), 'compress_css')(css) if not variant: - return css + return css, None elif variant == "datauri": - return self.with_data_uri(css) + return self.with_data_uri(css), None else: raise CompressorError("\"%s\" is not a valid variant" % variant) @@ -233,10 +251,11 @@ def filter_js(self, js): class SubProcessCompressor(CompressorBase): - def execute_command(self, command, content): + def execute_command(self, command, content, shell=True): import subprocess - pipe = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, - stdin=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = subprocess.PIPE if content else None + pipe = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, + stdin=stdin, stderr=subprocess.PIPE) if content: content = smart_bytes(content) stdout, stderr = pipe.communicate(content) diff --git a/pipeline/compressors/closure.py b/pipeline/compressors/closure.py index 521c7a28..5419105f 100644 --- a/pipeline/compressors/closure.py +++ b/pipeline/compressors/closure.py @@ -1,10 +1,71 @@ from __future__ import unicode_literals +import os +import re +import shlex +import tempfile + +from django.contrib.staticfiles.storage import staticfiles_storage +from django.utils import six + from pipeline.conf import settings from pipeline.compressors import SubProcessCompressor +source_map_re = re.compile(( + "(?:" + "/\\*" + "(?:\\s*\r?\n(?://)?)?" + "(?:%(inner)s)" + "\\s*" + "\\*/" + "|" + "//(?:%(inner)s)" + ")" + "\\s*$") % {'inner': r"""[#@] sourceMappingURL=([^\s'"]*)"""}) + + class ClosureCompressor(SubProcessCompressor): + def compress_js(self, js): command = '%s %s' % (settings.PIPELINE_CLOSURE_BINARY, settings.PIPELINE_CLOSURE_ARGUMENTS) return self.execute_command(command, js) + + def compress_js_with_source_map(self, paths): + args = re.split(r'\s+', settings.PIPELINE_CLOSURE_BINARY) + if settings.PIPELINE_CLOSURE_ARGUMENTS: + if isinstance(settings.PIPELINE_CLOSURE_ARGUMENTS, six.string_types): + args += shlex.split(settings.PIPELINE_CLOSURE_ARGUMENTS) + else: + args += settings.PIPELINE_CLOSURE_ARGUMENTS + abs_paths = [] + for path in paths: + abs_path = staticfiles_storage.path(path) + args += [ + '--source_map_location_mapping', + "%s|%s" % (abs_path, staticfiles_storage.url(path))] + abs_paths.append(abs_path) + with open(abs_path) as f: + content = f.read() + matches = source_map_re.search(content) + if matches: + input_source_map = filter(None, matches.groups())[0] + input_source_map_file = os.path.join(os.path.dirname(abs_path), input_source_map) + args += [ + '--source_map_input', + "%s|%s" % (abs_path, input_source_map_file)] + + temp_file = tempfile.NamedTemporaryFile() + + args += ["--create_source_map", temp_file.name] + for path in abs_paths: + args += ["--js", path] + + js = self.execute_command(args, None, shell=False) + + with open(temp_file.name) as f: + source_map = f.read() + + temp_file.close() + + return js, source_map diff --git a/pipeline/conf.py b/pipeline/conf.py index 9d6c6e19..c621dab5 100644 --- a/pipeline/conf.py +++ b/pipeline/conf.py @@ -25,6 +25,8 @@ 'PIPELINE_DISABLE_WRAPPER': False, + 'PIPELINE_OUTPUT_SOURCEMAPS': False, + 'PIPELINE_CSSTIDY_BINARY': '/usr/bin/env csstidy', 'PIPELINE_CSSTIDY_ARGUMENTS': '--template=highest', diff --git a/pipeline/packager.py b/pipeline/packager.py index 38b038e1..ca7cbb32 100644 --- a/pipeline/packager.py +++ b/pipeline/packager.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import os.path from django.contrib.staticfiles.storage import staticfiles_storage from django.contrib.staticfiles.finders import find @@ -92,24 +93,38 @@ def individual_url(self, filename): def pack_stylesheets(self, package, **kwargs): return self.pack(package, self.compressor.compress_css, css_compressed, + compress_type='css', output_filename=package.output_filename, variant=package.variant, **kwargs) def compile(self, paths, force=False): return self.compiler.compile(paths, force=force) - def pack(self, package, compress, signal, **kwargs): + def pack(self, package, compress, signal, compress_type, **kwargs): output_filename = package.output_filename if self.verbose: print("Saving: %s" % output_filename) paths = self.compile(package.paths, force=True) - content = compress(paths, **kwargs) + content, source_map = compress(paths, **kwargs) + if source_map is not None: + source_map_output_filename = output_filename + '.map' + if self.verbose: + print("Saving: %s" % source_map_output_filename) + self.save_file(source_map_output_filename, source_map) + source_map_comment = "sourceMappingURL=%s" % ( + os.path.basename(staticfiles_storage.url(source_map_output_filename))) + if compress_type == 'js': + content += "\n//# %s" % source_map_comment + else: + content += "\n/*# %s */" % source_map_comment + yield source_map_output_filename self.save_file(output_filename, content) signal.send(sender=self, package=package, **kwargs) - return output_filename + yield output_filename def pack_javascripts(self, package, **kwargs): - return self.pack(package, self.compressor.compress_js, js_compressed, templates=package.templates, **kwargs) + return self.pack(package, self.compressor.compress_js, js_compressed, + compress_type='js', templates=package.templates, **kwargs) def pack_templates(self, package): return self.compressor.compile_templates(package.templates) diff --git a/pipeline/storage.py b/pipeline/storage.py index c75d0ebd..f84e99df 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -21,18 +21,22 @@ def post_process(self, paths, dry_run=False, **options): packager = Packager(storage=self) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) - output_file = package.output_filename if self.packing: - packager.pack_stylesheets(package) - paths[output_file] = (self, output_file) - yield output_file, output_file, True + output_files = packager.pack_stylesheets(package) + else: + output_files = [package.output_filename] + for output_file in output_files: + paths[output_file] = (self, output_file) + yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) - output_file = package.output_filename if self.packing: - packager.pack_javascripts(package) - paths[output_file] = (self, output_file) - yield output_file, output_file, True + output_files = packager.pack_javascripts(package) + else: + output_files = [package.output_filename] + for output_file in output_files: + paths[output_file] = (self, output_file) + yield output_file, output_file, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'):