From b1850cbab43ec4c903e7eb2ada9cdfefe0468940 Mon Sep 17 00:00:00 2001 From: Nathan Vander Wilt Date: Tue, 10 Jan 2017 12:50:51 -0800 Subject: [PATCH] overhaul to make is_outdated consistent with compile_file w.r.t. to settings handling (fixes #15) as well as working despite changed SubProcessCompiler.execute_command in django-pipeline --- README.rst | 4 +- pipeline_browserify/compiler.py | 140 +++++++++++++++++--------------- 2 files changed, 75 insertions(+), 69 deletions(-) diff --git a/README.rst b/README.rst index 06f1bbb..0a43617 100755 --- a/README.rst +++ b/README.rst @@ -19,11 +19,11 @@ And add it as a compiler to pipeline in your django `settings.py`:: To add source maps during development (or any other browserify args):: if DEBUG: - PIPELINE['BROWSERIFY_ARGUMENTS'] = '-d' + PIPELINE['BROWSERIFY_ARGUMENTS'] = ['-d'] To add variable assignments before the browserify command:: - PIPELINE['BROWSERIFY_VARS'] = 'NODE_ENV=production' + PIPELINE['BROWSERIFY_VARS'] = {'NODE_ENV':'production'} **Important:** give your entry-point file a `.browserify.js` extension:: diff --git a/pipeline_browserify/compiler.py b/pipeline_browserify/compiler.py index 32e055d..7e230d6 100755 --- a/pipeline_browserify/compiler.py +++ b/pipeline_browserify/compiler.py @@ -1,91 +1,97 @@ -import json -import os - -from os.path import dirname -from tempfile import NamedTemporaryFile - -from django.core.exceptions import SuspiciousFileOperation - -from pipeline.conf import settings as pipeline_settings from pipeline.compilers import SubProcessCompiler +from django.conf import settings +from django.core.exceptions import SuspiciousFileOperation +from pipeline.exceptions import CompilerError class BrowserifyCompiler(SubProcessCompiler): output_extension = 'browserified.js' - + def match_file(self, path): + if self.verbose: + print('matching file:', path) return path.endswith('.browserify.js') - + + # similar to old removed in https://github.com/jazzband/django-pipeline/commit/1f6b48ae74026a12f955f2f15f9f08823d744515 + def simple_execute_command(self, cmd, **kwargs): + import subprocess + pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) + stdout, stderr = pipe.communicate() + if self.verbose: + print stdout + print stderr + if pipe.returncode != 0: + raise CompilerError("Compiler returned non-zero exit status %i" % pipe.returncode, command=cmd, error_output=stderr) + return stdout + + def _get_cmd_parts(self): + pipeline_settings = getattr(settings, 'PIPELINE', {}) + tool = pipeline_settings.get('BROWSERIFY_BINARY', "/usr/bin/env browserify") + args = pipeline_settings.get('BROWSERIFY_ARGUMENTS', []) + if not isinstance(args, list): + args = args.split() + + env = pipeline_settings.get('BROWSERIFY_VARS', {}) + if not isinstance(env, dict): + env = dict(map(lambda s: s.split('='), env.split())) + if len(env): + # even if there's custom variables, we need to pass along the original environment + import os + _env = {} + _env.update(os.environ) + _env.update(env) + env = _env + else: + # drop any empty dict and let subprocess retain environment automatically + env = None + + return tool, args, env + def compile_file(self, infile, outfile, outdated=False, force=False): if not force and not outdated: - return # File doesn't need to be recompiled - command = ( - pipeline_settings.get('BROWSERIFY_VARS', ''), - pipeline_settings.get('BROWSERIFY_BINARY', '/usr/bin/env browserify'), - pipeline_settings.get('BROWSERIFY_ARGUMENTS', ''), - infile, - "-o", - outfile, - ) - - return self.execute_command(command, cwd=dirname(infile)) - + return + + tool, args, env = self._get_cmd_parts() + args.extend([infile, '--outfile', outfile]) + cmd = [tool] + args + + if self.verbose: + print "compile_file command:", cmd, env + self.simple_execute_command(cmd, env=env) + def is_outdated(self, infile, outfile): """Check if the input file is outdated. - + The difficulty with the default implementation is that any file that is `require`d from the entry-point file will not trigger a recompile if it is modified. This overloaded version of the method corrects this by generating a list of all required files that are also a part of the storage manifest and checking if they've been modified since the last compile. - + The command used to generate the list of dependencies is the same as the compile - command but includes the `--deps` option. - + command but uses the `--list` option instead of `--outfile`. + WARNING: It seems to me that just generating the dependencies may take just as long as actually compiling, which would mean we would be better off just forcing a compile every time. """ - - # Check for missing file or modified entry-point file. + + # Preliminary check for simply missing file or modified entry-point file. if super(BrowserifyCompiler, self).is_outdated(infile, outfile): return True - - # Check if we've already calculated dependencies. - deps = getattr(self, '_deps', None) - if not deps: - # Collect dependency information. - command = ( - pipeline_settings.get('BROWSERIFY_VARS', ''), - pipeline_settings.get('BROWSERIFY_BINARY', '/usr/bin/env browserify'), - pipeline_settings.get('BROWSERIFY_ARGUMENTS', ''), - "--deps", - self.storage.path(infile), - ) - - with NamedTemporaryFile(delete=False, dir=dirname(outfile)) as dep_json: - self.execute_command(command, stdout_captured=dep_json.name) - - # Process the output data. It's JSON, and the file's path is coded - # in the "file" field. We also want to save the content of each file - # so we can check if they're outdated, which is coded under "source". - deps = [] - with open(dep_json.name) as command_output: - for dep in json.loads(command_output.read()): - # Is this file managed by the storage? - try: - if self.storage.exists(dep['file']): - deps.append(dep['file']) - except SuspiciousFileOperation: - pass - # dep_json must be removed afterwards - os.remove(dep_json.name) - - # Cache the dependencies for the next possible run. - self._deps = deps - - # Test the dependencies to see if they're out of date. - for dep in deps: - if super(BrowserifyCompiler, self).is_outdated(dep, outfile): + + # Otherwise we need to see what dependencies there are now, and if they're modified. + tool, args, env = self._get_cmd_parts() + args.extend(['--list', infile]) + cmd = [tool] + args + if self.verbose: + print "is_outdated command:", cmd, env + dep_list = self.simple_execute_command(cmd, env=env) + if self.verbose: + print "dep_list is:", dep_list + for dep_file in dep_list.strip().split('\n'): + if super(BrowserifyCompiler, self).is_outdated(dep_file, outfile): + if self.verbose: + print "Found dep_file \"%s\" updated." % dep_file return True - + return False