Merge pull request #8863 from gitcoinco/experiment/webpack-bundles

Adds POC for bundle template tags

.babelrc

@@ -0,0 +1,16 @@
+{
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "edge": "17",
+          "firefox": "60",
+          "chrome": "67",
+          "safari": "11.1",
+          "ie": "11"
+        }
+      }
+    ]
+  ]
+}

.gitignore

@@ -13,6 +13,7 @@ app/output/*.pdf
 app/assets/other/wp.pdf
 app/assets/tmp/*
 app/assets/other/avatars/
+app/assets/**/bundle*/
 app/gcoin/
 app/media/
 .idea/

app/app/urls.py

@@ -486,6 +486,7 @@
     re_path(r'^modal/extend_issue_deadline/?', dashboard.views.extend_issue_deadline, name='extend_issue_deadline'),
 
     # brochureware views
+    re_path(r'^bundle_experiment/?', retail.views.bundle_experiment, name='bundle_experiment'),
     re_path(r'^homeold/?$', retail.views.index_old, name='homeold'),
     re_path(r'^home/?$', retail.views.index, name='home'),
     re_path(r'^landing/?$', retail.views.index, name='landing'),

app/assets/v2/js/bundle_experiment.js

@@ -0,0 +1,5 @@
+// if you want to assign to window in a webpacked bundle - use this. or window.
+this.test = 1;
+
+// setting in the local scope will not assign to window
+const test = 2;

app/dashboard/management/commands/bundle.py

@@ -0,0 +1,91 @@
+import os
+import re
+import shutil
+
+from django.core.management.base import BaseCommand
+from django.template.loaders.app_directories import get_app_template_dirs
+from django.conf import settings
+
+from dashboard.templatetags.bundle import render
+
+
+def rmdir(loc):
+    # drop both the bundled and the bundles before recreating
+    if os.path.exists(loc) and os.path.isdir(loc):
+        print('- Deleting assets from: %s' % loc)
+        shutil.rmtree(loc)
+
+
+def rmdirs(loc, kind):
+    # base path of the assets
+    base = ('%s/%s/v2/' % (settings.BASE_DIR, loc)).replace('/', os.sep)
+    # delete both sets of assets
+    rmdir('%sbundles/%s' % (base, kind))
+    rmdir('%sbundled/%s' % (base, kind))
+
+
+class Command(BaseCommand):
+
+    help = 'generates .js/.scss files from bundle template tags'
+
+    def handle(self, *args, **options):
+        template_dir_list = []
+        for template_dir in get_app_template_dirs('templates'):
+            if settings.BASE_DIR in template_dir:
+                template_dir_list.append(template_dir)
+
+        template_list = []
+        for template_dir in (template_dir_list + settings.TEMPLATES[0]['DIRS']):
+            for base_dir, dirnames, filenames in os.walk(template_dir):
+                for filename in filenames:
+                    if ".html" in filename:
+                        template_list.append(os.path.join(base_dir, filename))
+
+        # using regex to grab the bundle tags content from html
+        block_pattern = re.compile(r'({%\sbundle(.|\n)*?(?<={%\sendbundle\s%}))')
+        open_pattern = re.compile(r'({%\s+bundle\s+(js|css|merge_js|merge_css)\s+?(file)?\s+?([^\s]*)?\s+?%})')
+        close_pattern = re.compile(r'({%\sendbundle\s%})')
+        static_open_pattern = re.compile(r'({%\sstatic\s["|\'])')
+        static_close_pattern = re.compile(r'(\s?%}(\"|\')?\s?\/?>)')
+
+        # remove the previously bundled files
+        rmdirs('assets', 'js')
+        rmdirs('assets', 'scss')
+        rmdirs('static', 'js')
+        rmdirs('static', 'scss')
+
+        print('\nStart generating bundle files\n')
+
+        # store unique entries for count
+        rendered = dict()
+
+        for template in template_list:
+            try:
+                f = open(('%s' % template).replace('/', os.sep), 'r', encoding='utf8')
+
+                t = f.read()
+                if re.search(block_pattern, t) is not None:
+                    for m in re.finditer(block_pattern, t):
+                        block = m.group(0)
+                        details = re.search(open_pattern, block)
+
+                        # kind and name from the tag
+                        kind = 'scss' if details.group(2) == 'css' else details.group(2)
+                        name = details.group(4)
+
+                        # remove open/close from the block
+                        block = re.sub(open_pattern, '', block)
+                        block = re.sub(close_pattern, '', block)
+
+                        # clean static helper if we havent ran this through parse
+                        block = re.sub(static_open_pattern, '', block)
+                        block = re.sub(static_close_pattern, '>', block)
+
+                        # render the template (producing a bundle file)
+                        rendered[render(block, kind, 'file', name, True)] = True
+
+            except Exception as e:
+                # print('-- X - failed to parse %s: %s' % (template, e))
+                pass
+
+        print('\nGenerated %s bundle files%s' % (len(rendered), ' - remember to run `yarn run build` then `./manage.py collectstatic --i other --no-input`\n' if settings.ENV in ['prod'] else ''))

app/dashboard/templatetags/bundle.py

@@ -0,0 +1,216 @@
+import hashlib
+import os
+import re
+
+from django import template
+from bs4 import BeautifulSoup
+from django.conf import settings
+from django.templatetags.static import static
+
+register = template.Library()
+
+"""
+    Creates bundles from linked and inline Javascript or SCSS into a single file - compressed by py or webpack.
+
+    Syntax:
+
+        {% bundle [js|css|merge_js|merge_css] file [block_name] %}
+        <script src="..."></script>
+        <script>
+            ...
+        </script>
+        --or--
+        <link href="..."/>
+        <style>
+            ...
+        </style>
+        {% endbundle %}
+
+    (dev) to compress:
+
+        yarn run webpack
+
+    (prod) to compress:
+
+        ./manage.py bundle && yarn run build
+"""
+
+def css_elems(soup):
+    return soup.find_all({'link': True, 'style': True})
+
+
+def js_elems(soup):
+    return soup.find_all('script')
+
+
+def get_tag(ext, src):
+    return '<script src="%s"></script>' % src if ext == "js" else '<link rel="stylesheet" href="%s"/>' % src
+
+
+def check_merge_changes(elems, attr, outputFile):
+    # fn checks if content is changed since last op
+    changed = False
+    # if the block exists as a file - get timestamp so that we can perform cheap comp
+    blockTs = 0
+    try:
+        blockTs = os.path.getmtime(outputFile)
+    except:
+        pass
+    # if any file has changed then we need to regenerate
+    for el in elems:
+        if el.get(attr):
+            # removes static url and erroneous quotes from path
+            asset = '%s/assets/%s' % (settings.BASE_DIR, el[attr])
+            # bundle straight to the bundled directory skipping 'bundles'
+            ts = -1
+            try:
+                ts = os.path.getmtime(asset.replace('/', os.sep))
+            except:
+                pass
+            # if any ts is changed then we regenerate
+            if ts < blockTs:
+                changed = True
+                break
+        else:
+            changed = True
+            break
+    return changed
+
+
+def get_content(elems, attr, kind, merge):
+    # concat all input in the block
+    content = ''
+    # construct the content by converting tags to import statements
+    for el in elems:
+        # is inclusion or inline tag?
+        if el.get(attr):
+            # removes static url and erroneous quotes from path
+            asset = '%s/assets/%s' % (settings.BASE_DIR, el[attr])
+            # if we're merging the content then run through minify and skip saving of intermediary
+            if merge:
+                # bundle straight to the bundled directory skipping 'bundles'
+                f = open(asset.replace('/', os.sep), 'r', encoding='utf8')
+                f.seek(0)
+                c = f.read()
+                # for production we should minifiy the assets
+                if settings.ENV in ['prod'] and kind == 'merge_js':
+                    import jsmin
+                    c = jsmin.jsmin(c, quote_chars="'\"`")
+                elif settings.ENV in ['prod'] and kind == 'merge_css':
+                    import cssmin
+                    c = cssmin.cssmin(c)
+                # place the content with a new line sep
+                content += c + '\n'
+            else:
+                # import the scripts from the assets dir
+                if kind == 'js':
+                    content += 'import \'%s\';\n' % asset
+                else:
+                    content += ' @import \'%s\';\n' % asset
+        else:
+            # content held within tags after cleaning up all whitespace on each newline (consistent content regardless of indentation)
+            content += '\n'.join(str(x).strip() for x in (''.join([str(x) for x in el.contents]).splitlines()))
+
+    return content
+
+
+def render(block, kind, mode, name='asset', forced=False):
+    # check if we're merging content
+    merge = True if 'merge' in kind else False
+    ext = kind.replace('merge_', '')
+
+    # output locations
+    bundled = 'bundled'
+    bundles = 'bundles' if not merge else bundled
+
+    # clean up the block -- essentially we want to drop anything that gets added by staticfinder (could we improve this by not using static in the templates?)
+    cleanBlock = block.replace(settings.STATIC_URL, '')
+
+    # drop any quotes that appear inside the tags - keep the input consistent bs4 will overlook missing quotes
+    findTags = re.compile(r'(<(script|link|style)(.*?)>)')
+    if re.search(findTags, cleanBlock) is not None:
+        for t in re.finditer(findTags, cleanBlock):
+            tag = t.group(0)
+            cleanBlock = cleanBlock.replace(tag, tag.replace('"', '').replace('\'', ''))
+
+    # in production staticfinder will attach an additional hash to the resource which doesnt exist on the local disk
+    if settings.ENV in ['prod'] and forced != True:
+        cleanBlock = re.sub(re.compile(r'(\..{12}\.(css|scss|js))'), r'.\2', cleanBlock)
+
+    # parse block with bs4
+    soup = BeautifulSoup(cleanBlock, "lxml")
+    # get a hash of the block we're working on (after parsing -- ensures we're always working against the same input)
+    blockHash = hashlib.sha256(str(soup).encode('utf')).hexdigest()
+
+    # In production we don't need to generate new content unless we're running this via the bundle command
+    if settings.ENV not in ['prod'] or forced == True:
+        # concat all input in the block
+        content = ''
+        # pull the appropriate tags from the block
+        elems = js_elems(soup) if ext == 'js' else css_elems(soup)
+        attr = 'src' if ext == 'js' else 'href'
+        # output disk location (hard-coding assets/v2 -- this could be a setting?)
+        outputFile = ('%s/assets/v2/%s/%s/%s.%s.%s' % (settings.BASE_DIR, bundles, ext, name, blockHash[0:6], ext)).replace('/', os.sep)
+        changed = True if merge == False or forced == True else check_merge_changes(elems, attr, outputFile)
+
+        # !merge kind is always tested - merge is only recreated if at least one of the inclusions has been altered
+        if changed:
+            # retrieve the content for the block/output file
+            content = get_content(elems, attr, kind, merge)
+            # ensure the bundles directory exists
+            os.makedirs(os.path.dirname(outputFile), exist_ok=True)
+            # open the file in read/write mode
+            f = open(outputFile, 'a+', encoding='utf8')
+            f.seek(0)
+
+            # if content (of the block) has changed - write new content
+            if merge or f.read() != content:
+                # clear the file before writing new content
+                f.truncate(0)
+                f.write(content)
+                f.close()
+                # print so that we have concise output in the bundle command
+                print('- Generated: %s' % outputFile)
+
+    # in production and not forced we will just return the static bundle
+    return get_tag(ext, static('v2/%s/%s/%s.%s.%s' % (bundled, ext, name, blockHash[0:6], 'css' if ext == 'scss' else ext)))
+
+
+class CompressorNode(template.Node):
+
+
+    def __init__(self, nodelist, kind=None, mode='file', name=None):
+        self.nodelist = nodelist
+        self.kind = kind
+        self.mode = mode
+        self.name = name
+
+
+    def render(self, context, forced=False):
+        return render(self.nodelist.render(context), self.kind, self.mode, self.name)
+
+
+@register.tag
+def bundle(parser, token):
+    # pull content and split args from bundle block
+    nodelist = parser.parse(('endbundle',))
+    parser.delete_first_token()
+
+    args = token.split_contents()
+
+    if not len(args) in (2, 3, 4):
+        raise template.TemplateSyntaxError(
+            "%r tag requires either one or three arguments." % args[0])
+
+    kind = 'scss' if args[1] == 'css' else args[1]
+
+    if len(args) >= 3:
+        mode = args[2]
+    else:
+        mode = 'file'
+    if len(args) == 4:
+        name = args[3]
+    else:
+        name = None
+
+    return CompressorNode(nodelist, kind, mode, name)