feat(react-tokens): unify react-token generation (#4058)

* feat(react-tokens): unify token generation strategies

* continue to use build script

* fix build

* node 10 doesn't have array.prototype.flat

* syntax

* bump theme

.eslintignore

@@ -12,8 +12,9 @@ coverage
 **/Generated
 **/build
 css
-packages/react-icons/src/icons
+packages/react-docs/.cache
 packages/react-docs/static
+packages/react-docs/public
 
 # package managers
 yarn-error.log

packages/react-docs/package.json

@@ -30,7 +30,7 @@
     "@patternfly/react-virtualized-extension": "^4.0.22",
     "gatsby": "2.20.2",
     "gatsby-cli": "2.11.0",
-    "gatsby-theme-patternfly-org": "^1.4.9",
+    "gatsby-theme-patternfly-org": "^1.4.16",
     "gatsby-transformer-react-docgen-typescript": "^0.2.5",
     "null-loader": "^3.0.0",
     "react": "^16.8.0",

packages/react-tokens/README.md

@@ -38,4 +38,4 @@ global_BackgroundColor_100.value === '#fff'; // true
 global_BackgroundColor_100.var === 'var(--pf-global--BackgroundColor--100)'; // true
 ```
 
-[token-page]: https://patternfly-react.surge.sh/patternfly-4/tokens/Global%20CSS%20variables/
+[token-page]: https://patternfly-react.surge.sh/documentation/overview/global-css-variables

packages/react-tokens/package.json

@@ -25,9 +25,8 @@
   },
   "homepage": "https://github.com/patternfly/patternfly-react#readme",
   "scripts": {
-    "build": "node src/generateTokens.js && node src/variablesByFile.js && yarn build:babel:umd && yarn build:babel:umd:variables",
+    "build": "node scripts/writeTokens.js && yarn build:babel:umd",
     "build:babel:umd": "babel dist/esm --out-dir dist/umd --plugins transform-es2015-modules-umd -q",
-    "build:babel:umd:variables": "babel dist/variables/esm --out-dir dist/variables/umd --plugins transform-es2015-modules-umd -q",
     "clean": "rimraf dist"
   },
   "devDependencies": {

packages/react-tokens/scripts/generateTokens.js

@@ -0,0 +1,237 @@
+const glob = require('glob');
+const { dirname, basename } = require('path');
+const { parse } = require('css');
+const { readFileSync } = require('fs');
+
+const pfStylesDir = dirname(require.resolve('@patternfly/patternfly/patternfly.css'));
+
+// Helpers
+const formatCustomPropertyName = key => key.replace('--pf-', '').replace(/-+/g, '_');
+
+const getRegexMatches = (string, regex) => {
+  const res = {};
+  let matches;
+  while ((matches = regex.exec(string))) {
+    res[matches[1]] = matches[2].trim();
+  }
+  return res;
+};
+
+const getDeclarations = cssAst =>
+  cssAst.stylesheet.rules
+    .filter(node => node.type === 'rule' && !node.selectors.includes('.pf-t-dark'))
+    .map(node => node.declarations.filter(decl => decl.type === 'declaration'))
+    .reduce((acc, val) => acc.concat(val), []); // flatten
+
+const formatFilePathToName = filePath => {
+  // const filePathArr = filePath.split('/');
+  let prefix = '';
+  if (filePath.includes('components/')) {
+    prefix = 'c_';
+  } else if (filePath.includes('layouts/')) {
+    prefix = 'l_';
+  }
+  return `${prefix}${basename(filePath, '.css').replace(/-+/g, '_')}`;
+};
+
+const getLocalVarsMap = cssFiles => {
+  const res = {};
+
+  cssFiles.forEach(filePath => {
+    const cssAst = parse(readFileSync(filePath, 'utf8'));
+
+    getDeclarations(cssAst).forEach(({ property, value, parent }) => {
+      if (property.startsWith('--pf')) {
+        res[property] = {
+          ...res[property],
+          [parent.selectors[0]]: value
+        };
+      }
+    });
+  });
+
+  return res;
+};
+
+/**
+ * Generates tokens from CSS in node_modules/@patternfly/patternfly/**
+ * @returns {object} of form {
+ *   c_about_modal_box: {
+ *     ".pf-c-about-modal-box" : {
+ *       "global_Color_100": {
+ *         "name": "--pf-global--Color--100",
+ *         "value": "#fff",
+ *         "values": [
+ *           "--pf-global--Color--light-100",
+ *           "$pf-global--Color--light-100",
+ *           "$pf-color-white",
+ *           "#fff"
+ *         ]
+ *       },
+ *     },
+ *     ".pf-c-about-modal-box .pf-c-card": {}
+ *   }
+ * }
+ */
+function generateTokens() {
+  const cssFiles = glob.sync(
+    ['{**/{components,layouts}/**/*.css', '**/patternfly-charts.css', '**/patternfly-variables.css}'].join(','),
+    {
+      cwd: pfStylesDir,
+      ignore: ['assets/**'],
+      absolute: true
+    }
+  );
+
+  // various lookup tables to resolve variables
+  const variables = readFileSync(require.resolve('@patternfly/patternfly/_variables.scss'), 'utf8');
+  const cssGlobalsToScssVarsMap = getRegexMatches(variables, /(--pf-.*):\s*(?:#{)?(\$?pf-[\w- _]+)}?;/g);
+
+  // contains default values and mappings to colors.scss for color values
+  const scssVariables = readFileSync(
+    require.resolve('@patternfly/patternfly/sass-utilities/scss-variables.scss'),
+    'utf8'
+  );
+  const scssVarsMap = getRegexMatches(scssVariables, /(\$.*):\s*([^;^!]+)/g);
+
+  // contains default values and mappings to colors.scss for color values
+  const scssColorVariables = readFileSync(require.resolve('@patternfly/patternfly/sass-utilities/colors.scss'), 'utf8');
+  const scssColorsMap = getRegexMatches(scssColorVariables, /(\$.*):\s*([^\s]+)\s*(?:!default);/g);
+
+  // contains default values and mappings to colors.scss for color values
+  const cssGlobalVariables = readFileSync(require.resolve('@patternfly/patternfly/patternfly-variables.css'), 'utf8');
+  const cssGlobalVariablesMap = getRegexMatches(cssGlobalVariables, /(--pf-[\w-]*):\s*([\w -_]+);/g);
+
+  const combinedScssVarsColorsMap = {
+    ...scssVarsMap,
+    ...scssColorsMap
+  };
+
+  const getComputedCSSVarValue = (value, selector, varMap) =>
+    value.replace(/var\(([\w-]*)\)/g, (full, match) => {
+      if (match.startsWith('--pf-global')) {
+        if (varMap[match]) {
+          return varMap[match];
+        } else {
+          return full;
+        }
+      } else {
+        if (selector) {
+          return getFromLocalVarsMap(match, selector);
+        }
+      }
+    });
+
+  const getComputedScssVarValue = value =>
+    value.replace(/\$pf[^,)\s*/]*/g, match => {
+      if (combinedScssVarsColorsMap[match]) {
+        return combinedScssVarsColorsMap[match];
+      } else {
+        return match;
+      }
+    });
+
+  const getVarsMap = (value, selector) => {
+    // evaluate the value and follow the variable chain
+    const varsMap = [value];
+
+    let computedValue = value;
+    let finalValue = value;
+    while (finalValue.includes('var(--pf') || computedValue.includes('var(--pf') || computedValue.includes('$pf-')) {
+      // keep following the variable chain until we get to a value
+      if (finalValue.includes('var(--pf')) {
+        finalValue = getComputedCSSVarValue(finalValue, selector, cssGlobalVariablesMap);
+      }
+      if (computedValue.includes('var(--pf')) {
+        computedValue = getComputedCSSVarValue(computedValue, selector, cssGlobalsToScssVarsMap);
+      } else {
+        computedValue = getComputedScssVarValue(computedValue);
+      }
+      varsMap.push(computedValue);
+    }
+    const lastElement = varsMap[varsMap.length - 1];
+    if (lastElement.includes('pf-')) {
+      varsMap.push(finalValue);
+    }
+    // all values should not be boxed by var()
+    return varsMap.map(variable => variable.replace(/var\(([\w-]*)\)/g, (full, match) => match));
+  };
+
+  // pre-populate the localVarsMap so we can lookup local variables within or across files, e.g. if we have the declaration:
+  // --pf-c-chip-group--MarginBottom: calc(var(--pf-c-chip-group--c-chip--MarginBottom) * -1);
+  // then we need to find:
+  // --pf-c-chip-group--c-chip--MarginBottom: var(--pf-global--spacer--xs);
+  const localVarsMap = getLocalVarsMap(cssFiles);
+
+  const getFromLocalVarsMap = (match, selector) => {
+    if (localVarsMap[match]) {
+      // have exact selectors match
+      if (localVarsMap[match][selector]) {
+        return localVarsMap[match][selector];
+      } else if (Object.keys(localVarsMap[match]).length === 1) {
+        // only one match, return its value
+        return Object.values(localVarsMap[match])[0];
+      } else {
+        // find the nearest parent selector and return its value
+        let bestMatch = '';
+        let bestValue = '';
+        for (const key in localVarsMap[match]) {
+          if (localVarsMap[match].hasOwnProperty(key)) {
+            // remove trailing * from key to compare
+            let sanitizedKey = key.replace(/\*$/, '').trim();
+            sanitizedKey = sanitizedKey.replace(/>$/, '').trim();
+            sanitizedKey = sanitizedKey.replace(/\[.*\]$/, '').trim();
+            // is key a parent of selector?
+            if (selector.indexOf(sanitizedKey) > -1) {
+              if (sanitizedKey.length > bestMatch.length) {
+                // longest matching key is the winner
+                bestMatch = key;
+                bestValue = localVarsMap[match][key];
+              }
+            }
+          }
+        }
+        if (!bestMatch) {
+          // eslint-disable-next-line no-console
+          console.error(`no matching selector found for ${match} in localVarsMap`);
+        }
+        return bestValue;
+      }
+    } else {
+      // eslint-disable-next-line no-console
+      console.error(`no matching property found for ${match} in localVarsMap`);
+    }
+  };
+
+  const fileTokens = {};
+  cssFiles.forEach(filePath => {
+    const cssAst = parse(readFileSync(filePath, 'utf8'));
+    // key is the formatted file name, e.g. c_about_modal_box
+    const key = formatFilePathToName(filePath);
+
+    getDeclarations(cssAst)
+      .filter(({ property }) => property.startsWith('--pf'))
+      .forEach(({ property, value, parent }) => {
+        const selector = parent.selectors[0];
+
+        const varsMap = getVarsMap(value, selector);
+        const propertyObj = {
+          name: property,
+          value: varsMap[varsMap.length - 1]
+        };
+        if (varsMap.length > 1) {
+          propertyObj.values = varsMap;
+        }
+
+        fileTokens[key] = fileTokens[key] || {};
+        fileTokens[key][selector] = fileTokens[key][selector] || {};
+        fileTokens[key][selector][formatCustomPropertyName(property)] = propertyObj;
+      });
+  });
+
+  return fileTokens;
+}
+
+module.exports = {
+  generateTokens
+};

packages/react-tokens/scripts/writeTokens.js

@@ -0,0 +1,87 @@
+const { outputFileSync } = require('fs-extra');
+const { resolve, join } = require('path');
+const { generateTokens } = require('./generateTokens');
+
+const outDir = resolve(__dirname, '../dist');
+
+const writeESMExport = (tokenName, tokenString) =>
+  outputFileSync(
+    join(outDir, 'esm/', `${tokenName}.js`),
+    `
+export const ${tokenName} = ${tokenString};
+export default ${tokenName};
+`.trim()
+  );
+
+const writeCJSExport = (tokenName, tokenString) =>
+  outputFileSync(
+    join(outDir, 'js', `${tokenName}.js`),
+    `
+"use strict";
+exports.__esModule = true;
+exports.${tokenName} = ${tokenString};
+exports["default"] = exports.${tokenName};
+`.trim()
+  );
+
+const writeDTSExport = (tokenName, tokenString) =>
+  outputFileSync(
+    join(outDir, 'js', `${tokenName}.d.ts`),
+    `
+export const ${tokenName}: ${tokenString};
+export default ${tokenName};
+`.trim()
+  );
+
+const index = [];
+
+/**
+ * Writes CJS and ESM tokens to `dist` directory
+ *
+ * @param {any} tokens tokens from generateTokens
+ */
+function writeTokens(tokens) {
+  Object.entries(tokens).forEach(([tokenName, tokenValue]) => {
+    const tokenString = JSON.stringify(tokenValue, null, 2);
+
+    writeESMExport(tokenName, tokenString);
+    writeCJSExport(tokenName, tokenString);
+    writeDTSExport(tokenName, tokenString);
+    index.push(tokenName);
+
+    // Legacy token support -- values may be incorrect.
+    Object.values(tokenValue)
+      .map(values => Object.entries(values))
+      .reduce((acc, val) => acc.concat(val), []) // flatten
+      .forEach(([oldTokenName, { name, value }]) => {
+        const oldToken = {
+          name,
+          value: oldTokenName.includes('chart') && !isNaN(+value) ? +value : value,
+          var: `var(${name})`
+        };
+        const oldTokenString = JSON.stringify(oldToken, null, 2);
+        writeESMExport(oldTokenName, oldTokenString);
+        writeCJSExport(oldTokenName, oldTokenString);
+        writeDTSExport(oldTokenName, oldTokenString);
+        index.push(oldTokenName);
+      });
+  });
+
+  // Index files including legacy tokens
+  const esmIndexString = index.map(file => `export * from './${file}';`).join('\n');
+  outputFileSync(join(outDir, 'esm', 'index.js'), esmIndexString);
+  outputFileSync(join(outDir, 'js', 'index.d.ts'), esmIndexString);
+  outputFileSync(
+    join(outDir, 'js', 'index.js'),
+    `
+"use strict";
+function __export(m) {
+    for (var p in m) if (!exports.hasOwnProperty(p)) exports[p] = m[p];
+}
+exports.__esModule = true;
+${index.map(file => `__export(require('./${file}'));`).join('\n')}
+`.trim()
+  );
+}
+
+writeTokens(generateTokens());