Merge pull request #277 from lo1tuma/improve-benchmark-reliability

Improve benchmark test reliability

benchmarks/measure.js

@@ -6,9 +6,24 @@ const times = require('ramda/src/times');
 const median = require('ramda/src/median');
 const map = require('ramda/src/map');
 const prop = require('ramda/src/prop');
+const semver = require('semver');
 
 const [ { speed: cpuSpeed } ] = os.cpus();
 
+function getNodeVersionMultiplier() {
+    const currentNodeVersion = process.version;
+
+    if (semver.lt(currentNodeVersion, '14.0.0') && semver.gte(currentNodeVersion, '12.0.0')) {
+        return 1.5;
+    }
+
+    if (semver.lt(currentNodeVersion, '12.0.0')) {
+        return 2;
+    }
+
+    return 1;
+}
+
 function clearRequireCache() {
     Object.keys(require.cache).forEach(function (key) {
         delete require.cache[key];
@@ -36,4 +51,4 @@ function runBenchmark(fn, count) {
     return { medianDuration, medianMemory };
 }
 
-module.exports = { runBenchmark, clearRequireCache, cpuSpeed };
+module.exports = { runBenchmark, clearRequireCache, cpuSpeed, getNodeVersionMultiplier };

benchmarks/runtime.bench.js

@@ -5,7 +5,7 @@ const { Linter } = require('eslint');
 const times = require('ramda/src/times');
 const toPairs = require('ramda/src/toPairs');
 const fromPairs = require('ramda/src/fromPairs');
-const { runBenchmark, cpuSpeed } = require('./measure');
+const { runBenchmark, cpuSpeed, getNodeVersionMultiplier } = require('./measure');
 const mochaPlugin = require('../');
 
 const recommendedRules = mochaPlugin.configs.recommended.rules;
@@ -85,7 +85,8 @@ function lintManyFilesWithAllRecommendedRules({ numberOfFiles }) {
 
 describe('runtime', () => {
     it('should not take longer as the defined budget to lint many files with the recommended config', () => {
-        const budget = 80000000 / cpuSpeed;
+        const nodeVersionMultiplier = getNodeVersionMultiplier();
+        const budget = 80000000 / cpuSpeed * nodeVersionMultiplier;
 
         const { medianDuration } = runBenchmark(() => {
             lintManyFilesWithAllRecommendedRules({ numberOfFiles: 350 });

package.json

@@ -34,7 +34,8 @@
         "eslint-plugin-unicorn": "^21.0.0",
         "mocha": "^8.1.0",
         "nyc": "^15.1.0",
-        "pr-log": "^4.0.0"
+        "pr-log": "^4.0.0",
+        "semver": "^7.3.4"
     },
     "peerDependencies": {
         "eslint": ">=7.0.0"