From dbb2b1abf49ce4582257c858189dcfcb0ce0e09b Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 16 May 2018 18:10:08 +0200 Subject: [PATCH 1/5] util: limit inspection output size to 128 MB The maximum hard limit that `util.inspect()` could theoretically handle is the maximum string size. That is ~2 ** 28 on 32 bit systems and ~2 ** 30 on 64 bit systems. Due to the recursive algorithm a complex object could easily exceed that limit without throwing an error right away and therefore crashing the application by exceeding the heap limit. `util.inspect()` is fast enough to compute 128 MB of data below one second on an Intel(R) Core(TM) i7-5600U CPU. This hard limit allows to inspect arbitrary big objects from now on without crashing the application or blocking the event loop significantly. --- doc/api/util.md | 15 +++-- lib/util.js | 62 ++++++++++++------- .../test-util-inspect-long-running.js | 20 ++++++ 3 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 test/parallel/test-util-inspect-long-running.js diff --git a/doc/api/util.md b/doc/api/util.md index 44495e977d13c3..c2fdc04f7c13f7 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -360,6 +360,10 @@ stream.write('With ES6'); - * `maxArrayLength` {number} Specifies the maximum number of `Array`, + * `maxArrayLength` {integer} Specifies the maximum number of `Array`, [`TypedArray`][], [`WeakMap`][] and [`WeakSet`][] elements to include when formatting. Set to `null` or `Infinity` to show all elements. Set to `0` or negative to show no elements. **Default:** `100`. - * `breakLength` {number} The length at which an object's keys are split + * `breakLength` {integer} The length at which an object's keys are split across multiple lines. Set to `Infinity` to format an object as a single line. **Default:** `60` for legacy compatibility. * `compact` {boolean} Setting this to `false` changes the default indentation @@ -532,9 +536,10 @@ console.log(inspect(weakSet, { showHidden: true })); ``` Please note that `util.inspect()` is a synchronous method that is mainly -intended as a debugging tool. Some input values can have a significant -performance overhead that can block the event loop. Use this function -with care and never in a hot code path. +intended as a debugging tool. It's maximum output length is limited to 128 MB +and input values that result in output bigger than that will not be inspected +fully. Such values can have a significant performance overhead that can block +the event loop for a significant amount of time. ### Customizing `util.inspect` colors diff --git a/lib/util.js b/lib/util.js index 35bd9ec46f5b6f..fe475e8fd4c5f4 100644 --- a/lib/util.js +++ b/lib/util.js @@ -406,24 +406,27 @@ function inspect(value, opts) { maxArrayLength: inspectDefaultOptions.maxArrayLength, breakLength: inspectDefaultOptions.breakLength, indentationLvl: 0, - compact: inspectDefaultOptions.compact + compact: inspectDefaultOptions.compact, + budget: {} }; - // Legacy... - if (arguments.length > 2) { - if (arguments[2] !== undefined) { - ctx.depth = arguments[2]; - } - if (arguments.length > 3 && arguments[3] !== undefined) { - ctx.colors = arguments[3]; + if (arguments.length > 1) { + // Legacy... + if (arguments.length > 2) { + if (arguments[2] !== undefined) { + ctx.depth = arguments[2]; + } + if (arguments.length > 3 && arguments[3] !== undefined) { + ctx.colors = arguments[3]; + } } - } - // Set user-specified options - if (typeof opts === 'boolean') { - ctx.showHidden = opts; - } else if (opts) { - const optKeys = Object.keys(opts); - for (var i = 0; i < optKeys.length; i++) { - ctx[optKeys[i]] = opts[optKeys[i]]; + // Set user-specified options + if (typeof opts === 'boolean') { + ctx.showHidden = opts; + } else if (opts) { + const optKeys = Object.keys(opts); + for (var i = 0; i < optKeys.length; i++) { + ctx[optKeys[i]] = opts[optKeys[i]]; + } } } if (ctx.colors) ctx.stylize = stylizeWithColor; @@ -623,7 +626,7 @@ function noPrototypeIterator(ctx, value, recurseTimes) { // corrected by setting `ctx.indentationLvL += diff` and then to decrease the // value afterwards again. function formatValue(ctx, value, recurseTimes) { - // Primitive types cannot have properties + // Primitive types cannot have properties. if (typeof value !== 'object' && typeof value !== 'function') { return formatPrimitive(ctx.stylize, value, ctx); } @@ -631,6 +634,11 @@ function formatValue(ctx, value, recurseTimes) { return ctx.stylize('null', 'null'); } + if (ctx.stop) { + const name = getConstructorName(value) || value[Symbol.toStringTag]; + return ctx.stylize(`[${name || 'Object'}]`, 'special'); + } + if (ctx.showProxy) { const proxy = getProxyDetails(value); if (proxy !== undefined) { @@ -639,11 +647,11 @@ function formatValue(ctx, value, recurseTimes) { } // Provide a hook for user-specified inspect functions. - // Check that value is an object with an inspect function on it + // Check that value is an object with an inspect function on it. if (ctx.customInspect) { const maybeCustom = value[customInspectSymbol]; if (typeof maybeCustom === 'function' && - // Filter out the util module, its inspect function is special + // Filter out the util module, its inspect function is special. maybeCustom !== exports.inspect && // Also filter out any prototype objects using the circular check. !(value.constructor && value.constructor.prototype === value)) { @@ -685,7 +693,7 @@ function formatRaw(ctx, value, recurseTimes) { let extrasType = kObjectType; - // Iterators and the rest are split to reduce checks + // Iterators and the rest are split to reduce checks. if (value[Symbol.iterator]) { noIterator = false; if (Array.isArray(value)) { @@ -766,7 +774,7 @@ function formatRaw(ctx, value, recurseTimes) { } base = dateToISOString(value); } else if (isError(value)) { - // Make error with message first say the error + // Make error with message first say the error. base = formatError(value); // Wrap the error in brackets in case it has no stack trace. const stackStart = base.indexOf('\n at'); @@ -885,7 +893,14 @@ function formatRaw(ctx, value, recurseTimes) { } ctx.seen.pop(); - return reduceToSingleString(ctx, output, base, braces); + const res = reduceToSingleString(ctx, output, base, braces); + const budget = ctx.budget[ctx.indentationLvl] || 0; + const newLength = budget + res.length; + ctx.budget[ctx.indentationLvl] = newLength; + if (newLength > 2 ** 27) { + ctx.stop = true; + } + return res; } function handleMaxCallStackSize(ctx, err, constructor, tag) { @@ -1057,8 +1072,9 @@ function formatTypedArray(ctx, value, recurseTimes) { formatBigInt; for (var i = 0; i < maxLength; ++i) output[i] = elementFormatter(ctx.stylize, value[i]); - if (remaining > 0) + if (remaining > 0) { output[i] = `... ${remaining} more item${remaining > 1 ? 's' : ''}`; + } if (ctx.showHidden) { // .buffer goes last, it's not a primitive like the others. ctx.indentationLvl += 2; diff --git a/test/parallel/test-util-inspect-long-running.js b/test/parallel/test-util-inspect-long-running.js new file mode 100644 index 00000000000000..167f72ba64631a --- /dev/null +++ b/test/parallel/test-util-inspect-long-running.js @@ -0,0 +1,20 @@ +'use strict'; + +require('../common'); + +// Test that huge objects don't crash due to exceeding the maximum heap size. + +const util = require('util'); + +// Create a difficult to stringify object. Without the artificial limitation +// this would crash or throw an maximum string size error. +let last = {}; +const obj = last; + +for (let i = 0; i < 1000; i++) { + last.next = { circular: obj, last, obj: { a: 1, b: 2, c: true } }; + last = last.next; + obj[i] = last; +} + +util.inspect(obj, { depth: Infinity }); From 8742df1b3e54b43390583f88561636948a07d0ca Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Sat, 8 Sep 2018 16:52:44 +0200 Subject: [PATCH 2/5] fixup: address comment --- doc/api/util.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/api/util.md b/doc/api/util.md index c2fdc04f7c13f7..1945d5f63847a7 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -536,10 +536,10 @@ console.log(inspect(weakSet, { showHidden: true })); ``` Please note that `util.inspect()` is a synchronous method that is mainly -intended as a debugging tool. It's maximum output length is limited to 128 MB -and input values that result in output bigger than that will not be inspected -fully. Such values can have a significant performance overhead that can block -the event loop for a significant amount of time. +intended as a debugging tool. Its maximum output length is limited to 128 MB and +input values that result in output bigger than that will not be inspected fully. +Such values can have a significant performance overhead that can block the event +loop for a significant amount of time. ### Customizing `util.inspect` colors From b1cb0157dd4196bba678a1991a6da4ab9d54da78 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 11 Sep 2018 22:44:24 +0200 Subject: [PATCH 3/5] fixup: add comment --- lib/util.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/util.js b/lib/util.js index fe475e8fd4c5f4..710845d53a933e 100644 --- a/lib/util.js +++ b/lib/util.js @@ -634,7 +634,7 @@ function formatValue(ctx, value, recurseTimes) { return ctx.stylize('null', 'null'); } - if (ctx.stop) { + if (ctx.stop !== undefined) { const name = getConstructorName(value) || value[Symbol.toStringTag]; return ctx.stylize(`[${name || 'Object'}]`, 'special'); } @@ -897,6 +897,13 @@ function formatRaw(ctx, value, recurseTimes) { const budget = ctx.budget[ctx.indentationLvl] || 0; const newLength = budget + res.length; ctx.budget[ctx.indentationLvl] = newLength; + // If any indentationLvl exceeds this limit, limit further inspecting to the + // minimum. Otherwise the recursive algorithm might continue inspecting the + // object even tough the maximum string size (~2 ** 28 on 32 bit systems and + // ~2 ** 64) exceeded. The actual output is not limited at exactly 2 ** 27 but + // a bit higher. This depends on the object shape. + // This limit also makes sure that huge objects don't block the event loop + // significantly. if (newLength > 2 ** 27) { ctx.stop = true; } From e133551c9ce0d6158bb59993040520b16a7553e5 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 11 Sep 2018 23:37:16 +0200 Subject: [PATCH 4/5] fixup: address comment --- lib/util.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/util.js b/lib/util.js index 710845d53a933e..985f455e5dc455 100644 --- a/lib/util.js +++ b/lib/util.js @@ -899,9 +899,9 @@ function formatRaw(ctx, value, recurseTimes) { ctx.budget[ctx.indentationLvl] = newLength; // If any indentationLvl exceeds this limit, limit further inspecting to the // minimum. Otherwise the recursive algorithm might continue inspecting the - // object even tough the maximum string size (~2 ** 28 on 32 bit systems and - // ~2 ** 64) exceeded. The actual output is not limited at exactly 2 ** 27 but - // a bit higher. This depends on the object shape. + // object even though the maximum string size (~2 ** 28 on 32 bit systems and + // ~2 ** 30 on 64 bit systems) exceeded. The actual output is not limited at + // exactly 2 ** 27 but a bit higher. This depends on the object shape. // This limit also makes sure that huge objects don't block the event loop // significantly. if (newLength > 2 ** 27) { From 0dde647e2bca4f48dcf07d65b16c95ac8f4c398f Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 12 Sep 2018 23:54:12 +0200 Subject: [PATCH 5/5] fixup: address comments in util.md --- doc/api/util.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/api/util.md b/doc/api/util.md index 1945d5f63847a7..e69a4849fefb73 100644 --- a/doc/api/util.md +++ b/doc/api/util.md @@ -362,8 +362,8 @@ added: v0.3.0 changes: - version: REPLACEME pr-url: https://github.com/nodejs/node/pull/REPLACEME - description: The inspection output is now limited to 128 MB. Data above that - size not be inspected fully anymore. + description: The inspection output is now limited to about 128 MB. Data + above that size will not be fully inspected. - version: v10.6.0 pr-url: https://github.com/nodejs/node/pull/20725 description: Inspecting linked lists and similar objects is now possible @@ -536,10 +536,10 @@ console.log(inspect(weakSet, { showHidden: true })); ``` Please note that `util.inspect()` is a synchronous method that is mainly -intended as a debugging tool. Its maximum output length is limited to 128 MB and -input values that result in output bigger than that will not be inspected fully. -Such values can have a significant performance overhead that can block the event -loop for a significant amount of time. +intended as a debugging tool. Its maximum output length is limited to +approximately 128 MB and input values that result in output bigger than that +will not be inspected fully. Such values can have a significant performance +overhead that can block the event loop for a significant amount of time. ### Customizing `util.inspect` colors