feat: hide samples with path containing @google-cloud/profiler by def…

…ault (#262)

* feat: hide samples with path containing @google-cloud/profiler by default

* clarify comment

ts/src/config.ts

@@ -78,6 +78,12 @@ export interface Config extends GoogleAuthOptions {
   // stack depth may increase overhead of profiling.
   heapMaxStackDepth?: number;
 
+  // Samples with stacks with any location containing this as a substring
+  // in their file name will not be included in heap profiles.
+  // By default this is set to "@google-cloud/profiler" to exclude samples from
+  // the profiler.
+  ignoreHeapSamplesPath?: string;
+
   // On each consecutive error in profile creation, the backoff envelope will
   // increase by this factor. The backoff will be a random value selected
   // from a uniform distribution between 0 and the backoff envelope.
@@ -134,6 +140,7 @@ export interface ProfilerConfig extends GoogleAuthOptions {
   timeIntervalMicros: number;
   heapIntervalBytes: number;
   heapMaxStackDepth: number;
+  ignoreHeapSamplesPath: string;
   initialBackoffMillis: number;
   backoffCapMillis: number;
   backoffMultiplier: number;
@@ -153,6 +160,7 @@ export const defaultConfig = {
   timeIntervalMicros: 1000,
   heapIntervalBytes: 512 * 1024,
   heapMaxStackDepth: 64,
+  ignoreHeapSamplesPath: '@google-cloud/profiler',
   initialBackoffMillis: 60 * 1000,  // 1 minute
   backoffCapMillis: parseDuration('1h'),
   backoffMultiplier: 1.3,

ts/src/profilers/heap-profiler.ts

@@ -30,7 +30,8 @@ let heapStackDepth = 0;
  * Collects a heap profile when heapProfiler is enabled. Otherwise throws
  * an error.
  */
-export function profile(): perftools.profiles.IProfile {
+export function profile(ignoreSamplePath?: string):
+    perftools.profiles.IProfile {
   if (!enabled) {
     throw new Error('Heap profiler is not enabled.');
   }
@@ -41,14 +42,17 @@ export function profile(): perftools.profiles.IProfile {
   // TODO: remove any once type definition is updated to include external.
   // tslint:disable-next-line: no-any
   const {external}: {external: number} = process.memoryUsage() as any;
-  const externalNode: AllocationProfileNode = {
-    name: '(external)',
-    scriptName: '',
-    children: [],
-    allocations: [{sizeBytes: external, count: 1}],
-  };
-  result.children.push(externalNode);
-  return serializeHeapProfile(result, startTimeNanos, heapIntervalBytes);
+  if (external > 0) {
+    const externalNode: AllocationProfileNode = {
+      name: '(external)',
+      scriptName: '',
+      children: [],
+      allocations: [{sizeBytes: external, count: 1}],
+    };
+    result.children.push(externalNode);
+  }
+  return serializeHeapProfile(
+      result, startTimeNanos, heapIntervalBytes, ignoreSamplePath);
 }
 
 /**

ts/src/profilers/profile-serializer.ts

@@ -80,7 +80,8 @@ class StringTable {
  */
 function serialize<T extends ProfileNode>(
     profile: perftools.profiles.IProfile, root: T,
-    appendToSamples: AppendEntryToSamples<T>, stringTable: StringTable) {
+    appendToSamples: AppendEntryToSamples<T>, stringTable: StringTable,
+    ignoreSamplesPath?: string) {
   const samples: perftools.profiles.Sample[] = [];
   const locations: perftools.profiles.Location[] = [];
   const functions: perftools.profiles.Function[] = [];
@@ -94,6 +95,9 @@ function serialize<T extends ProfileNode>(
   while (entries.length > 0) {
     const entry = entries.pop()!;
     const node = entry.node;
+    if (ignoreSamplesPath && node.scriptName.indexOf(ignoreSamplesPath) > -1) {
+      continue;
+    }
     const stack = entry.stack;
     const location = getLocation(node);
     stack.unshift(location.id as number);
@@ -246,8 +250,8 @@ export function serializeTimeProfile(
  * @param intervalBytes - bytes allocated between samples.
  */
 export function serializeHeapProfile(
-    prof: AllocationProfileNode, startTimeNanos: number,
-    intervalBytes: number): perftools.profiles.IProfile {
+    prof: AllocationProfileNode, startTimeNanos: number, intervalBytes: number,
+    ignoreSamplesPath?: string): perftools.profiles.IProfile {
   const appendHeapEntryToSamples: AppendEntryToSamples<AllocationProfileNode> =
       (entry: Entry<AllocationProfileNode>,
        samples: perftools.profiles.Sample[]) => {
@@ -274,6 +278,7 @@ export function serializeHeapProfile(
     period: intervalBytes,
   };
 
-  serialize(profile, prof, appendHeapEntryToSamples, stringTable);
+  serialize(
+      profile, prof, appendHeapEntryToSamples, stringTable, ignoreSamplesPath);
   return profile;
 }

ts/test/profiles-for-tests.ts

@@ -447,4 +447,170 @@ export const anonymousFunctionTimeProfile: perftools.profiles.IProfile =
       durationNanos: 10 * 1000 * 1000 * 1000,
       periodType: new perftools.profiles.ValueType({type: 3, unit: 4}),
       period: 1000,
-    });
+    });
+
+const heapWithPathLeaf1 = {
+  name: 'foo2',
+  scriptName: 'foo.ts',
+  scriptId: 0,
+  lineNumber: 3,
+  columnNumber: 3,
+  allocations: [{count: 1, sizeBytes: 2}],
+  children: []
+};
+
+
+const heapWithPathLeaf2 = {
+  name: 'bar',
+  scriptName: '@google-cloud/profiler/profiler.ts',
+  scriptId: 1,
+  lineNumber: 10,
+  columnNumber: 5,
+  allocations: [{count: 2, sizeBytes: 2}],
+  children: []
+};
+
+const heapWithPathLeaf3 = {
+  name: 'bar',
+  scriptName: 'bar.ts',
+  scriptId: 2,
+  lineNumber: 3,
+  columnNumber: 3,
+  allocations: [{count: 3, sizeBytes: 2}],
+  children: []
+};
+
+const heapWithPathNode2 = {
+  name: 'baz',
+  scriptName: 'foo.ts',
+  scriptId: 0,
+  lineNumber: 1,
+  columnNumber: 5,
+  allocations: [],
+  children: [heapWithPathLeaf1, heapWithPathLeaf2]
+};
+
+const heapWithPathNode1 = {
+  name: 'foo1',
+  scriptName: 'node_modules/@google-cloud/profiler/profiler.ts',
+  scriptId: 3,
+  lineNumber: 2,
+  columnNumber: 5,
+  allocations: [],
+  children: [heapWithPathLeaf3]
+};
+
+export const v8HeapWithPathProfile = Object.freeze({
+  name: '(root)',
+  scriptName: '(root)',
+  scriptId: 10000,
+  lineNumber: 0,
+  columnNumber: 5,
+  allocations: [],
+  children: [heapWithPathNode1, heapWithPathNode2]
+});
+
+const heapIncludePathFunctions = [
+  new perftools.profiles.Function({id: 1, name: 5, systemName: 5, filename: 6}),
+  new perftools.profiles.Function({id: 2, name: 7, systemName: 7, filename: 8}),
+  new perftools.profiles.Function({id: 3, name: 9, systemName: 9, filename: 6}),
+  new perftools.profiles.Function(
+      {id: 4, name: 10, systemName: 10, filename: 11}),
+  new perftools.profiles.Function(
+      {id: 5, name: 7, systemName: 7, filename: 12})
+];
+
+const heapIncludePathLocations = [
+  new perftools.profiles.Location({line: [{functionId: 1, line: 1}], id: 1}),
+  new perftools.profiles.Location({line: [{functionId: 2, line: 10}], id: 2}),
+  new perftools.profiles.Location({line: [{functionId: 3, line: 3}], id: 3}),
+  new perftools.profiles.Location({line: [{functionId: 4, line: 2}], id: 4}),
+  new perftools.profiles.Location({line: [{functionId: 5, line: 3}], id: 5}),
+];
+
+export const heapProfileIncludePath: perftools.profiles.IProfile =
+    Object.freeze({
+      sampleType: [
+        new perftools.profiles.ValueType({type: 1, unit: 2}),
+        new perftools.profiles.ValueType({type: 3, unit: 4}),
+      ],
+      sample: [
+        new perftools.profiles.Sample(
+            {locationId: [2, 1], value: [2, 4], label: []}),
+        new perftools.profiles.Sample(
+            {locationId: [3, 1], value: [1, 2], label: []}),
+        new perftools.profiles.Sample(
+            {locationId: [5, 4], value: [3, 6], label: []}),
+      ],
+      location: heapIncludePathLocations,
+      function: heapIncludePathFunctions,
+      stringTable: [
+        '',
+        'objects',
+        'count',
+        'space',
+        'bytes',
+        'baz',
+        'foo.ts',
+        'bar',
+        '@google-cloud/profiler/profiler.ts',
+        'foo2',
+        'foo1',
+        'node_modules/@google-cloud/profiler/profiler.ts',
+        'bar.ts',
+      ],
+      timeNanos: 0,
+      periodType: new perftools.profiles.ValueType({type: 3, unit: 4}),
+      period: 524288
+    });
+
+// heapProfile is encoded then decoded to convert numbers to longs, in
+// decodedHeapProfile
+const encodedHeapProfileIncludePath =
+    perftools.profiles.Profile.encode(heapProfileIncludePath).finish();
+export const decodedHeapProfileIncludePath = Object.freeze(
+    perftools.profiles.Profile.decode(encodedHeapProfileIncludePath));
+
+const heapExcludePathFunctions = [
+  new perftools.profiles.Function({id: 1, name: 5, systemName: 5, filename: 6}),
+  new perftools.profiles.Function({id: 2, name: 7, systemName: 7, filename: 6}),
+];
+
+const heapExcludePathLocations = [
+  new perftools.profiles.Location({line: [{functionId: 1, line: 1}], id: 1}),
+  new perftools.profiles.Location({line: [{functionId: 2, line: 3}], id: 2}),
+];
+
+export const heapProfileExcludePath: perftools.profiles.IProfile =
+    Object.freeze({
+      sampleType: [
+        new perftools.profiles.ValueType({type: 1, unit: 2}),
+        new perftools.profiles.ValueType({type: 3, unit: 4}),
+      ],
+      sample: [
+        new perftools.profiles.Sample(
+            {locationId: [2, 1], value: [1, 2], label: []}),
+      ],
+      location: heapExcludePathLocations,
+      function: heapExcludePathFunctions,
+      stringTable: [
+        '',
+        'objects',
+        'count',
+        'space',
+        'bytes',
+        'baz',
+        'foo.ts',
+        'foo2',
+      ],
+      timeNanos: 0,
+      periodType: new perftools.profiles.ValueType({type: 3, unit: 4}),
+      period: 524288
+    });
+
+// heapProfile is encoded then decoded to convert numbers to longs, in
+// decodedHeapProfile
+const encodedHeapProfileExcludePath =
+    perftools.profiles.Profile.encode(heapProfileExcludePath).finish();
+export const decodedHeapProfileExcludePath = Object.freeze(
+    perftools.profiles.Profile.decode(encodedHeapProfileExcludePath));

ts/test/test-heap-profiler.ts

@@ -20,7 +20,7 @@ import * as sinon from 'sinon';
 import {perftools} from '../../proto/profile';
 import * as heapProfiler from '../src/profilers/heap-profiler';
 
-import {heapProfileWithExternal, v8HeapProfile} from './profiles-for-tests';
+import {heapProfileExcludePath, heapProfileIncludePath, heapProfileWithExternal, v8HeapProfile, v8HeapWithPathProfile} from './profiles-for-tests';
 
 const copy = require('deep-copy');
 const assert = require('assert');
@@ -35,18 +35,7 @@ describe('HeapProfiler', () => {
   beforeEach(() => {
     startStub = sinon.stub(v8HeapProfiler, 'startSamplingHeapProfiler');
     stopStub = sinon.stub(v8HeapProfiler, 'stopSamplingHeapProfiler');
-
-    // returns copy of v8HeapProfile because heap profiler modifies the v8
-    // profile to add external memory usage.
-    profileStub = sinon.stub(v8HeapProfiler, 'getAllocationProfile')
-                      .returns(copy(v8HeapProfile));
     dateStub = sinon.stub(Date, 'now').returns(0);
-    memoryUsageStub = sinon.stub(process, 'memoryUsage').returns({
-      external: 1024,
-      rss: 2048,
-      heapTotal: 4096,
-      heapUse: 2048,
-    });
   });
 
   afterEach(() => {
@@ -58,13 +47,56 @@ describe('HeapProfiler', () => {
     memoryUsageStub.restore();
   });
   describe('profile', () => {
-    it('should return a profile equal to the expected profile', async () => {
-      const intervalBytes = 1024 * 512;
-      const stackDepth = 32;
-      heapProfiler.start(intervalBytes, stackDepth);
-      const profile = heapProfiler.profile();
-      assert.deepEqual(heapProfileWithExternal, profile);
-    });
+    it('should return a profile equal to the expected profile when external memory is allocated',
+       async () => {
+         profileStub = sinon.stub(v8HeapProfiler, 'getAllocationProfile')
+                           .returns(copy(v8HeapProfile));
+         memoryUsageStub = sinon.stub(process, 'memoryUsage').returns({
+           external: 1024,
+           rss: 2048,
+           heapTotal: 4096,
+           heapUse: 2048,
+         });
+         const intervalBytes = 1024 * 512;
+         const stackDepth = 32;
+         heapProfiler.start(intervalBytes, stackDepth);
+         const profile = heapProfiler.profile();
+         assert.deepEqual(heapProfileWithExternal, profile);
+       });
+
+    it('should return a profile equal to the expected profile when including all samples',
+       async () => {
+         profileStub = sinon.stub(v8HeapProfiler, 'getAllocationProfile')
+                           .returns(copy(v8HeapWithPathProfile));
+         memoryUsageStub = sinon.stub(process, 'memoryUsage').returns({
+           external: 0,
+           rss: 2048,
+           heapTotal: 4096,
+           heapUse: 2048,
+         });
+         const intervalBytes = 1024 * 512;
+         const stackDepth = 32;
+         heapProfiler.start(intervalBytes, stackDepth);
+         const profile = heapProfiler.profile();
+         assert.deepEqual(heapProfileIncludePath, profile);
+       });
+
+    it('should return a profile equal to the expected profile when excluding profiler samples',
+       async () => {
+         profileStub = sinon.stub(v8HeapProfiler, 'getAllocationProfile')
+                           .returns(copy(v8HeapWithPathProfile));
+         memoryUsageStub = sinon.stub(process, 'memoryUsage').returns({
+           external: 0,
+           rss: 2048,
+           heapTotal: 4096,
+           heapUse: 2048,
+         });
+         const intervalBytes = 1024 * 512;
+         const stackDepth = 32;
+         heapProfiler.start(intervalBytes, stackDepth);
+         const profile = heapProfiler.profile('@google-cloud/profiler');
+         assert.deepEqual(heapProfileExcludePath, profile);
+       });
 
     it('should throw error when not started', () => {
       assert.throws(

ts/test/test-init-config.ts

@@ -75,6 +75,7 @@ describe('createProfiler', () => {
     timeIntervalMicros: 1000,
     heapIntervalBytes: 512 * 1024,
     heapMaxStackDepth: 64,
+    ignoreHeapSamplesPath: '@google-cloud/profiler',
     initialBackoffMillis: 1000 * 60,
     backoffCapMillis: 60 * 60 * 1000,
     backoffMultiplier: 1.3,