Skip to content

Commit

Permalink
fix: start sampling heap profiler before any asynchronous calls (#161)
Browse files Browse the repository at this point in the history 
* fix: start sampling heap profiler before any asynchronous calls

* address comments

* clean up testing
  • Loading branch information
@nolanmar511
nolanmar511 authored May 3, 2018
1 parent 24bd4dd commit 33cd531
Show file tree
Hide file tree
Showing 6 changed files with 316 additions and 163 deletions.
108 changes: 73 additions & 35 deletions ts/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import {Logger, util} from '@google-cloud/common';
import * as delay from 'delay';
import * as extend from 'extend';
import * as fs from 'fs';
import * as gcpMetadata from 'gcp-metadata';
import * as path from 'path';
import {normalize} from 'path';
Expand All @@ -25,6 +26,7 @@ import * as semver from 'semver';

import {Config, defaultConfig, ProfilerConfig} from './config';
import {Profiler} from './profiler';
import * as heapProfiler from './profilers/heap-profiler';

const pjson = require('../../package.json');

Expand All @@ -45,12 +47,10 @@ function hasService(config: Config):

/**
* Sets unset values in the configuration to the value retrieved from
* environment variables, metadata, or specified in defaultConfig.
* environment variables or specified in defaultConfig.
* Throws error if value that must be set cannot be initialized.
*
* Exported for testing purposes.
*/
export async function initConfig(config: Config): Promise<ProfilerConfig> {
function initConfigLocal(config: Config): ProfilerConfig {
config = util.normalizeArguments(null, config);

const envConfig: Config = {
Expand All @@ -69,15 +69,38 @@ export async function initConfig(config: Config): Promise<ProfilerConfig> {
}

let envSetConfig: Config = {};
const val = process.env.GCLOUD_PROFILER_CONFIG;
if (val) {
envSetConfig = require(path.resolve(val)) as Config;
const configPath = process.env.GCLOUD_PROFILER_CONFIG;
if (configPath) {
let envSetConfigBuf;
try {
envSetConfigBuf = fs.readFileSync(configPath);
} catch (e) {
throw Error(`Could not read GCLOUD_PROFILER_CONFIG ${configPath}: ${e}`);
}
try {
envSetConfig = JSON.parse(envSetConfigBuf.toString());
} catch (e) {
throw Error(`Could not parse GCLOUD_PROFILER_CONFIG ${configPath}: ${e}`);
}
}

const mergedConfig =
extend(true, {}, defaultConfig, envSetConfig, envConfig, config);

if (!mergedConfig.zone || !mergedConfig.instance) {
if (!hasService(mergedConfig)) {
throw new Error('Service must be specified in the configuration.');
}

return mergedConfig;
}

/**
* Sets unset values in the configuration which can be retrieved from GCP
* metadata.
*/
async function initConfigMetadata(config: ProfilerConfig):
Promise<ProfilerConfig> {
if (!config.zone || !config.instance) {
const [instance, zone] =
await Promise
.all([
Expand All @@ -88,22 +111,40 @@ export async function initConfig(config: Config): Promise<ProfilerConfig> {
// ignore errors, which will occur when not on GCE.
}) ||
[undefined, undefined];
if (!mergedConfig.zone && zone) {
mergedConfig.zone = zone.substring(zone.lastIndexOf('/') + 1);
if (!config.zone && zone) {
config.zone = zone.substring(zone.lastIndexOf('/') + 1);
}
if (!mergedConfig.instance && instance) {
mergedConfig.instance = instance;
if (!config.instance && instance) {
config.instance = instance;
}
}
return config;
}

if (!hasService(mergedConfig)) {
throw new Error('Service must be specified in the configuration.');
/**
* Initializes the config, and starts heap profiler if the heap profiler is
* needed. Returns a profiler if creation is successful. Otherwise, returns
* rejected promise.
*/
export async function createProfiler(config: Config): Promise<Profiler> {
if (!semver.satisfies(process.version, pjson.engines.node)) {
throw new Error(
`Could not start profiler: node version ${process.version}` +
` does not satisfies "${pjson.engines.node}"`);
}

return mergedConfig;
}
let profilerConfig: ProfilerConfig = initConfigLocal(config);

let profiler: Profiler|undefined = undefined;
// Start the heap profiler if profiler config does not indicate heap profiling
// is disabled. This must be done before any asynchronous calls are made so
// all memory allocations made after start() is called can be captured.
if (!profilerConfig.disableHeap) {
heapProfiler.start(
profilerConfig.heapIntervalBytes, profilerConfig.heapMaxStackDepth);
}
profilerConfig = await initConfigMetadata(profilerConfig);
return new Profiler(profilerConfig);
}

/**
* Starts the profiling agent and returns a promise.
Expand All @@ -120,21 +161,13 @@ let profiler: Profiler|undefined = undefined;
*
*/
export async function start(config: Config = {}): Promise<void> {
if (!semver.satisfies(process.version, pjson.engines.node)) {
logError(
`Could not start profiler: node version ${process.version}` +
` does not satisfies "${pjson.engines.node}"`,
config);
return;
}
let normalizedConfig: ProfilerConfig;
let profiler: Profiler;
try {
normalizedConfig = await initConfig(config);
profiler = await createProfiler(config);
} catch (e) {
logError(`Could not start profiler: ${e}`, config);
logError(`${e}`, config);
return;
}
profiler = new Profiler(normalizedConfig);
profiler.start();
}

Expand All @@ -152,12 +185,17 @@ function logError(msg: string, config: Config) {
* profiles.
*/
export async function startLocal(config: Config = {}): Promise<void> {
const normalizedConfig = await initConfig(config);
const profiler = new Profiler(normalizedConfig);
let profiler: Profiler;
try {
profiler = await createProfiler(config);
} catch (e) {
logError(`${e}`, config);
return;
}

// Set up periodic logging.
const logger = new Logger({
level: Logger.DEFAULT_OPTIONS.levels[normalizedConfig.logLevel],
level: Logger.DEFAULT_OPTIONS.levels[profiler.config.logLevel],
tag: pjson.name
});
let heapProfileCount = 0;
Expand Down Expand Up @@ -189,7 +227,7 @@ export async function startLocal(config: Config = {}): Promise<void> {
heapProfileCount = 0;
timeProfileCount = 0;
prevLogTime = curTime;
}, normalizedConfig.localLogPeriodMillis);
}, profiler.config.localLogPeriodMillis);

// Periodic profiling
setInterval(async () => {
Expand All @@ -198,16 +236,16 @@ export async function startLocal(config: Config = {}): Promise<void> {
{name: 'Heap-Profile' + new Date(), profileType: 'HEAP'});
heapProfileCount++;
}
await delay(normalizedConfig.localProfilingPeriodMillis / 2);
await delay(profiler.config.localProfilingPeriodMillis / 2);
if (!config.disableTime) {
const wall = await profiler.profile({
name: 'Time-Profile' + new Date(),
profileType: 'WALL',
duration: normalizedConfig.localTimeDurationMillis.toString() + 'ms'
duration: profiler.config.localTimeDurationMillis.toString() + 'ms'
});
timeProfileCount++;
}
}, normalizedConfig.localProfilingPeriodMillis);
}, profiler.config.localProfilingPeriodMillis);
}

// If the module was --require'd from the command line, start the agent.
Expand Down
17 changes: 8 additions & 9 deletions ts/src/profiler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import * as zlib from 'zlib';
import {perftools} from '../../proto/profile';

import {ProfilerConfig} from './config';
import {HeapProfiler} from './profilers/heap-profiler';
import * as heapProfiler from './profilers/heap-profiler';
import {TimeProfiler} from './profilers/time-profiler';

const parseDuration: (str: string) => number = require('parse-duration');
Expand Down Expand Up @@ -204,10 +204,12 @@ function responseToProfileOrError(

/**
* Polls profiler server for instructions on behalf of a task and
* collects and uploads profiles as requested
* collects and uploads profiles as requested.
*
* If heap profiling is enabled, the heap profiler must be enabled before heap
* profiles can be collected.
*/
export class Profiler extends ServiceObject {
private config: ProfilerConfig;
private logger: Logger;
private profileLabels: {instance?: string};
private deployment: Deployment;
Expand All @@ -216,7 +218,7 @@ export class Profiler extends ServiceObject {

// Public for testing.
timeProfiler: TimeProfiler|undefined;
heapProfiler: HeapProfiler|undefined;
config: ProfilerConfig;

constructor(config: ProfilerConfig) {
config = util.normalizeArguments(null, config) as ProfilerConfig;
Expand Down Expand Up @@ -258,10 +260,7 @@ export class Profiler extends ServiceObject {
}
if (!this.config.disableHeap) {
this.profileTypes.push(ProfileTypes.Heap);
this.heapProfiler = new HeapProfiler(
this.config.heapIntervalBytes, this.config.heapMaxStackDepth);
}

this.retryer = new Retryer(
this.config.initialBackoffMillis, this.config.backoffCapMillis,
this.config.backoffMultiplier);
Expand Down Expand Up @@ -458,10 +457,10 @@ export class Profiler extends ServiceObject {
* Public to allow for testing.
*/
async writeHeapProfile(prof: RequestProfile): Promise<RequestProfile> {
if (!this.heapProfiler) {
if (this.config.disableHeap) {
throw Error('Cannot collect heap profile, heap profiler not enabled.');
}
const p = this.heapProfiler.profile();
const p = heapProfiler.profile();
prof.profileBytes = await profileBytes(p);
return prof;
}
Expand Down
63 changes: 37 additions & 26 deletions ts/src/profilers/heap-profiler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,52 @@
*/

import {perftools} from '../../../proto/profile';
import {defaultConfig} from '../config';

import {serializeHeapProfile} from './profile-serializer';

const profiler = require('bindings')('sampling_heap_profiler');

export class HeapProfiler {
private enabled = false;

/**
* @param intervalBytes - average bytes between samples.
* @param stackDepth - upper limit on number of frames in stack for sample.
*/
constructor(private intervalBytes: number, private stackDepth: number) {
this.enable();
}
let enabled = false;
let heapIntervalBytes = 0;
let heapStackDepth = 0;

/**
* Collects a heap profile when heapProfiler is enabled. Otherwise throws
* an error.
*/
profile(): perftools.profiles.IProfile {
if (!this.enabled) {
throw new Error('Heap profiler is not enabled.');
}
const result = profiler.getAllocationProfile();
const startTimeNanos = Date.now() * 1000 * 1000;
return serializeHeapProfile(result, startTimeNanos, this.intervalBytes);
/*
* Collects a heap profile when heapProfiler is enabled. Otherwise throws
* an error.
*/
export function profile(): perftools.profiles.IProfile {
if (!enabled) {
throw new Error('Heap profiler is not enabled.');
}
const startTimeNanos = Date.now() * 1000 * 1000;
const result = profiler.getAllocationProfile();
return serializeHeapProfile(result, startTimeNanos, heapIntervalBytes);
}

enable() {
profiler.startSamplingHeapProfiler(this.intervalBytes, this.stackDepth);
this.enabled = true;
/**
* Starts heap profiling. If heap profiling has already been started with
* the same parameters, this is a noop. If heap profiler has already been
* started with different parameters, this throws an error.
*
* @param intervalBytes - average number of bytes between samples.
* @param stackDepth - maximum stack depth for samples collected.
*/
export function start(intervalBytes: number, stackDepth: number) {
if (enabled) {
throw new Error(`Heap profiler is already started with intervalBytes ${
heapIntervalBytes} and stackDepth ${stackDepth}`);
}
heapIntervalBytes = intervalBytes;
heapStackDepth = stackDepth;
profiler.startSamplingHeapProfiler(heapIntervalBytes, heapStackDepth);
enabled = true;
}

disable() {
this.enabled = false;
// Stops heap profiling. If heap profiling has not been started, does nothing.
export function stop() {
if (enabled) {
enabled = false;
profiler.stopSamplingHeapProfiler();
}
}
Loading

0 comments on commit 33cd531

Please sign in to comment.