Merge pull request #12 from bigbluebutton/develop

chore: update from develop (1.3.0)

config/settings.json.template

@@ -41,5 +41,13 @@
         "from-etherpad-redis-channel"
       ]
     }
+  },
+  "prometheus": {
+    "enabled": false,
+    "host": "localhost",
+    "port": 9003,
+    "path": "/metrics",
+    "collectCustomMetrics": true,
+    "collectDefaultMetrics": true
   }
 }

docs/prometheus.md

@@ -0,0 +1,51 @@
+# Prometheus metrics
+
+bbb-pads provides direct Prometheus instrumentation for monitoring purposes.
+The instrumentation code is **disabled by default**.
+
+The underlying mechanisms of the Prometheus client as well as the default Node.js
+metrics come from https://github.com/siimon/prom-client.
+
+## Enabling instrumentation
+
+It can be enabled via configuration file (settings.json).
+
+### Configuration file (settings.json)
+
+See the `prometheus` object in `/config/settings.json.template`.
+
+The default configuration is:
+
+```JSON5
+"prometheus": {
+  // Whether to enable or disable metric exporting altogether.
+  "enabled": false,
+  // host: scrape route host
+  "host": "localhost",
+  // port: scrape route port
+  "port": "9003",
+  // path: metrics endpoint path
+  "path": "/metrics",
+  // collectCustomMetrics: whether custom bbb-pads metrics should be exported
+  "collectCustomMetrics": true
+  // collectDefaultMetrics: whether default Node.js metrics should be exported
+  "collectDefaultMetrics": true
+}
+```
+
+## Exposed metrics
+
+The custom metric set currently is:
+
+```
+# HELP bbb_pads_etherpad_requests_total Total Etherpad API requests
+# TYPE bbb_pads_etherpad_requests_total counter
+bbb_pads_etherpad_requests_total{method="<method_name>"} 0
+
+# HELP bbb_pads_etherpad_requests_errors_total Total Etherpad API request failures
+# TYPE bbb_pads_etherpad_requests_errors_total counter
+bbb_pads_etherpad_requests_errors_total{method="<method_name>"} 0
+
+```
+
+The default Node.js metrics come from https://github.com/siimon/prom-client.

index.js

@@ -3,6 +3,7 @@ const subscriber = require('./lib/redis/subscriber');
 const monitor = require('./lib/utils/monitor');
 const server = require('./lib/express/server');
 const Logger = require('./lib/utils/logger');
+const prometheus = require('./lib/utils/prometheus');
 
 const logger = new Logger('bbb-pads');
 
@@ -30,6 +31,7 @@ const start = () => {
       subscriber.start();
       server.start();
       monitor.start();
+      prometheus.start();
     }).catch(() => abort('key-mismatch'));
   }).catch((error) => {
     logger.warn('starting', error);

lib/etherpad/api.js

@@ -5,6 +5,10 @@ const {
 } = require('./methods');
 const config = require('../../config');
 const Logger = require('../utils/logger');
+const {
+  registerAPIError,
+  registerAPICall,
+} = require('../utils/prometheus');
 
 const logger = new Logger('api');
 
@@ -41,7 +45,13 @@ const locked = (id) => {
 
 const call = (method, params = {}) => {
   return new Promise((resolve, reject) => {
-    if (!validate(method, params)) return reject();
+    registerAPICall(method);
+
+    if (!validate(method, params)) {
+      registerAPIError(method);
+
+      return reject();
+    }
 
     const id = buildId(method, params);
     if (locked(id)) return reject();
@@ -55,6 +65,7 @@ const call = (method, params = {}) => {
       const { status } = response;
       if (status !== 200) {
         logger.error('call', { status });
+        registerAPIError(method);
 
         return reject();
       }
@@ -67,14 +78,18 @@ const call = (method, params = {}) => {
 
       if (code !== 0) {
         logger.error('call', { message });
+        registerAPIError(method);
 
         return reject();
       }
 
       logger.debug('call', { method, data });
 
       resolve(data);
-    }).catch(() => reject()).finally(() => release(id));
+    }).catch(() => {
+      registerAPIError(method);
+      reject();
+    }).finally(() => release(id));
   });
 };
 

lib/utils/http-server.js

@@ -0,0 +1,43 @@
+"use strict";
+
+const http = require("http");
+const Logger = require('./logger.js');
+
+const logger = new Logger('http-server');
+
+module.exports = class HttpServer {
+  constructor(host, port, callback) {
+    this.host = host;
+    this.port = port;
+    this.requestCallback = callback;
+  }
+
+  start () {
+    this.server = http.createServer(this.requestCallback)
+      .on('error', this.handleError.bind(this))
+      .on('clientError', this.handleError.bind(this));
+  }
+
+  close (callback) {
+    return this.server.close(callback);
+  }
+
+  handleError (error) {
+    if (error.code === 'EADDRINUSE') {
+      logger.error('EADDRINUSE', { host: this.host, port: this.port });
+      this.server.close();
+    } else if (error.code === 'ECONNRESET') {
+      Logger.warn('ECONNRESET');
+    } else {
+      Logger.error('failure', { errorMessage: error.message, errorCode: error.code });
+    }
+  }
+
+  getServerObject() {
+    return this.server;
+  }
+
+  listen(callback) {
+    this.server.listen(this.port, this.host, callback);
+  }
+}

lib/utils/prometheus/index.js

@@ -0,0 +1,77 @@
+const config = require('../../../config');
+const PrometheusAgent = require('./prometheus-agent.js');
+const { Counter } = require('prom-client');
+const Logger = require('../logger.js');
+
+const logger = new Logger('prometheus');
+const { prometheus = {} } = config;
+const PREFIX = 'bbb_pads_';
+const PROM_NAMES = {
+  ETH_REQS_TOTAL: `${PREFIX}etherpad_requests_total`,
+  ETH_REQS_ERRORS: `${PREFIX}etherpad_requests_errors_total`,
+}
+const {
+  enabled: PROM_ENABLED = false,
+  host: PROM_HOST = 'localhost',
+  port: PROM_PORT = '9003',
+  path: PROM_PATH = '/metrics',
+  collectDefaultMetrics: COLLECT_DEFAULT_METRICS = false,
+  collectCustomMetrics: COLLECT_CUSTOM_METRICS = false,
+} = prometheus;
+const PADSPrometheusAgent = new PrometheusAgent(PROM_HOST, PROM_PORT, {
+  path: PROM_PATH,
+  prefix: PREFIX,
+  collectDefaultMetrics: COLLECT_DEFAULT_METRICS,
+});
+
+let PADS_METRICS;
+const _buildDefaultMetrics = () => {
+  if (PADS_METRICS == null) {
+    PADS_METRICS = {
+      [PROM_NAMES.ETH_REQS_TOTAL]: new Counter({
+        name: PROM_NAMES.ETH_REQS_TOTAL,
+        help: 'Total Etherpad API requests',
+        labelNames: ['method'],
+      }),
+      [PROM_NAMES.ETH_REQS_ERRORS]: new Counter({
+        name: PROM_NAMES.ETH_REQS_ERRORS,
+        help: 'Total Etherpad API request failures',
+        labelNames: ['method'],
+      }),
+    }
+  }
+
+  return PADS_METRICS;
+};
+
+const start = () => {
+  if (PROM_ENABLED) {
+    try {
+      if (COLLECT_CUSTOM_METRICS) {
+        PADSPrometheusAgent.injectMetrics(_buildDefaultMetrics());
+      }
+
+      PADSPrometheusAgent.start();
+    } catch (error) {
+      logger.error('prometheus-startup', {
+        errorCode: error.code, errorMessage: error.message
+      });
+    }
+  }
+}
+
+const registerAPIError = (method) => {
+  if (method == null) return;
+  PADSPrometheusAgent.increment(PROM_NAMES.ETH_REQS_ERRORS, { method });
+};
+
+const registerAPICall = (method) => {
+  if (method == null) return;
+  PADSPrometheusAgent.increment(PROM_NAMES.ETH_REQS_TOTAL, { method });
+}
+
+module.exports = {
+  start,
+  registerAPIError,
+  registerAPICall,
+};

lib/utils/prometheus/prometheus-agent.js

@@ -0,0 +1,113 @@
+const {
+  register,
+  collectDefaultMetrics,
+} = require('prom-client');
+const HTTPServer = require('../http-server.js');
+const Logger = require('../logger.js');
+
+const logger = new Logger('prometheus');
+
+module.exports = class PrometheusScrapeAgent {
+  constructor (host, port, options) {
+    this.host = host;
+    this.port = port;
+    this.metrics = {};
+    this.started = false;
+
+    this.path = options.path || '/metrics';
+    this.collectDefaultMetrics = options.collectDefaultMetrics || false;
+    this.metricsPrefix = options.prefix || '';
+    this.collectionTimeout = options.collectionTimeout || 10000;
+  }
+
+  getMetric (name) {
+    return this.metrics[name];
+  }
+
+  async collect (response) {
+    try {
+      response.writeHead(200, { 'Content-Type': register.contentType });
+      const content = await register.metrics();
+      response.end(content);
+    } catch (error) {
+      response.writeHead(500)
+      response.end(error.message);
+      logger.error('collecting-metrics', {
+        errorCode: error.code, errorMessage: error.message
+      });
+    }
+  }
+
+  getMetricsHandler (request, response) {
+    switch (request.method) {
+      case 'GET':
+        if (request.url === this.path) return this.collect(response);
+        response.writeHead(404).end();
+        break;
+      default:
+        response.writeHead(501)
+        response.end();
+        break;
+    }
+  }
+
+  start (requestHandler = this.getMetricsHandler.bind(this)) {
+    if (this.collectDefaultMetrics) collectDefaultMetrics({
+      prefix: this.metricsPrefix,
+      timeout: this.collectionTimeout,
+    });
+
+    this.metricsServer = new HTTPServer(this.host, this.port, requestHandler);
+    this.metricsServer.start();
+    this.metricsServer.listen();
+    this.started = true;
+  }
+
+  injectMetrics (metricsDictionary) {
+    this.metrics = { ...this.metrics, ...metricsDictionary }
+  }
+
+  increment (metricName, labelsObject) {
+    if (!this.started) return;
+
+    const metric = this.metrics[metricName];
+    if (metric) {
+      metric.inc(labelsObject)
+    }
+  }
+
+  decrement (metricName, labelsObject) {
+    if (!this.started) return;
+
+    const metric = this.metrics[metricName];
+    if (metric) {
+      metric.dec(labelsObject)
+    }
+  }
+
+  set (metricName, value, labelsObject = {}) {
+    if (!this.started) return;
+
+    const metric = this.metrics[metricName];
+    if (metric) {
+      metric.set(labelsObject, value)
+    }
+  }
+
+  setCollectorWithGenerator (metricName, generator) {
+    const metric = this.getMetric(metricName);
+    if (metric) {
+      metric.collect = () => {
+        metric.set(generator());
+      };
+    }
+  }
+
+  setCollector (metricName, collector) {
+    const metric = this.getMetric(metricName);
+
+    if (metric) {
+      metric.collect = collector.bind(metric);
+    }
+  }
+}