Merge pull request #459 from AikidoSec/esbuild

Add end2end test for apps bundled with esbuild

docs/esbuild.md

@@ -0,0 +1,35 @@
+# Installing Zen in a Node.js Application Bundled with esbuild
+
+Note: Zen runs only on the server side, it does not run in the browser.
+
+Note: If `bundle` is set to `false` in the esbuild configuration, Zen will work without any additional configuration.
+
+Modify your esbuild configuration to include the external option using this utility:
+
+```js
+const { build } = require("esbuild");
+const { externals } = require("@aikidosec/firewall/bundler"); // <-- Add this line
+
+build({
+  entryPoints: ["./app.js"],
+  bundle: true,
+  platform: "node",
+  target: "node18",
+  outfile: "./dist/app.js",
+  external: externals(), // <-- Add this line
+});
+```
+
+This tells esbuild to exclude @aikidosec/firewall and any packages that Zen hooks into from the bundle.
+
+⚠️ Don't forget to copy the node_modules directory to the output directory.
+
+## Why do I need to do this?
+
+Zen works by intercepting `require()` calls that a Node.js application makes when loading modules. This includes modules that are built-in to Node.js, like the `fs` module for accessing the filesystem, as well as modules installed from the NPM registry, like the `pg` database module.
+
+Bundlers like esbuild crawl all of the `require()` calls that an application makes to files on disk. It replaces the `require()` calls with custom code and combines all the resulting JavaScript into one "bundled" file. When a built-in module is loaded, such as `require('fs')`, that call can then remain the same in the resulting bundle.
+
+Zen can continue to intercept the calls for built-in modules but cannot intercept calls to third party libraries under those conditions. This means that when you bundle a Zen app with a bundler Zen is likely to capture information about disk access (through `fs`) and outbound HTTP requests (through `http`), but omit calls to third party libraries.
+
+The solution is to treat all third party modules that Zen needs to instrument as being "external" to the bundler. With this setting the instrumented modules remain on disk and continue to be loaded with `require()` while the non-instrumented modules are bundled.

end2end/tests/express-postgres.test.js

@@ -1,153 +1,169 @@
 const t = require("tap");
-const { spawn } = require("child_process");
+const { spawn, spawnSync } = require("child_process");
 const { resolve } = require("path");
 const timeout = require("../timeout");
 
-const pathToApp = resolve(
-  __dirname,
-  "../../sample-apps/express-postgres",
-  "app.js"
-);
+const directory = resolve(__dirname, "../../sample-apps/express-postgres");
 
-t.test("it blocks in blocking mode", (t) => {
-  const server = spawn(`node`, ["--preserve-symlinks", pathToApp, "4000"], {
-    env: { ...process.env, AIKIDO_DEBUG: "true", AIKIDO_BLOCKING: "true" },
-  });
+const entrypoints = ["app.js", "compiled.js"];
 
-  server.on("close", () => {
-    t.end();
+t.before(() => {
+  const { stderr } = spawnSync("node", ["esbuild.js"], {
+    cwd: directory,
   });
 
-  server.on("error", (err) => {
-    t.fail(err.message);
-  });
+  if (stderr && stderr.toString().length > 0) {
+    throw new Error(`Failed to build: ${stderr.toString()}`);
+  }
+});
 
-  let stdout = "";
-  server.stdout.on("data", (data) => {
-    stdout += data.toString();
-  });
+entrypoints.forEach((entrypoint) => {
+  t.test(`it blocks in blocking mode (${entrypoint})`, (t) => {
+    const server = spawn(`node`, ["--preserve-symlinks", entrypoint, "4000"], {
+      env: { ...process.env, AIKIDO_DEBUG: "true", AIKIDO_BLOCKING: "true" },
+      cwd: directory,
+    });
 
-  let stderr = "";
-  server.stderr.on("data", (data) => {
-    stderr += data.toString();
-  });
+    server.on("close", () => {
+      t.end();
+    });
 
-  // Wait for the server to start
-  timeout(2000)
-    .then(() => {
-      return Promise.all([
-        fetch(
-          `http://localhost:4000/?petname=${encodeURIComponent("Njuska'); DELETE FROM cats_2;-- H")}`,
-          {
+    server.on("error", (err) => {
+      t.fail(err);
+    });
+
+    let stdout = "";
+    server.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
+
+    let stderr = "";
+    server.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    // Wait for the server to start
+    timeout(2000)
+      .then(() => {
+        return Promise.all([
+          fetch(
+            `http://localhost:4000/?petname=${encodeURIComponent("Njuska'); DELETE FROM cats_2;-- H")}`,
+            {
+              signal: AbortSignal.timeout(5000),
+            }
+          ),
+          fetch(`http://localhost:4000/string-concat`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ petname: ["'", "1)", "(0,1)", "(1", "'"] }),
             signal: AbortSignal.timeout(5000),
-          }
-        ),
-        fetch(`http://localhost:4000/string-concat`, {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ petname: ["'", "1)", "(0,1)", "(1", "'"] }),
-          signal: AbortSignal.timeout(5000),
-        }),
-        fetch(
-          `http://localhost:4000/string-concat?petname='&petname=1)&petname=(0,1)&petname=(1&petname='`,
-          {
+          }),
+          fetch(
+            `http://localhost:4000/string-concat?petname='&petname=1)&petname=(0,1)&petname=(1&petname='`,
+            {
+              signal: AbortSignal.timeout(5000),
+            }
+          ),
+          fetch("http://localhost:4000/?petname=Njuska", {
             signal: AbortSignal.timeout(5000),
-          }
-        ),
-        fetch("http://localhost:4000/?petname=Njuska", {
-          signal: AbortSignal.timeout(5000),
-        }),
-      ]);
-    })
-    .then(
-      async ([sqlInjection, sqlInjection2, sqlInjection3, normalSearch]) => {
-        t.equal(sqlInjection.status, 500);
-        t.equal(sqlInjection2.status, 500);
-        t.equal(sqlInjection3.status, 500);
-        t.equal(normalSearch.status, 200);
-        t.match(stdout, /Starting agent/);
-        t.match(stderr, /Zen has blocked an SQL injection/);
-      }
-    )
-    .catch((error) => {
-      t.fail(error.message);
-    })
-    .finally(() => {
-      server.kill();
+          }),
+        ]);
+      })
+      .then(
+        async ([sqlInjection, sqlInjection2, sqlInjection3, normalSearch]) => {
+          t.equal(sqlInjection.status, 500);
+          t.equal(sqlInjection2.status, 500);
+          t.equal(sqlInjection3.status, 500);
+          t.equal(normalSearch.status, 200);
+          t.match(stdout, /Starting agent/);
+          t.match(stderr, /Zen has blocked an SQL injection/);
+        }
+      )
+      .catch((error) => {
+        t.fail(error);
+      })
+      .finally(() => {
+        server.kill();
+      });
+  });
+
+  t.test(`it does not block in dry mode (${entrypoint})`, (t) => {
+    const server = spawn(`node`, ["--preserve-symlinks", entrypoint, "4001"], {
+      env: { ...process.env, AIKIDO_DEBUG: "true" },
+      cwd: directory,
     });
-});
 
-t.test("it does not block in dry mode", (t) => {
-  const server = spawn(`node`, ["--preserve-symlinks", pathToApp, "4001"], {
-    env: { ...process.env, AIKIDO_DEBUG: "true" },
-  });
+    server.on("close", () => {
+      t.end();
+    });
 
-  server.on("close", () => {
-    t.end();
-  });
+    server.on("error", (err) => {
+      t.fail(err);
+    });
 
-  let stdout = "";
-  server.stdout.on("data", (data) => {
-    stdout += data.toString();
-  });
+    let stdout = "";
+    server.stdout.on("data", (data) => {
+      stdout += data.toString();
+    });
 
-  let stderr = "";
-  server.stderr.on("data", (data) => {
-    stderr += data.toString();
-  });
+    let stderr = "";
+    server.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
 
-  // Wait for the server to start
-  timeout(2000)
-    .then(() =>
-      Promise.all([
-        fetch(
-          `http://localhost:4001/?petname=${encodeURIComponent("Njuska'); DELETE FROM cats_2;-- H")}`,
-          {
+    // Wait for the server to start
+    timeout(2000)
+      .then(() =>
+        Promise.all([
+          fetch(
+            `http://localhost:4001/?petname=${encodeURIComponent("Njuska'); DELETE FROM cats_2;-- H")}`,
+            {
+              signal: AbortSignal.timeout(5000),
+            }
+          ),
+          fetch(`http://localhost:4001/string-concat`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ petname: ["'", "1)", "(0,1)", "(1", "'"] }),
             signal: AbortSignal.timeout(5000),
-          }
-        ),
-        fetch(`http://localhost:4001/string-concat`, {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ petname: ["'", "1)", "(0,1)", "(1", "'"] }),
-          signal: AbortSignal.timeout(5000),
-        }),
-        fetch(
-          `http://localhost:4001/string-concat?petname='&petname=1)&petname=(0,1)&petname=(1&petname='`,
-          {
+          }),
+          fetch(
+            `http://localhost:4001/string-concat?petname='&petname=1)&petname=(0,1)&petname=(1&petname='`,
+            {
+              signal: AbortSignal.timeout(5000),
+            }
+          ),
+          fetch("http://localhost:4001/?petname=Njuska", {
             signal: AbortSignal.timeout(5000),
-          }
-        ),
-        fetch("http://localhost:4001/?petname=Njuska", {
-          signal: AbortSignal.timeout(5000),
-        }),
-      ])
-    )
-    .then(
-      async ([sqlInjection, sqlInjection2, sqlInjection3, normalSearch]) => {
-        t.equal(sqlInjection.status, 200);
-        t.equal(sqlInjection2.status, 200);
-        t.equal(sqlInjection3.status, 200);
-        t.equal(normalSearch.status, 200);
-        t.match(stdout, /Starting agent/);
-        t.notMatch(stderr, /Zen has blocked an SQL injection/);
-      }
-    )
-    .catch((error) => {
-      t.fail(error.message);
-    })
-    .finally(() => {
-      server.kill();
-    });
+          }),
+        ])
+      )
+      .then(
+        async ([sqlInjection, sqlInjection2, sqlInjection3, normalSearch]) => {
+          t.equal(sqlInjection.status, 200);
+          t.equal(sqlInjection2.status, 200);
+          t.equal(sqlInjection3.status, 200);
+          t.equal(normalSearch.status, 200);
+          t.match(stdout, /Starting agent/);
+          t.notMatch(stderr, /Zen has blocked an SQL injection/);
+        }
+      )
+      .catch((error) => {
+        t.fail(error);
+      })
+      .finally(() => {
+        server.kill();
+      });
+  });
 });
 
 t.test("it blocks in blocking mode (with dd-trace)", (t) => {
   const server = spawn(
     `node`,
-    ["--preserve-symlinks", "--require", "dd-trace/init", pathToApp, "4002"],
+    ["--preserve-symlinks", "--require", "dd-trace/init", "app.js", "4002"],
     {
       env: { ...process.env, AIKIDO_DEBUG: "true", AIKIDO_BLOCKING: "true" },
-      cwd: resolve(__dirname, "../../sample-apps/express-postgres"),
+      cwd: directory,
     }
   );
 

library/agent/protect.ts

@@ -105,7 +105,7 @@ function getAgent({ serverless }: { serverless: string | undefined }) {
   return agent;
 }
 
-function getWrappers() {
+export function getWrappers() {
   return [
     new Express(),
     new MongoDB(),

library/bundler/externals.test.ts

@@ -0,0 +1,8 @@
+import * as t from "tap";
+import { externals } from "./externals";
+
+t.test("it returns externals", async (t) => {
+  t.ok(externals().includes("@aikidosec/firewall"));
+  t.ok(externals().includes("pg"));
+  t.ok(externals().includes("mysql"));
+});

library/bundler/externals.ts

@@ -0,0 +1,18 @@
+import { Hooks } from "../agent/hooks/Hooks";
+import { getWrappers } from "../agent/protect";
+
+export function externals() {
+  const wrappers = getWrappers();
+  const hooks = new Hooks();
+
+  wrappers.forEach((wrapper) => {
+    wrapper.wrap(hooks);
+  });
+
+  const packages = ["@aikidosec/firewall"].concat(
+    hooks.getPackages().map((pkg) => pkg.getName())
+  );
+
+  // Remove duplicates
+  return Array.from(new Set(packages));
+}

library/bundler/index.ts

@@ -0,0 +1,4 @@
+import { externals } from "./externals";
+
+// eslint-disable-next-line import/no-unused-modules
+export { externals };

sample-apps/express-postgres/.gitignore

@@ -0,0 +1 @@
+/compiled.js

sample-apps/express-postgres/esbuild.js

@@ -0,0 +1,11 @@
+const { build } = require("esbuild");
+const { externals } = require("@aikidosec/firewall/bundler");
+
+build({
+  entryPoints: ["./app.js"],
+  bundle: true,
+  platform: "node",
+  target: "node18",
+  outfile: "./compiled.js",
+  external: externals(),
+});