node compatibility typescript infrastructure (#258)

build/wd_api_bundle.bzl

@@ -0,0 +1,103 @@
+load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
+
+CAPNP_TEMPLATE = """@{schema_id};
+
+using Modules = import "/workerd/jsg/modules.capnp";
+
+const {const_name} :Modules.Bundle = (
+  modules = [
+{modules}
+]);
+"""
+
+MODULE_TEMPLATE = """(name = "{name}", src = embed "{path}", internal = {internal})"""
+
+def _relative_path(file_path, dir_path):
+    if not file_path.startswith(dir_path):
+        fail("file_path need to start with dir_path: " + file_path + " vs " + dir_path)
+    return file_path.removeprefix(dir_path)
+
+def _gen_api_bundle_capnpn_impl(ctx):
+    output_dir = ctx.outputs.out.dirname + "/"
+
+    def _render_module(name, label, internal):
+        return MODULE_TEMPLATE.format(
+            name = name,
+            # capnp doesn't allow ".." dir escape, make paths relative.
+            # this won't work for embedding paths outside of rule directory subtree.
+            path = _relative_path(
+                ctx.expand_location("$(location {})".format(label), ctx.attr.data),
+                output_dir,
+            ),
+            internal = "true" if internal else "false",
+        )
+
+    modules = [
+        _render_module(ctx.attr.builtin_modules[m], m.label, False)
+        for m in ctx.attr.builtin_modules
+    ]
+    modules += [
+        _render_module(ctx.attr.internal_modules[m], m.label, True)
+        for m in ctx.attr.internal_modules
+    ]
+
+    content = CAPNP_TEMPLATE.format(
+        schema_id = ctx.attr.schema_id,
+        modules = ",\n".join(modules),
+        const_name = ctx.attr.const_name,
+    )
+    ctx.actions.write(ctx.outputs.out, content)
+
+gen_api_bundle_capnpn = rule(
+    implementation = _gen_api_bundle_capnpn_impl,
+    attrs = {
+        "schema_id": attr.string(mandatory = True),
+        "out": attr.output(mandatory = True),
+        "builtin_modules": attr.label_keyed_string_dict(allow_files = True),
+        "internal_modules": attr.label_keyed_string_dict(allow_files = True),
+        "data": attr.label_list(allow_files = True),
+        "const_name": attr.string(mandatory = True),
+    },
+)
+
+def wd_api_bundle(
+        name,
+        schema_id,
+        const_name,
+        builtin_modules = {},
+        internal_modules = {},
+        **kwargs):
+    """Generate cc capnp library with api bundle.
+
+    NOTE: Due to capnpc embed limitation all modules must be in the same or sub directory of the
+          actual rule usage.
+
+    Args:
+     name: cc_capnp_library rule name
+     builtin_modules: js src label -> module name dictionary
+     internal_modules: js src label -> module name dictionary
+     const_name: capnp constant name that will contain bundle definition
+     schema_id: capnpn schema id
+     **kwargs: rest of cc_capnp_library arguments
+    """
+    data = list(builtin_modules) + list(internal_modules)
+
+    gen_api_bundle_capnpn(
+        name = name + "@gen",
+        out = name + ".capnp",
+        schema_id = schema_id,
+        const_name = const_name,
+        builtin_modules = builtin_modules,
+        internal_modules = internal_modules,
+        data = data,
+    )
+
+    cc_capnp_library(
+        name = name,
+        srcs = [name + ".capnp"],
+        strip_include_prefix = "",
+        visibility = ["//visibility:public"],
+        data = data,
+        deps = ["//src/workerd/jsg:modules_capnp"],
+        **kwargs
+    )

src/node/BUILD.bazel

@@ -0,0 +1,35 @@
+load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
+load("@//:build/wd_api_bundle.bzl", "wd_api_bundle")
+
+modules = glob(["*.ts"])
+
+internal_modules = glob(["internal/*.ts"])
+
+ts_config(
+    name = "node@tsconfig",
+    src = ":tsconfig.json",
+)
+
+ts_project(
+    name = "node",
+    srcs = modules + internal_modules,
+    tsconfig = "node@tsconfig",
+)
+
+wd_api_bundle(
+    name = "bundle",
+    # builtin modules are accessible under "node:<module_name>" name
+    builtin_modules = dict([(
+        m.removesuffix(".ts") + ".js",
+        "node:" + m.removesuffix(".ts"),
+    ) for m in modules]),
+    const_name = "nodeBundle",
+    include_prefix = "node",
+    # internal modules are accessible under "node-internal:<module_name>" name without "internal/"
+    # folder prefix.
+    internal_modules = dict([(
+        m.removesuffix(".ts") + ".js",
+        "node-internal:" + m.removeprefix("internal/").removesuffix(".ts"),
+    ) for m in internal_modules]),
+    schema_id = "0xbcc8f57c63814005",
+)

src/node/README.md

@@ -0,0 +1 @@
+# Node Compatibility Layer

src/node/buffer.ts

@@ -0,0 +1,10 @@
+// NOTE: this file is a temporary placeholder to test ts/workerd integration.
+// It will be rewritten/replaced with a real one eventually.
+
+import * as bufferImpl from 'node-internal:bufferImpl';
+
+export class Buffer {
+  public toString(): string {
+    return bufferImpl.toString(this);
+  }
+}

src/node/internal/bufferImpl.ts

@@ -0,0 +1,8 @@
+// NOTE: this file is a temporary placeholder to test ts/workerd integration.
+// It will be rewritten/replaced with a real one eventually.
+
+import * as buffer from "node:buffer";
+
+export function toString(buf: buffer.Buffer): string {
+  return `Buffer[${buf}]`;
+}

src/node/tsconfig.json

@@ -0,0 +1,32 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "lib": [ "ESNext" ],
+    "alwaysStrict": true,
+    "strict": true,
+    "allowUnreachableCode": false,
+    "allowUnusedLabels": false,
+    "exactOptionalPropertyTypes": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitOverride": true,
+    "noImplicitReturns": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noUncheckedIndexedAccess": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "strictNullChecks": true,
+    "types": [
+      // todo: consume generated workerd types
+      // "@cloudflare/workers-types"
+    ],
+    "paths": {
+      "node:*": ["./*"],
+      "node-internal:*": ["./internal/*"],
+    }
+  },
+  "include": [
+    "*.ts",
+    "internal/*.ts"
+  ],
+}

src/workerd/jsg/BUILD.bazel

@@ -16,9 +16,10 @@ wd_cc_library(
     ),
     visibility = ["//visibility:public"],
     deps = [
+        ":modules_capnp",
+        "//src/workerd/util",
         "@capnp-cpp//src/kj",
         "@workerd-v8//:v8",
-        "//src/workerd/util",
     ],
 )
 
@@ -37,10 +38,10 @@ js_capnp_library(
 npm_package(
     name = "jsg_js",
     srcs = [":rtti_capnp_js"],
-    visibility = ["//visibility:public"],
     # Required to ensure source files are copied when running internal builds
     # that depend on `workerd` as an external repository
     include_external_repositories = ["workerd"],
+    visibility = ["//visibility:public"],
 )
 
 wd_cc_library(
@@ -55,6 +56,12 @@ wd_cc_library(
     ],
 )
 
+wd_cc_capnp_library(
+    name = "modules_capnp",
+    srcs = ["modules.capnp"],
+    visibility = ["//visibility:public"],
+)
+
 [kj_test(
     src = f,
     deps = [

src/workerd/jsg/modules.capnp

@@ -0,0 +1,21 @@
+@0xc8cbb234694939d5;
+
+using Cxx = import "/capnp/c++.capnp";
+$Cxx.namespace("workerd::jsg");
+
+struct Bundle {
+  # Group of modules to be loaded together.
+  # Bundles are currently generated during compilation process and linked with the workerd,
+  # but loading bundles from somewhere else will also be possible.
+  modules @0 :List(Module);
+}
+
+struct Module {
+  # Javascript module with its source code.
+
+  name @0 :Text;
+  src @1 :Data;
+
+  internal @2 :Bool;
+  # internal modules can't be imported by user's code
+}

src/workerd/jsg/modules.h

@@ -7,6 +7,7 @@
 #include "jsg.h"
 #include <kj/filesystem.h>
 #include <kj/map.h>
+#include <workerd/jsg/modules.capnp.h>
 
 namespace workerd::jsg {
 
@@ -238,6 +239,14 @@ class ModuleRegistryImpl final: public ModuleRegistry {
     entries.insert(Entry(specifier, Type::BUNDLE, kj::fwd<ModuleInfo>(info)));
   }
 
+  void addBuiltinBundle(Bundle::Reader bundle) {
+    for (auto module: bundle.getModules()) {
+      // TODO: asChars() might be wrong for wide characters
+      addBuiltinModule(module.getName(), module.getSrc().asChars(),
+          module.getInternal() ? Type::INTERNAL : Type::BUILTIN);
+    }
+  }
+
   void addBuiltinModule(kj::StringPtr specifier,
                         kj::ArrayPtr<const char> sourceCode,
                         Type type = Type::BUILTIN) {

src/workerd/server/BUILD.bazel

@@ -30,6 +30,7 @@ wd_cc_library(
     deps = [
         ":workerd_capnp",
         "@capnp-cpp//src/capnp:capnpc",
+        "//src/node:bundle",
         "//src/workerd/io",
         "//src/workerd/jsg",
     ],

src/workerd/server/workerd-api.c++

@@ -23,6 +23,8 @@
 #include <openssl/hmac.h>
 #include <openssl/rand.h>
 
+#include <node/bundle.capnp.h>
+
 namespace workerd::server {
 
 JSG_DECLARE_ISOLATE_TYPE(JsgWorkerdIsolate,
@@ -232,6 +234,7 @@ kj::Own<jsg::ModuleRegistry> WorkerdApiIsolate::compileModules(
     Worker::ValidationErrorReporter& errorReporter) const {
   auto& lock = kj::downcast<JsgWorkerdIsolate::Lock>(lockParam);
   v8::HandleScope scope(lock.v8Isolate);
+  auto& featureFlags = *impl->features;
 
   auto modules = kj::heap<jsg::ModuleRegistryImpl<JsgWorkerdIsolate_TypeWrapper>>();
 
@@ -312,6 +315,10 @@ kj::Own<jsg::ModuleRegistry> WorkerdApiIsolate::compileModules(
     }
   }
 
+  if (featureFlags.getNodeJsCompat()) {
+    modules->addBuiltinBundle(NODE_BUNDLE);
+  }
+
   jsg::setModulesForResolveCallback<JsgWorkerdIsolate_TypeWrapper>(lock, modules);
 
   return modules;