module: support requiring .mjs files

This implements the ability to use require on .mjs files, loaded
via the esm loader, using the same tradeoffs that top level await
makes in esm itself.

What this means: If possible, all execution and evaluation is done
synchronously, via immediately unwrapping the execution's component
promises. This means that any and all existing code should have no
observable change in behavior, as there exist no asynchronous modules as
of yet. The catch is that once a module which requires asynchronous
execution is used, it must yield to the event loop to perform that
execution, which, in turn, can allow other code to execute before the
continuation after the async action, which is observable to callers of
the now asynchronous module. If this matters to your callers, this means
making your module execution asynchronous could be considered a breaking
change to your library, however in practice, it will not matter for most
callers. Moreover, as the ecosystem exists today, there are zero
asynchronously executing modules, and so until there are, there are no
downsides to this approach at all, as no execution is changed from what
one would expect today (excepting, ofc, that it's no longer an error to
require("./foo.mjs").

Ref: nodejs/modules#308
Ref: https://github.com/nodejs/modules/issues/299
Ref: nodejs/modules#454

lib/internal/modules/cjs/loader.js

@@ -86,6 +86,7 @@ const {
   ERR_REQUIRE_ESM
 } = require('internal/errors').codes;
 const { validateString } = require('internal/validators');
+const { promiseWait } = internalBinding('task_queue');
 const pendingDeprecation = getOptionValue('--pending-deprecation');
 
 module.exports = {
@@ -1038,36 +1039,34 @@ Module.prototype.load = function(filename) {
   this.paths = Module._nodeModulePaths(path.dirname(filename));
 
   const extension = findLongestRegisteredExtension(filename);
-  // allow .mjs to be overridden
-  if (filename.endsWith('.mjs') && !Module._extensions['.mjs']) {
-    throw new ERR_REQUIRE_ESM(filename);
-  }
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
-  const ESMLoader = asyncESM.ESMLoader;
-  const url = `${pathToFileURL(filename)}`;
-  const module = ESMLoader.moduleMap.get(url);
-  // Create module entry at load time to snapshot exports correctly
-  const exports = this.exports;
-  // Called from cjs translator
-  if (module !== undefined && module.module !== undefined) {
-    if (module.module.getStatus() >= kInstantiated)
-      module.module.setExport('default', exports);
-  } else {
-    // Preemptively cache
-    // We use a function to defer promise creation for async hooks.
-    ESMLoader.moduleMap.set(
-      url,
-      // Module job creation will start promises.
-      // We make it a function to lazily trigger those promises
-      // for async hooks compatibility.
-      () => new ModuleJob(ESMLoader, url, () =>
-        new ModuleWrap(url, undefined, ['default'], function() {
-          this.setExport('default', exports);
-        })
-      , false /* isMain */, false /* inspectBrk */)
-    );
+  if (extension !== '.mjs') {
+    const ESMLoader = asyncESM.ESMLoader;
+    const url = `${pathToFileURL(filename)}`;
+    const module = ESMLoader.moduleMap.get(url);
+    // Create module entry at load time to snapshot exports correctly
+    const exports = this.exports;
+    // Called from cjs translator
+    if (module !== undefined && module.module !== undefined) {
+      if (module.module.getStatus() >= kInstantiated)
+        module.module.setExport('default', exports);
+    } else {
+      // Preemptively cache
+      // We use a function to defer promise creation for async hooks.
+      ESMLoader.moduleMap.set(
+        url,
+        // Module job creation will start promises.
+        // We make it a function to lazily trigger those promises
+        // for async hooks compatibility.
+        () => new ModuleJob(ESMLoader, url, () =>
+          new ModuleWrap(url, undefined, ['default'], function() {
+            this.setExport('default', exports);
+          })
+        , false /* isMain */, false /* inspectBrk */)
+      );
+    }
   }
 };
 
@@ -1246,6 +1245,18 @@ Module._extensions['.node'] = function(module, filename) {
   return process.dlopen(module, path.toNamespacedPath(filename));
 };
 
+Module._extensions['.mjs'] = function(module, filename) {
+  const ESMLoader = asyncESM.ESMLoader;
+  const url = `${pathToFileURL(filename)}`;
+  const job = ESMLoader.getModuleJobWorker(
+    url,
+    'module'
+  );
+  const instantiated = promiseWait(job.instantiate()); // so long as builtin esm resolve is sync, this will complete sync (if the loader is extensible, an async loader will be have an observable effect here)
+  module.exports = instantiated.getNamespace();
+  promiseWait(instantiated.evaluate(-1, false)); // so long as the module doesn't contain TLA, this will be sync, otherwise it appears async
+}
+
 function createRequireFromPath(filename) {
   // Allow a directory to be passed as the filename
   const trailingSlash =

lib/internal/modules/esm/loader.js

@@ -152,8 +152,7 @@ class Loader {
     }
   }
 
-  async getModuleJob(specifier, parentURL) {
-    const { url, format } = await this.resolve(specifier, parentURL);
+  getModuleJobWorker(url, format, parentURL) {
     let job = this.moduleMap.get(url);
     // CommonJS will set functions for lazy job evaluation.
     if (typeof job === 'function')
@@ -188,6 +187,11 @@ class Loader {
     this.moduleMap.set(url, job);
     return job;
   }
+
+  async getModuleJob(specifier, parentURL) {
+    const { url, format } = await this.resolve(specifier, parentURL);
+    return this.getModuleJobWorker(url, format, parentURL);
+  }
 }
 
 ObjectSetPrototypeOf(Loader.prototype, null);

src/node_task_queue.cc

@@ -119,6 +119,45 @@ static void SetPromiseRejectCallback(
   env->set_promise_reject_callback(args[0].As<Function>());
 }
 
+/**
+ * Immediately unwraps a promise into a return value or throw, if possible
+ * If not, runs the event loop and microtask queue until it is unwrapable. 
+ */
+static void PromiseWait(const FunctionCallbackInfo<Value>& args) {
+  if (!args[0]->IsPromise()) {
+    args.GetReturnValue().Set(args[0]);
+    return;
+  }
+  v8::Local<v8::Promise> promise = args[0].As<v8::Promise>();
+  if (promise->State() == v8::Promise::kFulfilled) {
+    args.GetReturnValue().Set(promise->Result());
+    return;
+  }
+  Isolate* isolate = args.GetIsolate();
+  if (promise->State() == v8::Promise::kRejected) {
+    isolate->ThrowException(promise->Result());
+    return;
+  }
+
+  Environment* env = Environment::GetCurrent(args);
+
+  uv_loop_t* loop = env->event_loop();
+  int state = promise->State();
+  while (state == v8::Promise::kPending) {
+    isolate->RunMicrotasks();
+    if (uv_loop_alive(loop) && promise->State() == v8::Promise::kPending) {
+      uv_run(loop, UV_RUN_ONCE);
+    }
+    state = promise->State();
+  }
+
+  if (promise->State() == v8::Promise::kRejected) {
+    isolate->ThrowException(promise->Result());
+    return;
+  }
+  args.GetReturnValue().Set(promise->Result());
+}
+
 static void Initialize(Local<Object> target,
                        Local<Value> unused,
                        Local<Context> context,
@@ -145,6 +184,8 @@ static void Initialize(Local<Object> target,
   env->SetMethod(target,
                  "setPromiseRejectCallback",
                  SetPromiseRejectCallback);
+
+  env->SetMethod(target, "promiseWait", PromiseWait);
 }
 
 }  // namespace task_queue