From c8af31547d0a5f19e9e904035a95c00e68ef8c48 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Wed, 13 Jul 2022 09:12:20 +0200 Subject: [PATCH] Remove usage of a WebWorker/worker thread (#2498) * Remove liveness pings * Remove "current task" system * Rename instance.ts -> raw-instance.ts * Turn worker.ts into a simple object * Remove the utilities that deal with the worker * Provide the configuration when starting and not asychronously * Use a Promise for state rather than an array * Use queueOperation instead of handleMessage * Turn generic handleMessage into individual functions * Make worker.addChain return a Promise * addChain is now properly async * Make removeChain, request, and databaseContent always synchronous * Track the active chains within worker.ts * Fix state wasn't being set properly * Make databaseContent return a Promise * Provide logCallback in config and finish removing messages * Remove mention of the worker * Remove unused node:worker_threads import * Finish removing the ToWorker messages * Get rid of messages module * Add some console.asserts * No longer track chains in client.ts * Move the number handling of databaseContent to worker.ts * Rename worker to instance * Pass onProcExit rather than throwing directly * Add onPanic to smoldot bindings * Clarify panic handling situation * Finally properly handle crashes * Properly format instance.ts * Remove obsolete code * Implement clean shutdown of the instance * Fix removeChain and databaseContent still callable * Document start_shutdown * Clarify alreadyDestroyedError * CHANGELOG * Fix startShutdown handling of errors * Spellcheck * Rename * More rename Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- bin/wasm-node/CHANGELOG.md | 4 + bin/wasm-node/javascript/README.md | 19 -- bin/wasm-node/javascript/package.json | 3 +- bin/wasm-node/javascript/prepare.js | 10 +- bin/wasm-node/javascript/src/client.ts | 306 ++++-------------- .../src/compat/index-browser-overwrite.js | 32 -- .../javascript/src/compat/index.d.ts | 14 - bin/wasm-node/javascript/src/compat/index.js | 21 -- .../{worker => instance}/autogen/.gitignore | 0 .../bindings-smoldot-light.ts | 81 ++++- .../src/{worker => instance}/bindings-wasi.ts | 9 +- .../src/{worker => instance}/bindings.ts | 1 + .../src/{worker => instance}/connection.ts | 0 .../javascript/src/instance/instance.ts | 305 +++++++++++++++++ .../instance.ts => instance/raw-instance.ts} | 33 +- .../javascript/src/worker/messages.ts | 127 -------- .../src/worker/spawn-browser-overwrite.js | 39 --- .../javascript/src/worker/spawn.d.ts | 20 -- bin/wasm-node/javascript/src/worker/spawn.js | 32 -- bin/wasm-node/javascript/src/worker/worker.ts | 196 ----------- bin/wasm-node/rust/src/bindings.rs | 12 + bin/wasm-node/rust/src/lib.rs | 5 + 22 files changed, 497 insertions(+), 772 deletions(-) rename bin/wasm-node/javascript/src/{worker => instance}/autogen/.gitignore (100%) rename bin/wasm-node/javascript/src/{worker => instance}/bindings-smoldot-light.ts (77%) rename bin/wasm-node/javascript/src/{worker => instance}/bindings-wasi.ts (96%) rename bin/wasm-node/javascript/src/{worker => instance}/bindings.ts (98%) rename bin/wasm-node/javascript/src/{worker => instance}/connection.ts (100%) create mode 100644 bin/wasm-node/javascript/src/instance/instance.ts rename bin/wasm-node/javascript/src/{worker/instance.ts => instance/raw-instance.ts} (73%) delete mode 100644 bin/wasm-node/javascript/src/worker/messages.ts delete mode 100644 bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js delete mode 100644 bin/wasm-node/javascript/src/worker/spawn.d.ts delete mode 100644 bin/wasm-node/javascript/src/worker/spawn.js delete mode 100644 bin/wasm-node/javascript/src/worker/worker.ts diff --git a/bin/wasm-node/CHANGELOG.md b/bin/wasm-node/CHANGELOG.md index da8f195b4f..015645f2ca 100644 --- a/bin/wasm-node/CHANGELOG.md +++ b/bin/wasm-node/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Changed + +- No WebWorker/worker thread is spawned anymore by the JavaScript code. The WebAssembly virtual machine that runs smoldot is now directly instantiated by the `start` function. This should fix compatibility issues with various JavaScript bundlers. ([#2498](https://github.com/paritytech/smoldot/pull/2498)) + ## 0.6.23 - 2022-07-11 ### Fixed diff --git a/bin/wasm-node/javascript/README.md b/bin/wasm-node/javascript/README.md index 3c211dd6f4..c9df517ff5 100644 --- a/bin/wasm-node/javascript/README.md +++ b/bin/wasm-node/javascript/README.md @@ -73,22 +73,3 @@ chains must be passed as parameter to `addChain` as well. In situations where th specifications passed to `addChain` are not trusted, it is important for security reasons to not establish a parachain-relay-chain link between two chains that aren't part of the same "trust sandbox". - -# About the worker - -The code in this package uses a web worker (in browsers) or a worker thread (on NodeJS). The -line of JavaScript that creates the worker is of the following form: - -``` js -new Worker(new URL('./worker.js', import.meta.url), { type: "module" }); -``` - -This format is compatible [with Webpack 5](https://webpack.js.org/guides/web-workers/), meaning -that Webpack will be able to resolve the imports in `worker.js` and adjust this little snippet. - -This format also works in NodeJS without any issue. - -However, at the time of writing of this comment, this format doesn't work with Parcel (both 1 and -2) due to various bugs. - -As a general warning, be aware of the fact that this line might cause issues if you use a bundler. diff --git a/bin/wasm-node/javascript/package.json b/bin/wasm-node/javascript/package.json index 8408cfc547..7c29e47b25 100644 --- a/bin/wasm-node/javascript/package.json +++ b/bin/wasm-node/javascript/package.json @@ -23,8 +23,7 @@ "test": "node prepare.js --debug && rimraf ./dist && tsc && ava --timeout=2m --concurrency 2 --no-worker-threads" }, "browser": { - "./dist/compat/index.js": "./dist/compat/index-browser-overwrite.js", - "./dist/worker/spawn.js": "./dist/worker/spawn-browser-overwrite.js" + "./dist/compat/index.js": "./dist/compat/index-browser-overwrite.js" }, "dependencies": { "buffer": "^6.0.1", diff --git a/bin/wasm-node/javascript/prepare.js b/bin/wasm-node/javascript/prepare.js index f410aa2cf5..42acf56d49 100755 --- a/bin/wasm-node/javascript/prepare.js +++ b/bin/wasm-node/javascript/prepare.js @@ -69,13 +69,13 @@ child_process.execSync( { 'stdio': 'inherit' } ); -// The code below will write a variable number of files to the `src/worker/autogen` directory. +// The code below will write a variable number of files to the `src/instance/autogen` directory. // Start by clearing all existing files from this directory in case there are some left from past // builds. -const filesToRemove = fs.readdirSync('./src/worker/autogen'); +const filesToRemove = fs.readdirSync('./src/instance/autogen'); for (const file of filesToRemove) { if (!file.startsWith('.')) // Don't want to remove the `.gitignore` or `.npmignore` or similar - fs.unlinkSync(path.join("./src/worker/autogen", file)); + fs.unlinkSync(path.join("./src/instance/autogen", file)); } // We then do an optimization pass on the Wasm file, using `wasm-opt`. @@ -119,13 +119,13 @@ try { const chunk = base64Data.slice(0, 1024 * 1024); // We could simply export the chunk instead of a function that returns the chunk, but that // would cause TypeScript to generate a definitions file containing a copy of the entire chunk. - fs.writeFileSync('./src/worker/autogen/wasm' + fileNum + '.ts', 'export default function(): string { return "' + chunk + '"; }'); + fs.writeFileSync('./src/instance/autogen/wasm' + fileNum + '.ts', 'export default function(): string { return "' + chunk + '"; }'); imports += 'import { default as wasm' + fileNum + ' } from \'./wasm' + fileNum + '.js\';\n'; chunksSum += ' + wasm' + fileNum + '()'; fileNum += 1; base64Data = base64Data.slice(1024 * 1024); } - fs.writeFileSync('./src/worker/autogen/wasm.ts', imports + 'export default ' + chunksSum + ';'); + fs.writeFileSync('./src/instance/autogen/wasm.ts', imports + 'export default ' + chunksSum + ';'); } finally { fs.rmSync(tmpDir, { recursive: true }); diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 6da0617b8e..fdfa774c3d 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -15,9 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -import { CompatWorker, workerOnMessage, workerOnError, workerTerminate } from './compat/index.js'; -import * as messages from './worker/messages.js'; -import spawnWorker from './worker/spawn.js'; +import { start as startInstance } from './instance/instance.js'; /** * Thrown in case of a problem when initializing the chain. @@ -365,207 +363,20 @@ export function start(options?: ClientOptions): Client { } }); - // The actual execution of Smoldot is performed in a worker thread. - // Because this specific line of code is a bit sensitive, it is done in a separate file. - const worker = spawnWorker(); - let workerError: null | Error = null; - - // Whenever an `addChain` or `removeChain` message is sent to the worker, a corresponding entry - // is pushed to this array. The worker needs to send back a confirmation, which pops the first - // element of this array. In the case of `addChain`, additional fields are stored in this array - // to finish the initialization of the chain. - let pendingConfirmations: PendingConfirmation[] = []; - - // Contains the information of each chain that is currently. - // Entries are instantly removed when the user desires to remove a chain even before the worker - // has confirmed the removal. Doing so avoids a race condition where the worker sends back a - // database content or a JSON-RPC response/notification even though we've already sent a - // `removeChain` message to it. - // - // This map is also used in general as a way to check whether a chain still exists. - let chains: Map = new Map(); - // For each chain object returned by `addChain`, the associated internal chain id. // // Immediately cleared when `remove()` is called on a chain. let chainIds: WeakMap = new WeakMap(); - // The worker periodically reports the name of the task it is currently in. This makes it - // possible, when the worker is frozen, to know which task it was in when frozen. - const workerCurrentTask: { name: string | null } = { name: null }; - - // The worker periodically sends a message of kind 'livenessPing' in order to notify that it is - // still alive. - // If this liveness ping isn't received for a long time, an error is reported in the logs. - // The first check is delayed in order to account for the fact that the worker has to perform - // an expensive initialization step when initializing the Wasm VM. - let livenessTimeout: null | ReturnType = null; - const resetLivenessTimeout = () => { - if (livenessTimeout !== null) - globalThis.clearTimeout(livenessTimeout); - livenessTimeout = globalThis.setTimeout(() => { - livenessTimeout = null; - if (workerError) - return; // The unresponsiveness is due to a crash. No need to print more warnings. - console.warn( - "Smoldot appears unresponsive" + - (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + - ". Please open an issue at https://github.com/paritytech/smoldot/issues. If you have a " + - "debugger available, please pause execution, generate a stack trace of the thread " + - "that isn't the main execution thread, and paste it in the issue. Please also include " + - "any other log found in the console or elsewhere." - ); - }, 10000); - }; - globalThis.setTimeout(() => resetLivenessTimeout(), 15000); - - // The worker can send us messages whose type is identified through a `kind` field. - workerOnMessage(worker, (message: messages.FromWorker): void => { - switch (message.kind) { - case 'jsonrpc': { - const cb = chains.get(message.chainId)?.jsonRpcCallback; - if (cb) cb(message.data); - break; - } - - case 'chainAddedOk': { - const expected = pendingConfirmations.shift()!; - const chainId = message.chainId; - - if (chains.has(chainId)) // Sanity check. - throw 'Unexpected reuse of a chain ID'; - chains.set(chainId, { - jsonRpcCallback: expected.jsonRpcCallback, - databasePromises: new Array() - }); - - // `expected` was pushed by the `addChain` method. - // Resolve the promise that `addChain` returned to the user. - const newChain: Chain = { - sendJsonRpc: (request) => { - if (workerError) - throw workerError; - if (!chains.has(chainId)) - throw new AlreadyDestroyedError(); - if (!(chains.get(chainId)?.jsonRpcCallback)) - throw new JsonRpcDisabledError(); - if (request.length >= 8 * 1024 * 1024) - return; - postMessage(worker, { ty: 'request', request, chainId }); - }, - databaseContent: (maxUtf8BytesSize) => { - if (workerError) - return Promise.reject(workerError); - - const databaseContentPromises = chains.get(chainId)?.databasePromises; - if (!databaseContentPromises) - return Promise.reject(new AlreadyDestroyedError()); - - const promise: Promise = new Promise((resolve, reject) => { - databaseContentPromises.push({ resolve, reject }); - }); - - const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. - const maxSize = maxUtf8BytesSize || (twoPower32 - 1); - const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - - postMessage(worker, { ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); - - return promise; - }, - remove: () => { - if (workerError) - throw workerError; - // Because the `removeChain` message is asynchronous, it is possible for a JSON-RPC - // response or database content concerning that `chainId` to arrive after the `remove` - // function has returned. We solve that by removing the information immediately. - if (!chains.delete(chainId)) - throw new AlreadyDestroyedError(); - console.assert(chainIds.has(newChain)); - chainIds.delete(newChain); - postMessage(worker, { ty: 'removeChain', chainId }); - }, - }; - - chainIds.set(newChain, chainId); - expected.resolve(newChain); - break; - } - - case 'chainAddedErr': { - const expected = pendingConfirmations.shift()!; - // `expected` was pushed by the `addChain` method. - // Reject the promise that `addChain` returned to the user. - expected.reject(new AddChainError(message.error)); - break; - } - - case 'databaseContent': { - const promises = chains.get(message.chainId)?.databasePromises; - if (promises) (promises.shift() as DatabasePromise).resolve(message.data); - break; - } + // If `Client.terminate()̀ is called, this error is set to a value. + // All the functions of the public API check if this contains a value. + let alreadyDestroyedError: null | AlreadyDestroyedError = null; - case 'log': { - logCallback(message.level, message.target, message.message); - break; - } - - case 'livenessPing': { - resetLivenessTimeout(); - break; - } - - case 'currentTask': { - workerCurrentTask.name = message.taskName; - break; - } - - default: { - // Exhaustive check. - const _exhaustiveCheck: never = message; - return _exhaustiveCheck; - } - } - }); - - workerOnError(worker, (error) => { - // A worker error should only happen in case of a critical error as the result of a bug - // somewhere. Consequently, nothing is really in place to cleanly report the error. - const errorToString = error.toString(); - console.error( - "Smoldot has panicked" + - (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + - ". This is a bug in smoldot. Please open an issue at " + - "https://github.com/paritytech/smoldot/issues with the following message:\n" + - errorToString - ); - workerError = new CrashError(errorToString); - - // Reject all promises returned by `addChain`. - for (var pending of pendingConfirmations) { - if (pending.ty == 'chainAdded') - pending.reject(workerError); - } - pendingConfirmations = []; - - // Reject all promises for database contents. - for (const chain of chains) { - for (const promise of chain[1].databasePromises) { - promise.reject(workerError) - } - } - chains.clear(); - }); - - // The first message expected by the worker contains the configuration. - postMessage(worker, { + const instance = startInstance({ // Maximum level of log entries sent by the client. // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace maxLogLevel: options.maxLogLevel || 3, + logCallback, // `enableCurrentTask` adds a small performance hit, but adds some additional information to // crash reports. Whether this should be enabled is very opiniated and not that important. At // the moment, we enable it all the time, except if the user has logging disabled altogether. @@ -578,9 +389,9 @@ export function start(options?: ClientOptions): Client { }); return { - addChain: (options: AddChainOptions): Promise => { - if (workerError) - throw workerError; + addChain: async (options: AddChainOptions): Promise => { + if (alreadyDestroyedError) + throw alreadyDestroyedError; // Passing a JSON object for the chain spec is an easy mistake, so we provide a more // readable error. @@ -599,58 +410,55 @@ export function start(options?: ClientOptions): Client { } } - // Build a promise that will be resolved or rejected after the chain has been added. - // TODO: because of https://github.com/microsoft/TypeScript/issues/11498 we need to define the callbacks as possibly null, and go through `unknown` - let chainAddedPromiseResolve; - let chainAddedPromiseReject; - const chainAddedPromise: Promise = new Promise((resolve, reject) => { - chainAddedPromiseResolve = resolve; - chainAddedPromiseReject = reject; - }); - - pendingConfirmations.push({ - ty: 'chainAdded', - reject: chainAddedPromiseReject as unknown as (error: AddChainError) => void, - resolve: chainAddedPromiseResolve as unknown as (c: Chain) => void, - jsonRpcCallback: options.jsonRpcCallback, - }); - - postMessage(worker, { - ty: 'addChain', - chainSpec: options.chainSpec, - databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", - potentialRelayChains: potentialRelayChainsIds, - jsonRpcRunning: !!options.jsonRpcCallback, - }); - - return chainAddedPromise; + const outcome = await instance.addChain(options.chainSpec, typeof options.databaseContent === 'string' ? options.databaseContent : "", potentialRelayChainsIds, options.jsonRpcCallback); + + if (!outcome.success) + throw new AddChainError(outcome.error); + + const chainId = outcome.chainId; + const wasDestroyed = { destroyed: false }; + + // `expected` was pushed by the `addChain` method. + // Resolve the promise that `addChain` returned to the user. + const newChain: Chain = { + sendJsonRpc: (request) => { + if (alreadyDestroyedError) + throw alreadyDestroyedError; + if (wasDestroyed.destroyed) + throw new AlreadyDestroyedError(); + if (!options.jsonRpcCallback) + throw new JsonRpcDisabledError(); + if (request.length >= 8 * 1024 * 1024) + return; + instance.request(request, chainId); + }, + databaseContent: (maxUtf8BytesSize) => { + if (alreadyDestroyedError) + return Promise.reject(alreadyDestroyedError); + if (wasDestroyed.destroyed) + throw new AlreadyDestroyedError(); + return instance.databaseContent(chainId, maxUtf8BytesSize); + }, + remove: () => { + if (alreadyDestroyedError) + throw alreadyDestroyedError; + if (wasDestroyed.destroyed) + throw new AlreadyDestroyedError(); + wasDestroyed.destroyed = true; + console.assert(chainIds.has(newChain)); + chainIds.delete(newChain); + instance.removeChain(chainId); + }, + }; + + chainIds.set(newChain, chainId); + return newChain; }, - terminate: () => { - if (workerError) - return Promise.reject(workerError) - workerError = new AlreadyDestroyedError(); - - if (livenessTimeout !== null) - globalThis.clearTimeout(livenessTimeout) - - return workerTerminate(worker) + terminate: async () => { + if (alreadyDestroyedError) + throw alreadyDestroyedError + alreadyDestroyedError = new AlreadyDestroyedError(); + instance.startShutdown() } } } - -// Separate function in order to enforce types. -function postMessage(worker: CompatWorker, message: messages.ToWorker) { - worker.postMessage(message) -} - -interface PendingConfirmation { - ty: 'chainAdded', - resolve: (c: Chain) => void, - reject: (error: AddChainError) => void, - jsonRpcCallback?: JsonRpcCallback, -} - -interface DatabasePromise { - resolve: (data: string) => void, - reject: (error: Error) => void, -} diff --git a/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js b/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js index 96844eb5d3..962cd65fc1 100644 --- a/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js +++ b/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js @@ -17,38 +17,6 @@ // Overrides `index.js` when in a browser. -export function workerOnMessage(worker, callback) { - worker.onmessage = (event) => callback(event.data) -} - -export function workerOnError(worker, callback) { - worker.onerror = (event) => { - // For reference: - // https://html.spec.whatwg.org/multipage/indices.html#event-error - // https://html.spec.whatwg.org/multipage/webappapis.html#errorevent - - // If `event.error` exists, then it will likely be an instance of `Error`. - // However, that's not guaranteed by the spec and it could also be - // something else. So, our second best try is the `event.message` - // property. Finally, our last resort is to create an Error from the event. - if (event.error instanceof Error) callback(event.error); - else callback(new Error(event.message || event)); - } -} - -export function workerTerminate(worker) { - worker.terminate(); - return Promise.resolve(); -} - -export function postMessage(msg) { - self.postMessage(msg); -} - -export function setOnMessage(callback) { - self.onmessage = (event) => callback(event.data); -} - export function performanceNow() { return performance.now() } diff --git a/bin/wasm-node/javascript/src/compat/index.d.ts b/bin/wasm-node/javascript/src/compat/index.d.ts index f8abcd1a8e..5c752022c1 100644 --- a/bin/wasm-node/javascript/src/compat/index.d.ts +++ b/bin/wasm-node/javascript/src/compat/index.d.ts @@ -22,20 +22,6 @@ import type { Socket as TcpSocket, NetConnectOpts } from 'node:net'; export type WasmModuleImports = WebAssembly.ModuleImports; -export class CompatWorker { - postMessage(value: any, transferList?: ReadonlyArray): void; -} - -export function workerOnMessage(worker: CompatWorker, callback: (value: any) => void): void; - -export function workerOnError(worker: CompatWorker, callback: (err: Error) => void): void; - -export function workerTerminate(worker: CompatWorker): Promise; - -export function postMessage(message: any): void; - -export function setOnMessage(callback: (message: any) => void): void; - export function performanceNow(): number; export function isTcpAvailable(): boolean; diff --git a/bin/wasm-node/javascript/src/compat/index.js b/bin/wasm-node/javascript/src/compat/index.js index e3d7aa13da..bc86e6a468 100644 --- a/bin/wasm-node/javascript/src/compat/index.js +++ b/bin/wasm-node/javascript/src/compat/index.js @@ -19,31 +19,10 @@ // // A rule in the `package.json` overrides it with `index-browser-override.js` when in a browser. -import { parentPort } from 'node:worker_threads'; import { hrtime } from 'node:process'; import { createConnection as nodeCreateConnection } from 'node:net'; import { randomFillSync } from 'node:crypto'; -export function workerOnMessage(worker, callback) { - worker.on('message', callback); -} - -export function workerOnError(worker, callback) { - worker.on('error', callback); -} - -export function workerTerminate(worker) { - return worker.terminate().then(() => { }); -} - -export function postMessage(msg) { - parentPort.postMessage(msg); -} - -export function setOnMessage(callback) { - parentPort.on('message', callback); -} - export function performanceNow() { const time = hrtime(); return ((time[0] * 1e3) + (time[1] / 1e6)); diff --git a/bin/wasm-node/javascript/src/worker/autogen/.gitignore b/bin/wasm-node/javascript/src/instance/autogen/.gitignore similarity index 100% rename from bin/wasm-node/javascript/src/worker/autogen/.gitignore rename to bin/wasm-node/javascript/src/instance/autogen/.gitignore diff --git a/bin/wasm-node/javascript/src/worker/bindings-smoldot-light.ts b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts similarity index 77% rename from bin/wasm-node/javascript/src/worker/bindings-smoldot-light.ts rename to bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts index 5feabf0134..7c3fae321a 100644 --- a/bin/wasm-node/javascript/src/worker/bindings-smoldot-light.ts +++ b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts @@ -27,6 +27,14 @@ import type { SmoldotWasmInstance } from './bindings.js'; export interface Config { instance?: SmoldotWasmInstance, + + /** + * Closure to call when the Wasm instance calls `panic`. + * + * This callback will always be invoked from within a binding called the Wasm instance. + */ + onPanic: (message: string) => never, + logCallback: (level: number, target: string, message: string) => void, jsonRpcCallback: (response: string, chainId: number) => void, databaseContentCallback: (data: string, chainId: number) => void, @@ -37,12 +45,25 @@ export interface Config { forbidWss: boolean, } -export default function (config: Config): compat.WasmModuleImports { +export default function (config: Config): { imports: compat.WasmModuleImports, killAll: () => void } { // Used below to store the list of all connections. // The indices within this array are chosen by the Rust code. let connections: Record = {}; - return { + // Object containing a boolean indicating whether the `killAll` function has been invoked by + // the user. + const killedTracked = { killed: false }; + + const killAll = () => { + killedTracked.killed = true; + // TODO: kill timers as well? + for (const connection in connections) { + connections[connection]!.close() + delete connections[connection] + } + }; + + const imports = { // Must exit with an error. A human-readable message can be found in the WebAssembly // memory in the given buffer. panic: (ptr: number, len: number) => { @@ -52,11 +73,13 @@ export default function (config: Config): compat.WasmModuleImports { len >>>= 0; const message = Buffer.from(instance.exports.memory.buffer).toString('utf8', ptr, ptr + len); - throw new Error(message); + config.onPanic(message); }, // Used by the Rust side to emit a JSON-RPC response or subscription notification. json_rpc_respond: (ptr: number, len: number, chainId: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -70,6 +93,8 @@ export default function (config: Config): compat.WasmModuleImports { // Used by the Rust side in response to asking for the database content of a chain. database_content_ready: (ptr: number, len: number, chainId: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -84,6 +109,8 @@ export default function (config: Config): compat.WasmModuleImports { // Used by the Rust side to emit a log entry. // See also the `max_log_level` parameter in the configuration. log: (level: number, targetPtr: number, targetLen: number, messagePtr: number, messageLen: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; targetPtr >>>= 0; @@ -108,6 +135,8 @@ export default function (config: Config): compat.WasmModuleImports { // Must call `timer_finished` after the given number of milliseconds has elapsed. start_timer: (id: number, ms: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; // In both NodeJS and browsers, if `setTimeout` is called with a value larger than @@ -122,11 +151,17 @@ export default function (config: Config): compat.WasmModuleImports { // with `1`) and wants you to use `setImmediate` instead. if (ms == 0 && typeof setImmediate === "function") { setImmediate(() => { - instance.exports.timer_finished(id); + if (killedTracked.killed) return; + try { + instance.exports.timer_finished(id); + } catch(_error) {} }) } else { setTimeout(() => { - instance.exports.timer_finished(id); + if (killedTracked.killed) return; + try { + instance.exports.timer_finished(id); + } catch(_error) {} }, ms) } }, @@ -145,6 +180,9 @@ export default function (config: Config): compat.WasmModuleImports { } try { + if (killedTracked.killed) + throw new Error("killAll invoked"); + const address = Buffer.from(instance.exports.memory.buffer) .toString('utf8', addrPtr, addrPtr + addrLen); @@ -155,18 +193,27 @@ export default function (config: Config): compat.WasmModuleImports { forbidNonLocalWs: config.forbidNonLocalWs, forbidWss: config.forbidWss, onOpen: () => { - instance.exports.connection_open_single_stream(connectionId); + if (killedTracked.killed) return; + try { + instance.exports.connection_open_single_stream(connectionId); + } catch(_error) {} }, onClose: (message: string) => { - const len = Buffer.byteLength(message, 'utf8'); - const ptr = instance.exports.alloc(len) >>> 0; - Buffer.from(instance.exports.memory.buffer).write(message, ptr); - instance.exports.connection_closed(connectionId, ptr, len); + if (killedTracked.killed) return; + try { + const len = Buffer.byteLength(message, 'utf8'); + const ptr = instance.exports.alloc(len) >>> 0; + Buffer.from(instance.exports.memory.buffer).write(message, ptr); + instance.exports.connection_closed(connectionId, ptr, len); + } catch(_error) {} }, onMessage: (message: Buffer) => { - const ptr = instance.exports.alloc(message.length) >>> 0; - message.copy(Buffer.from(instance.exports.memory.buffer), ptr); - instance.exports.stream_message(connectionId, 0, ptr, message.length); + if (killedTracked.killed) return; + try { + const ptr = instance.exports.alloc(message.length) >>> 0; + message.copy(Buffer.from(instance.exports.memory.buffer), ptr); + instance.exports.stream_message(connectionId, 0, ptr, message.length); + } catch(_error) {} } }); @@ -192,6 +239,7 @@ export default function (config: Config): compat.WasmModuleImports { // Must close and destroy the connection object. connection_close: (connectionId: number) => { + if (killedTracked.killed) return; const connection = connections[connectionId]!; connection.close(); delete connections[connectionId]; @@ -212,6 +260,8 @@ export default function (config: Config): compat.WasmModuleImports { // Must queue the data found in the WebAssembly memory at the given pointer. It is assumed // that this function is called only when the connection is in an open state. stream_send: (connectionId: number, _streamId: number, ptr: number, len: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -223,6 +273,8 @@ export default function (config: Config): compat.WasmModuleImports { }, current_task_entered: (ptr: number, len: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -234,8 +286,11 @@ export default function (config: Config): compat.WasmModuleImports { }, current_task_exit: () => { + if (killedTracked.killed) return; if (config.currentTaskCallback) config.currentTaskCallback(null); } }; + + return { imports, killAll } } diff --git a/bin/wasm-node/javascript/src/worker/bindings-wasi.ts b/bin/wasm-node/javascript/src/instance/bindings-wasi.ts similarity index 96% rename from bin/wasm-node/javascript/src/worker/bindings-wasi.ts rename to bin/wasm-node/javascript/src/instance/bindings-wasi.ts index 915435f501..17b6c89c4b 100644 --- a/bin/wasm-node/javascript/src/worker/bindings-wasi.ts +++ b/bin/wasm-node/javascript/src/instance/bindings-wasi.ts @@ -37,6 +37,13 @@ export interface Config { * Must never be modified after the bindings have been initialized. */ envVars: string[], + + /** + * Closure to call when the Wasm instance calls `proc_exit`. + * + * This callback will always be invoked from within a binding called the Wasm instance. + */ + onProcExit: (retCode: number) => never, } export default (config: Config): compat.WasmModuleImports => { @@ -130,7 +137,7 @@ export default (config: Config): compat.WasmModuleImports => { // Used by Rust in catastrophic situations, such as a double panic. proc_exit: (retCode: number) => { - throw new Error(`proc_exit called: ${retCode}`); + config.onProcExit(retCode) }, // Return the number of environment variables and the total size of all environment diff --git a/bin/wasm-node/javascript/src/worker/bindings.ts b/bin/wasm-node/javascript/src/instance/bindings.ts similarity index 98% rename from bin/wasm-node/javascript/src/worker/bindings.ts rename to bin/wasm-node/javascript/src/instance/bindings.ts index dfe54985e9..17b6b8435e 100644 --- a/bin/wasm-node/javascript/src/worker/bindings.ts +++ b/bin/wasm-node/javascript/src/instance/bindings.ts @@ -24,6 +24,7 @@ export interface SmoldotWasmExports extends WebAssembly.Exports { memory: WebAssembly.Memory, init: (maxLogLevel: number, enableCurrentTask: number, cpuRateLimit: number) => void, + start_shutdown: () => void, alloc: (len: number) => number, add_chain: (chainSpecPointer: number, chainSpecLen: number, databaseContentPointer: number, databaseContentLen: number, jsonRpcRunning: number, potentialRelayChainsPtr: number, potentialRelayChainsLen: number) => number; remove_chain: (chainId: number) => void, diff --git a/bin/wasm-node/javascript/src/worker/connection.ts b/bin/wasm-node/javascript/src/instance/connection.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/connection.ts rename to bin/wasm-node/javascript/src/instance/connection.ts diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts new file mode 100644 index 0000000000..b3d5d498ec --- /dev/null +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -0,0 +1,305 @@ +// Smoldot +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import { Buffer } from 'buffer'; +import * as instance from './raw-instance.js'; +import { SmoldotWasmInstance } from './bindings.js'; +import { CrashError } from '../client.js'; + +/** + * Contains the configuration of the instance. + */ +export interface Config { + logCallback: (level: number, target: string, message: string) => void + maxLogLevel: number; + enableCurrentTask: boolean; + cpuRateLimit: number, + forbidTcp: boolean; + forbidWs: boolean; + forbidNonLocalWs: boolean; + forbidWss: boolean; +} + +export interface Instance { + request: (request: string, chainId: number) => void + addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void) => Promise<{ success: true, chainId: number } | { success: false, error: string }> + removeChain: (chainId: number) => void + databaseContent: (chainId: number, maxUtf8BytesSize?: number) => Promise + startShutdown: () => void +} + +export function start(configMessage: Config): Instance { + + // This variable represents the state of the instance, and serves two different purposes: + // + // - At initialization, it is a Promise containing the Wasm VM is still initializing. + // - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. + // + let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; + + const crashError: { error?: CrashError } = {}; + + const currentTask: { name: string | null } = { name: null }; + + const printError = { printError: true } + + // Contains the information of each chain that is currently alive. + let chains: Map void, + databasePromises: DatabasePromise[], + }> = new Map(); + + // Start initialization of the Wasm VM. + const config: instance.Config = { + onWasmPanic: (message) => { + // TODO: consider obtaining a backtrace here + crashError.error = new CrashError(message); + if (!printError.printError) + return; + console.error( + "Smoldot has panicked" + + (currentTask.name ? (" while executing task `" + currentTask.name + "`") : "") + + ". This is a bug in smoldot. Please open an issue at " + + "https://github.com/paritytech/smoldot/issues with the following message:\n" + + message + ); + }, + logCallback: (level, target, message) => { + configMessage.logCallback(level, target, message) + }, + jsonRpcCallback: (data, chainId) => { + const cb = chains.get(chainId)?.jsonRpcCallback; + if (cb) cb(data); + }, + databaseContentCallback: (data, chainId) => { + const promises = chains.get(chainId)?.databasePromises!; + (promises.shift() as DatabasePromise).resolve(data); + }, + currentTaskCallback: (taskName) => { + currentTask.name = taskName + }, + cpuRateLimit: configMessage.cpuRateLimit, + forbidTcp: configMessage.forbidTcp, + forbidWs: configMessage.forbidWs, + forbidNonLocalWs: configMessage.forbidNonLocalWs, + forbidWss: configMessage.forbidWss, + }; + + state = { + initialized: false, promise: instance.startInstance(config).then((instance) => { + // `config.cpuRateLimit` is a floating point that should be between 0 and 1, while the value + // to pass as parameter must be between `0` and `2^32-1`. + // The few lines of code below should handle all possible values of `number`, including + // infinites and NaN. + let cpuRateLimit = Math.round(config.cpuRateLimit * 4294967295); // `2^32 - 1` + if (cpuRateLimit < 0) cpuRateLimit = 0; + if (cpuRateLimit > 4294967295) cpuRateLimit = 4294967295; + if (!Number.isFinite(cpuRateLimit)) cpuRateLimit = 4294967295; // User might have passed NaN + + // Smoldot requires an initial call to the `init` function in order to do its internal + // configuration. + instance.exports.init(configMessage.maxLogLevel, configMessage.enableCurrentTask ? 1 : 0, cpuRateLimit); + + state = { initialized: true, instance }; + return instance; + }) + }; + + async function queueOperation(operation: (instance: SmoldotWasmInstance) => T): Promise { + // What to do depends on the type of `state`. + // See the documentation of the `state` variable for information. + if (!state.initialized) { + // A message has been received while the Wasm VM is still initializing. Queue it for when + // initialization is over. + return state.promise.then((instance) => operation(instance)) + + } else { + // Everything is already initialized. Process the message synchronously. + return operation(state.instance) + } + } + + return { + request: (request: string, chainId: number) => { + // Because `request` is passed as parameter an identifier returned by `addChain`, it is + // always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + if (crashError.error) + throw crashError.error; + + try { + const len = Buffer.byteLength(request, 'utf8'); + const ptr = state.instance.exports.alloc(len) >>> 0; + Buffer.from(state.instance.exports.memory.buffer).write(request, ptr); + state.instance.exports.json_rpc_send(ptr, len, chainId); + } catch (_error) { + console.assert(crashError.error); + throw crashError.error + } + }, + + addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { + return queueOperation((instance) => { + if (crashError.error) + throw crashError.error; + + try { + // Write the chain specification into memory. + const chainSpecLen = Buffer.byteLength(chainSpec, 'utf8'); + const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; + Buffer.from(instance.exports.memory.buffer) + .write(chainSpec, chainSpecPtr); + + // Write the database content into memory. + const databaseContentLen = Buffer.byteLength(databaseContent, 'utf8'); + const databaseContentPtr = instance.exports.alloc(databaseContentLen) >>> 0; + Buffer.from(instance.exports.memory.buffer) + .write(databaseContent, databaseContentPtr); + + // Write the potential relay chains into memory. + const potentialRelayChainsLen = potentialRelayChains.length; + const potentialRelayChainsPtr = instance.exports.alloc(potentialRelayChainsLen * 4) >>> 0; + for (let idx = 0; idx < potentialRelayChains.length; ++idx) { + Buffer.from(instance.exports.memory.buffer) + .writeUInt32LE(potentialRelayChains[idx]!, potentialRelayChainsPtr + idx * 4); + } + + // `add_chain` unconditionally allocates a chain id. If an error occurs, however, this chain + // id will refer to an *erroneous* chain. `chain_is_ok` is used below to determine whether it + // has succeeeded or not. + // Note that `add_chain` properly de-allocates buffers even if it failed. + const chainId = instance.exports.add_chain( + chainSpecPtr, chainSpecLen, + databaseContentPtr, databaseContentLen, + !!jsonRpcCallback ? 1 : 0, + potentialRelayChainsPtr, potentialRelayChainsLen + ); + + if (instance.exports.chain_is_ok(chainId) != 0) { + console.assert(!chains.has(chainId)); + chains.set(chainId, { + jsonRpcCallback, + databasePromises: new Array() + }); + return { success: true, chainId }; + } else { + const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; + const errorMsgPtr = instance.exports.chain_error_ptr(chainId) >>> 0; + const errorMsg = Buffer.from(instance.exports.memory.buffer) + .toString('utf8', errorMsgPtr, errorMsgPtr + errorMsgLen); + instance.exports.remove_chain(chainId); + return { success: false, error: errorMsg }; + } + } catch (_error) { + console.assert(crashError.error); + throw crashError.error + } + }) + }, + + removeChain: (chainId: number) => { + // Because `removeChain` is passed as parameter an identifier returned by `addChain`, it is + // always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + if (crashError.error) + throw crashError.error; + + // Removing the chain synchronously avoids having to deal with race conditions such as a + // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. + // These kind of race conditions are already delt with within smoldot. + console.assert(chains.has(chainId)); + chains.delete(chainId); + try { + state.instance.exports.remove_chain(chainId); + } catch (_error) { + console.assert(crashError.error); + throw crashError.error + } + }, + + databaseContent: (chainId: number, maxUtf8BytesSize?: number): Promise => { + // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it + // is always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + + if (crashError.error) + throw crashError.error; + + console.assert(chains.has(chainId)); + const databaseContentPromises = chains.get(chainId)?.databasePromises!; + const promise: Promise = new Promise((resolve, reject) => { + databaseContentPromises.push({ resolve, reject }); + }); + + // Cap `maxUtf8BytesSize` and set a default value. + const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. + const maxSize = maxUtf8BytesSize || (twoPower32 - 1); + const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; + + // The value of `maxUtf8BytesSize` is guaranteed to always fit in 32 bits, in + // other words, that `maxUtf8BytesSize < (1 << 32)`. + // We need to perform a conversion in such a way that the the bits of the output of + // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. + // See ToInt32 here: https://tc39.es/ecma262/#sec-toint32 + // Note that the code below has been tested against example values. Please be very careful + // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI + // layer between JS and Rust, writing unit tests would be extremely complicated. + const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. + const converted = (cappedMaxSize >= twoPower31) ? + (cappedMaxSize - twoPower32) : cappedMaxSize; + + try { + state.instance.exports.database_content(chainId, converted); + return promise; + } catch (_error) { + console.assert(crashError.error); + throw crashError.error + } + }, + + startShutdown: () => { + return queueOperation((instance) => { + // `startShutdown` is a bit special in its handling of crashes. + // Shutting down will lead to `onWasmPanic` being called at some point, possibly during + // the call to `start_shutdown` itself. As such, we move into "don't print errors anymore" + // mode even before calling `start_shutdown`. + // + // Furthermore, if a crash happened in the past, there is no point in throwing an + // exception when the user wants the shutdown to happen. + if (crashError.error) + return; + try { + printError.printError = false + instance.exports.start_shutdown() + } catch (_error) { + } + }) + } + } + +} + +interface DatabasePromise { + resolve: (data: string) => void, + reject: (error: Error) => void, +} diff --git a/bin/wasm-node/javascript/src/worker/instance.ts b/bin/wasm-node/javascript/src/instance/raw-instance.ts similarity index 73% rename from bin/wasm-node/javascript/src/worker/instance.ts rename to bin/wasm-node/javascript/src/instance/raw-instance.ts index 36579a9107..a21487e1b0 100644 --- a/bin/wasm-node/javascript/src/worker/instance.ts +++ b/bin/wasm-node/javascript/src/instance/raw-instance.ts @@ -19,7 +19,7 @@ import pako from 'pako'; import { Buffer } from 'buffer'; -import { Config as SmoldotBindingsConfig, default as smolditLightBindingsBuilder } from './bindings-smoldot-light.js'; +import { Config as SmoldotBindingsConfig, default as smoldotLightBindingsBuilder } from './bindings-smoldot-light.js'; import { Config as WasiConfig, default as wasiBindingsBuilder } from './bindings-wasi.js'; import { default as wasmBase64 } from './autogen/wasm.js'; @@ -27,6 +27,18 @@ import { default as wasmBase64 } from './autogen/wasm.js'; import { SmoldotWasmInstance } from './bindings.js'; export interface Config { + /** + * Closure to call when the Wasm instance panics. + * + * This callback will always be invoked from within a binding called the Wasm instance. + * + * After this callback has been called, it is forbidden to invoke any function from the Wasm + * VM. + * + * If this callback is called while invoking a function from the Wasm VM, this function will + * throw a dummy exception. + */ + onWasmPanic: (message: string) => void, logCallback: (level: number, target: string, message: string) => void, jsonRpcCallback: (response: string, chainId: number) => void, databaseContentCallback: (data: string, chainId: number) => void, @@ -45,22 +57,39 @@ export async function startInstance(config: Config): Promise void; + // Used to bind with the smoldot-light bindings. See the `bindings-smoldot-light.js` file. const smoldotJsConfig: SmoldotBindingsConfig = { + onPanic: (message) => { + killAll(); + config.onWasmPanic(message); + throw new Error(); + }, ...config }; // Used to bind with the Wasi bindings. See the `bindings-wasi.js` file. const wasiConfig: WasiConfig = { envVars: [], + onProcExit: (retCode) => { + killAll(); + config.onWasmPanic(`proc_exit called: ${retCode}`) + throw new Error(); + } }; + const { imports: smoldotBindings, killAll: smoldotBindingsKillAll } = + smoldotLightBindingsBuilder(smoldotJsConfig); + + killAll = smoldotBindingsKillAll; + // Start the Wasm virtual machine. // The Rust code defines a list of imports that must be fulfilled by the environment. The second // parameter provides their implementations. const result = await WebAssembly.instantiate(wasmBytecode, { // The functions with the "smoldot" prefix are specific to smoldot. - "smoldot": smolditLightBindingsBuilder(smoldotJsConfig), + "smoldot": smoldotBindings, // As the Rust code is compiled for wasi, some more wasi-specific imports exist. "wasi_snapshot_preview1": wasiBindingsBuilder(wasiConfig), }); diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts deleted file mode 100644 index 5d895f3eb0..0000000000 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ /dev/null @@ -1,127 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -/** - * Message to the worker. - * - * The first ever message sent to the worker must be a `ToWorkerConfig`, then all subsequent - * messages must be `ToWorkerNonConfig`s. - */ -export type ToWorker = ToWorkerConfig | ToWorkerNonConfig; -export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorkerRemoveChain | ToWorkerDatabaseContent; - -/** - * Message that the worker can send to the outside. - */ -export type FromWorker = FromWorkerChainAddedOk | FromWorkerChainAddedError | FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent | FromWorkerLivenessPing | FromWorkerCurrentTask; - -/** - * Contains the initial configuration of the worker. - * - * This message is only ever sent once, and it is always the first ever message sent to the - * worker. - */ -export interface ToWorkerConfig { - maxLogLevel: number; - enableCurrentTask: boolean; - cpuRateLimit: number, - forbidTcp: boolean; - forbidWs: boolean; - forbidNonLocalWs: boolean; - forbidWss: boolean; -} - -/** - * Start a JSON-RPC request. - */ -export interface ToWorkerRpcRequest { - ty: 'request', - request: string, - chainId: number, -} - -/** - * Add a new chain. - * - * The worker must reply with either a `FromWorkerChainAddedOk` or a `FromWorkerChainAddedError`. - */ -export interface ToWorkerAddChain { - ty: 'addChain', - chainSpec: string, - databaseContent: string, - potentialRelayChains: number[], - jsonRpcRunning: boolean, -} - -/** - * Remove a chain. - * - * The worker must reply with a `FromWorkerChainRemoved`. - */ -export interface ToWorkerRemoveChain { - ty: 'removeChain', - chainId: number, -} - -/** - * Get the database content of a chain. - * - * The worker must reply with a `FromWorkerDatabaseContent`. - */ -export interface ToWorkerDatabaseContent { - ty: 'databaseContent', - chainId: number, - maxUtf8BytesSize: number, -} - -export interface FromWorkerChainAddedOk { - kind: 'chainAddedOk', - chainId: number, -} - -export interface FromWorkerChainAddedError { - kind: 'chainAddedErr', - error: string, -} - -export interface FromWorkerLog { - kind: 'log', - level: number, - target: string, - message: string, -} - -export interface FromWorkerJsonRpc { - kind: 'jsonrpc', - data: string, - chainId: number, -} - -export interface FromWorkerDatabaseContent { - kind: 'databaseContent', - data: string, - chainId: number, -} - -export interface FromWorkerLivenessPing { - kind: 'livenessPing', -} - -export interface FromWorkerCurrentTask { - kind: 'currentTask', - taskName: string | null, -} diff --git a/bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js b/bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js deleted file mode 100644 index 8020c208fb..0000000000 --- a/bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js +++ /dev/null @@ -1,39 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -export default function () { - if (!window.Worker) - throw new Error("Workers not available"); - - // The line of code below (`new Worker(...)`) is designed to hopefully work across all - // platforms and bundlers. - // Because this line is precisely recognized by bundlers, we extract it to a separate - // JavaScript file. - // See also the README.md for more context. - - // Note that, at the time of writing, Firefox doesn't support the `type: "module"` option. - // Because browsers don't fully support modules yet, this code is expected to be run through - // a bundler (e.g. WebPack) before being given to a browser, which will remove all usage of - // modules in the worker code. It is thus also the role of this bundler to tweak or remove - // the value of this `type` property to indicate to the browser that modules aren't in use. - // - // WebPack in particular does this, but it is unclear whether *all* bundlers do it. - // Whether bundlers actually do this or not, it is nonetheless more correct to indicate - // `type: "module"` and doing so doesn't have any drawback. - const worker = new Worker(new URL('./worker.js', import.meta.url), { name: "smoldot", type: "module" }); - return worker; -} diff --git a/bin/wasm-node/javascript/src/worker/spawn.d.ts b/bin/wasm-node/javascript/src/worker/spawn.d.ts deleted file mode 100644 index 670e46b531..0000000000 --- a/bin/wasm-node/javascript/src/worker/spawn.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -import type { CompatWorker } from '../compat/index.js'; - -export default function (): CompatWorker; diff --git a/bin/wasm-node/javascript/src/worker/spawn.js b/bin/wasm-node/javascript/src/worker/spawn.js deleted file mode 100644 index 6b7fa06533..0000000000 --- a/bin/wasm-node/javascript/src/worker/spawn.js +++ /dev/null @@ -1,32 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -import { Worker } from 'node:worker_threads'; - -export default function () { - // The line of code below (`new Worker(...)`) is designed to hopefully work across all - // platforms and bundlers. - // Because this line is precisely recognized by bundlers, we extract it to a separate - // JavaScript file. - // See also the README.md for more context. - - // Note that at the time of writing of this comment, NodeJS doesn't support the - // `type: "module"` option, as modules "just work". But we put it anyway because Deno throws - // an exception if this option isn't present. - const worker = new Worker(new URL('./worker.js', import.meta.url), { type: "module" }); - return worker; -} diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts deleted file mode 100644 index 2bd9243df8..0000000000 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ /dev/null @@ -1,196 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -import { Buffer } from 'buffer'; -import * as compat from './../compat/index.js'; -import * as instance from './instance.js'; -import * as messages from './messages.js'; -import { SmoldotWasmInstance } from './bindings.js'; - -// This variable represents the state of the worker, and serves three different purposes: -// -// - At initialization, it is set to `null`. -// - Once the first message, containing the configuration, has been received from the parent, it -// becomes an array filled with the messages that are received while the Wasm VM is still -// initializing. -// - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. -// -let state: null | messages.ToWorkerNonConfig[] | SmoldotWasmInstance = null; - -// Inject a message coming from `index.js` to a running Wasm VM. -function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorkerNonConfig): void { - switch (message.ty) { - case 'request': { - const len = Buffer.byteLength(message.request, 'utf8'); - const ptr = instance.exports.alloc(len) >>> 0; - Buffer.from(instance.exports.memory.buffer).write(message.request, ptr); - instance.exports.json_rpc_send(ptr, len, message.chainId); - break; - } - - case 'addChain': { - // Write the chain specification into memory. - const chainSpecLen = Buffer.byteLength(message.chainSpec, 'utf8'); - const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; - Buffer.from(instance.exports.memory.buffer) - .write(message.chainSpec, chainSpecPtr); - - // Write the database content into memory. - const databaseContentLen = Buffer.byteLength(message.databaseContent, 'utf8'); - const databaseContentPtr = instance.exports.alloc(databaseContentLen) >>> 0; - Buffer.from(instance.exports.memory.buffer) - .write(message.databaseContent, databaseContentPtr); - - // Write the potential relay chains into memory. - const potentialRelayChainsLen = message.potentialRelayChains.length; - const potentialRelayChainsPtr = instance.exports.alloc(potentialRelayChainsLen * 4) >>> 0; - for (let idx = 0; idx < message.potentialRelayChains.length; ++idx) { - Buffer.from(instance.exports.memory.buffer) - .writeUInt32LE(message.potentialRelayChains[idx]!, potentialRelayChainsPtr + idx * 4); - } - - // `add_chain` unconditionally allocates a chain id. If an error occurs, however, this chain - // id will refer to an *erroneous* chain. `chain_is_ok` is used below to determine whether it - // has succeeeded or not. - // Note that `add_chain` properly de-allocates buffers even if it failed. - const chainId = instance.exports.add_chain( - chainSpecPtr, chainSpecLen, - databaseContentPtr, databaseContentLen, - message.jsonRpcRunning ? 1 : 0, - potentialRelayChainsPtr, potentialRelayChainsLen - ); - - if (instance.exports.chain_is_ok(chainId) != 0) { - postMessage({ kind: 'chainAddedOk', chainId }); - } else { - const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; - const errorMsgPtr = instance.exports.chain_error_ptr(chainId) >>> 0; - const errorMsg = Buffer.from(instance.exports.memory.buffer) - .toString('utf8', errorMsgPtr, errorMsgPtr + errorMsgLen); - instance.exports.remove_chain(chainId); - postMessage({ kind: 'chainAddedErr', error: errorMsg }); - } - - break; - } - - case 'removeChain': { - instance.exports.remove_chain(message.chainId); - break; - } - - case 'databaseContent': { - // The value of `maxUtf8BytesSize` is guaranteed (by `index.js`) to always fit in 32 bits, in - // other words, that `maxUtf8BytesSize < (1 << 32)`. - // We need to perform a conversion in such a way that the the bits of the output of - // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. - // See ToInt32 here: https://tc39.es/ecma262/#sec-toint32 - // Note that the code below has been tested against example values. Please be very careful - // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI - // layer between JS and Rust, writing unit tests would be extremely complicated. - const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. - const converted = (message.maxUtf8BytesSize >= twoPower31) ? - (message.maxUtf8BytesSize - (twoPower31 * 2)) : message.maxUtf8BytesSize; - instance.exports.database_content(message.chainId, converted); - break; - } - - default: { - // Exhaustive check. - const _exhaustiveCheck: never = message; - return _exhaustiveCheck; - } - } -}; - -function postMessage(message: messages.FromWorker) { - // `compat.postMessage` is the same as `postMessage`, but works across environments. - compat.postMessage(message) -} - -// `compat.setOnMessage` is the same as `onmessage = ...`, but works across environments. -compat.setOnMessage((message: messages.ToWorker) => { - // What to do depends on the type of `state`. - // See the documentation of the `state` variable for information. - if (state == null) { - // First ever message received by the worker. Always contains the initial configuration. - const configMessage = message as messages.ToWorkerConfig; - - // Transition to the next phase: an array during which messages are stored while the - // initialization is in progress. - state = []; - - // Start initialization of the Wasm VM. - const config: instance.Config = { - logCallback: (level, target, message) => { - postMessage({ kind: 'log', level, target, message }); - }, - jsonRpcCallback: (data, chainId) => { - postMessage({ kind: 'jsonrpc', data, chainId }); - }, - databaseContentCallback: (data, chainId) => { - postMessage({ kind: 'databaseContent', data, chainId }); - }, - currentTaskCallback: (taskName) => { - postMessage({ kind: 'currentTask', taskName }); - }, - cpuRateLimit: configMessage.cpuRateLimit, - forbidTcp: configMessage.forbidTcp, - forbidWs: configMessage.forbidWs, - forbidNonLocalWs: configMessage.forbidNonLocalWs, - forbidWss: configMessage.forbidWss, - }; - - instance.startInstance(config).then((instance) => { - // `config.cpuRateLimit` is a floating point that should be between 0 and 1, while the value - // to pass as parameter must be between `0` and `2^32-1`. - // The few lines of code below should handle all possible values of `number`, including - // infinites and NaN. - let cpuRateLimit = Math.round(config.cpuRateLimit * 4294967295); // `2^32 - 1` - if (cpuRateLimit < 0) cpuRateLimit = 0; - if (cpuRateLimit > 4294967295) cpuRateLimit = 4294967295; - if (!Number.isFinite(cpuRateLimit)) cpuRateLimit = 4294967295; // User might have passed NaN - - // Smoldot requires an initial call to the `init` function in order to do its internal - // configuration. - instance.exports.init(configMessage.maxLogLevel, configMessage.enableCurrentTask ? 1 : 0, cpuRateLimit); - - // Smoldot has finished initializing. - // Since this function is an asynchronous function, it is possible that messages have been - // received from the parent while it was executing. These messages are now handled. - (state as messages.ToWorkerNonConfig[]).forEach((message) => { - injectMessage(instance, message); - }); - - state = instance; - }); - - } else if (Array.isArray(state)) { - // A message has been received while the Wasm VM is still initializing. Queue it for when - // initialization is over. - state.push(message as messages.ToWorkerNonConfig); - - } else { - // Everything is already initialized. Process the message synchronously. - injectMessage(state, message as messages.ToWorkerNonConfig); - } -}); - -// Periodically send a ping message to the outside, as a way to report liveness. -setInterval(() => { - postMessage({ kind: 'livenessPing' }); -}, 2500); diff --git a/bin/wasm-node/rust/src/bindings.rs b/bin/wasm-node/rust/src/bindings.rs index d40a50a5d8..5be2385223 100644 --- a/bin/wasm-node/rust/src/bindings.rs +++ b/bin/wasm-node/rust/src/bindings.rs @@ -280,6 +280,18 @@ pub extern "C" fn init(max_log_level: u32, enable_current_task: u32, cpu_rate_li crate::init(max_log_level, enable_current_task, cpu_rate_limit) } +/// Instructs the client to start shutting down. +/// +/// Later, the client will use `exit` to stop. +/// +/// It is still legal to call all the other functions of these bindings. The client continues to +/// operate normally until the call to `exit`, which happens at some point in the future. +// TODO: can this be called multiple times? +#[no_mangle] +pub extern "C" fn start_shutdown() { + crate::start_shutdown() +} + /// Allocates a buffer of the given length, with an alignment of 1. /// /// This must be used in the context of [`add_chain`] and other functions that similarly require diff --git a/bin/wasm-node/rust/src/lib.rs b/bin/wasm-node/rust/src/lib.rs index 0d513b1e66..2ded7e32c1 100644 --- a/bin/wasm-node/rust/src/lib.rs +++ b/bin/wasm-node/rust/src/lib.rs @@ -125,6 +125,11 @@ fn init(max_log_level: u32, enable_current_task: u32, cpu_rate_limit: u32) { *client_lock = Some(init_out); } +fn start_shutdown() { + // TODO: do this in a clean way + std::process::exit(0) +} + fn add_chain( chain_spec_pointer: u32, chain_spec_len: u32,