From 231a57fcb2075373c4d0b3286ad958361508bf81 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:03:55 +0200 Subject: [PATCH 01/39] Remove liveness pings --- bin/wasm-node/javascript/src/client.ts | 34 ------------------- .../javascript/src/worker/messages.ts | 6 +--- bin/wasm-node/javascript/src/worker/worker.ts | 5 --- 3 files changed, 1 insertion(+), 44 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 6da0617b8e..f236fda71d 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -397,31 +397,6 @@ export function start(options?: ClientOptions): Client { // possible, when the worker is frozen, to know which task it was in when frozen. const workerCurrentTask: { name: string | null } = { name: null }; - // The worker periodically sends a message of kind 'livenessPing' in order to notify that it is - // still alive. - // If this liveness ping isn't received for a long time, an error is reported in the logs. - // The first check is delayed in order to account for the fact that the worker has to perform - // an expensive initialization step when initializing the Wasm VM. - let livenessTimeout: null | ReturnType = null; - const resetLivenessTimeout = () => { - if (livenessTimeout !== null) - globalThis.clearTimeout(livenessTimeout); - livenessTimeout = globalThis.setTimeout(() => { - livenessTimeout = null; - if (workerError) - return; // The unresponsiveness is due to a crash. No need to print more warnings. - console.warn( - "Smoldot appears unresponsive" + - (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + - ". Please open an issue at https://github.com/paritytech/smoldot/issues. If you have a " + - "debugger available, please pause execution, generate a stack trace of the thread " + - "that isn't the main execution thread, and paste it in the issue. Please also include " + - "any other log found in the console or elsewhere." - ); - }, 10000); - }; - globalThis.setTimeout(() => resetLivenessTimeout(), 15000); - // The worker can send us messages whose type is identified through a `kind` field. workerOnMessage(worker, (message: messages.FromWorker): void => { switch (message.kind) { @@ -514,11 +489,6 @@ export function start(options?: ClientOptions): Client { break; } - case 'livenessPing': { - resetLivenessTimeout(); - break; - } - case 'currentTask': { workerCurrentTask.name = message.taskName; break; @@ -629,10 +599,6 @@ export function start(options?: ClientOptions): Client { if (workerError) return Promise.reject(workerError) workerError = new AlreadyDestroyedError(); - - if (livenessTimeout !== null) - globalThis.clearTimeout(livenessTimeout) - return workerTerminate(worker) } } diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts index 5d895f3eb0..1a712afcc5 100644 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ b/bin/wasm-node/javascript/src/worker/messages.ts @@ -27,7 +27,7 @@ export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorker /** * Message that the worker can send to the outside. */ -export type FromWorker = FromWorkerChainAddedOk | FromWorkerChainAddedError | FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent | FromWorkerLivenessPing | FromWorkerCurrentTask; +export type FromWorker = FromWorkerChainAddedOk | FromWorkerChainAddedError | FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent | FromWorkerCurrentTask; /** * Contains the initial configuration of the worker. @@ -117,10 +117,6 @@ export interface FromWorkerDatabaseContent { chainId: number, } -export interface FromWorkerLivenessPing { - kind: 'livenessPing', -} - export interface FromWorkerCurrentTask { kind: 'currentTask', taskName: string | null, diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 2bd9243df8..5fd7857d65 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -189,8 +189,3 @@ compat.setOnMessage((message: messages.ToWorker) => { injectMessage(state, message as messages.ToWorkerNonConfig); } }); - -// Periodically send a ping message to the outside, as a way to report liveness. -setInterval(() => { - postMessage({ kind: 'livenessPing' }); -}, 2500); From 5a75d3833fbbdfe63e1f33621a96940eafec1e9a Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:12:02 +0200 Subject: [PATCH 02/39] Remove "current task" system --- bin/wasm-node/javascript/src/client.ts | 14 +++----------- bin/wasm-node/javascript/src/worker/messages.ts | 7 +------ bin/wasm-node/javascript/src/worker/worker.ts | 2 +- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index f236fda71d..73836ab869 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -393,10 +393,6 @@ export function start(options?: ClientOptions): Client { // Immediately cleared when `remove()` is called on a chain. let chainIds: WeakMap = new WeakMap(); - // The worker periodically reports the name of the task it is currently in. This makes it - // possible, when the worker is frozen, to know which task it was in when frozen. - const workerCurrentTask: { name: string | null } = { name: null }; - // The worker can send us messages whose type is identified through a `kind` field. workerOnMessage(worker, (message: messages.FromWorker): void => { switch (message.kind) { @@ -489,11 +485,6 @@ export function start(options?: ClientOptions): Client { break; } - case 'currentTask': { - workerCurrentTask.name = message.taskName; - break; - } - default: { // Exhaustive check. const _exhaustiveCheck: never = message; @@ -506,13 +497,14 @@ export function start(options?: ClientOptions): Client { // A worker error should only happen in case of a critical error as the result of a bug // somewhere. Consequently, nothing is really in place to cleanly report the error. const errorToString = error.toString(); - console.error( + // TODO: restore after having updated `workerCurrentTask` + /*console.error( "Smoldot has panicked" + (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + ". This is a bug in smoldot. Please open an issue at " + "https://github.com/paritytech/smoldot/issues with the following message:\n" + errorToString - ); + );*/ workerError = new CrashError(errorToString); // Reject all promises returned by `addChain`. diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts index 1a712afcc5..4800491ce2 100644 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ b/bin/wasm-node/javascript/src/worker/messages.ts @@ -27,7 +27,7 @@ export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorker /** * Message that the worker can send to the outside. */ -export type FromWorker = FromWorkerChainAddedOk | FromWorkerChainAddedError | FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent | FromWorkerCurrentTask; +export type FromWorker = FromWorkerChainAddedOk | FromWorkerChainAddedError | FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent; /** * Contains the initial configuration of the worker. @@ -116,8 +116,3 @@ export interface FromWorkerDatabaseContent { data: string, chainId: number, } - -export interface FromWorkerCurrentTask { - kind: 'currentTask', - taskName: string | null, -} diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 5fd7857d65..e534aa07b3 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -146,7 +146,7 @@ compat.setOnMessage((message: messages.ToWorker) => { postMessage({ kind: 'databaseContent', data, chainId }); }, currentTaskCallback: (taskName) => { - postMessage({ kind: 'currentTask', taskName }); + // TODO: do something here? }, cpuRateLimit: configMessage.cpuRateLimit, forbidTcp: configMessage.forbidTcp, From ad9d586a30d3094a7fb54392b22dcfbd86b0ef96 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:16:36 +0200 Subject: [PATCH 03/39] Rename instance.ts -> raw-instance.ts --- .../javascript/src/worker/{instance.ts => raw-instance.ts} | 0 bin/wasm-node/javascript/src/worker/worker.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename bin/wasm-node/javascript/src/worker/{instance.ts => raw-instance.ts} (100%) diff --git a/bin/wasm-node/javascript/src/worker/instance.ts b/bin/wasm-node/javascript/src/worker/raw-instance.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/instance.ts rename to bin/wasm-node/javascript/src/worker/raw-instance.ts diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index e534aa07b3..80da730482 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -17,7 +17,7 @@ import { Buffer } from 'buffer'; import * as compat from './../compat/index.js'; -import * as instance from './instance.js'; +import * as instance from './raw-instance.js'; import * as messages from './messages.js'; import { SmoldotWasmInstance } from './bindings.js'; From 5acb49405bf3384246a05c55ebe2e9ca24b024bf Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:25:36 +0200 Subject: [PATCH 04/39] Turn worker.ts into a simple object --- bin/wasm-node/javascript/src/client.ts | 44 ++++++++----------- bin/wasm-node/javascript/src/worker/worker.ts | 35 ++++++++------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 73836ab869..92f428fb0f 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -15,9 +15,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -import { CompatWorker, workerOnMessage, workerOnError, workerTerminate } from './compat/index.js'; import * as messages from './worker/messages.js'; -import spawnWorker from './worker/spawn.js'; +import { start as startWorker } from './worker/worker.js'; /** * Thrown in case of a problem when initializing the chain. @@ -365,11 +364,6 @@ export function start(options?: ClientOptions): Client { } }); - // The actual execution of Smoldot is performed in a worker thread. - // Because this specific line of code is a bit sensitive, it is done in a separate file. - const worker = spawnWorker(); - let workerError: null | Error = null; - // Whenever an `addChain` or `removeChain` message is sent to the worker, a corresponding entry // is pushed to this array. The worker needs to send back a confirmation, which pops the first // element of this array. In the case of `addChain`, additional fields are stored in this array @@ -393,8 +387,11 @@ export function start(options?: ClientOptions): Client { // Immediately cleared when `remove()` is called on a chain. let chainIds: WeakMap = new WeakMap(); - // The worker can send us messages whose type is identified through a `kind` field. - workerOnMessage(worker, (message: messages.FromWorker): void => { + // The actual execution of Smoldot is performed in a worker thread. + // Because this specific line of code is a bit sensitive, it is done in a separate file. + let workerError: null | Error = null; + const worker = startWorker((message: messages.FromWorker): void => { + // The worker can send us messages whose type is identified through a `kind` field. switch (message.kind) { case 'jsonrpc': { const cb = chains.get(message.chainId)?.jsonRpcCallback; @@ -425,7 +422,7 @@ export function start(options?: ClientOptions): Client { throw new JsonRpcDisabledError(); if (request.length >= 8 * 1024 * 1024) return; - postMessage(worker, { ty: 'request', request, chainId }); + worker.handleMessage({ ty: 'request', request, chainId }); }, databaseContent: (maxUtf8BytesSize) => { if (workerError) @@ -443,7 +440,7 @@ export function start(options?: ClientOptions): Client { const maxSize = maxUtf8BytesSize || (twoPower32 - 1); const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - postMessage(worker, { ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); + worker.handleMessage({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); return promise; }, @@ -457,7 +454,7 @@ export function start(options?: ClientOptions): Client { throw new AlreadyDestroyedError(); console.assert(chainIds.has(newChain)); chainIds.delete(newChain); - postMessage(worker, { ty: 'removeChain', chainId }); + worker.handleMessage({ ty: 'removeChain', chainId }); }, }; @@ -493,18 +490,19 @@ export function start(options?: ClientOptions): Client { } }); - workerOnError(worker, (error) => { + // TODO: restore + /*workerOnError(worker, (error) => { // A worker error should only happen in case of a critical error as the result of a bug // somewhere. Consequently, nothing is really in place to cleanly report the error. const errorToString = error.toString(); // TODO: restore after having updated `workerCurrentTask` - /*console.error( + console.error( "Smoldot has panicked" + (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + ". This is a bug in smoldot. Please open an issue at " + "https://github.com/paritytech/smoldot/issues with the following message:\n" + errorToString - );*/ + ); workerError = new CrashError(errorToString); // Reject all promises returned by `addChain`. @@ -521,10 +519,10 @@ export function start(options?: ClientOptions): Client { } } chains.clear(); - }); + });*/ // The first message expected by the worker contains the configuration. - postMessage(worker, { + worker.handleMessage({ // Maximum level of log entries sent by the client. // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace maxLogLevel: options.maxLogLevel || 3, @@ -577,7 +575,7 @@ export function start(options?: ClientOptions): Client { jsonRpcCallback: options.jsonRpcCallback, }); - postMessage(worker, { + worker.handleMessage({ ty: 'addChain', chainSpec: options.chainSpec, databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", @@ -587,20 +585,16 @@ export function start(options?: ClientOptions): Client { return chainAddedPromise; }, - terminate: () => { + terminate: async () => { if (workerError) return Promise.reject(workerError) workerError = new AlreadyDestroyedError(); - return workerTerminate(worker) + // TODO: restore + //return workerTerminate(worker) } } } -// Separate function in order to enforce types. -function postMessage(worker: CompatWorker, message: messages.ToWorker) { - worker.postMessage(message) -} - interface PendingConfirmation { ty: 'chainAdded', resolve: (c: Chain) => void, diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 80da730482..a3dd42b779 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -16,11 +16,16 @@ // along with this program. If not, see . import { Buffer } from 'buffer'; -import * as compat from './../compat/index.js'; import * as instance from './raw-instance.js'; import * as messages from './messages.js'; import { SmoldotWasmInstance } from './bindings.js'; +export interface Worker { + handleMessage: (msg: messages.ToWorker) => void +} + +export function start(messagesCallback: (msg: messages.FromWorker) => void): Worker { + // This variable represents the state of the worker, and serves three different purposes: // // - At initialization, it is set to `null`. @@ -75,14 +80,14 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker ); if (instance.exports.chain_is_ok(chainId) != 0) { - postMessage({ kind: 'chainAddedOk', chainId }); + messagesCallback({ kind: 'chainAddedOk', chainId }); } else { const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; const errorMsgPtr = instance.exports.chain_error_ptr(chainId) >>> 0; const errorMsg = Buffer.from(instance.exports.memory.buffer) .toString('utf8', errorMsgPtr, errorMsgPtr + errorMsgLen); instance.exports.remove_chain(chainId); - postMessage({ kind: 'chainAddedErr', error: errorMsg }); + messagesCallback({ kind: 'chainAddedErr', error: errorMsg }); } break; @@ -117,13 +122,7 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker } }; -function postMessage(message: messages.FromWorker) { - // `compat.postMessage` is the same as `postMessage`, but works across environments. - compat.postMessage(message) -} - -// `compat.setOnMessage` is the same as `onmessage = ...`, but works across environments. -compat.setOnMessage((message: messages.ToWorker) => { +function handleMessage(message: messages.ToWorker) { // What to do depends on the type of `state`. // See the documentation of the `state` variable for information. if (state == null) { @@ -137,15 +136,15 @@ compat.setOnMessage((message: messages.ToWorker) => { // Start initialization of the Wasm VM. const config: instance.Config = { logCallback: (level, target, message) => { - postMessage({ kind: 'log', level, target, message }); + messagesCallback({ kind: 'log', level, target, message }); }, jsonRpcCallback: (data, chainId) => { - postMessage({ kind: 'jsonrpc', data, chainId }); + messagesCallback({ kind: 'jsonrpc', data, chainId }); }, databaseContentCallback: (data, chainId) => { - postMessage({ kind: 'databaseContent', data, chainId }); + messagesCallback({ kind: 'databaseContent', data, chainId }); }, - currentTaskCallback: (taskName) => { + currentTaskCallback: (_taskName) => { // TODO: do something here? }, cpuRateLimit: configMessage.cpuRateLimit, @@ -188,4 +187,10 @@ compat.setOnMessage((message: messages.ToWorker) => { // Everything is already initialized. Process the message synchronously. injectMessage(state, message as messages.ToWorkerNonConfig); } -}); +} + +return { + handleMessage +} + +} From 063679bb04fb1a46a5965db15f1d75662da557f9 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:26:39 +0200 Subject: [PATCH 05/39] Remove the utilities that deal with the worker --- bin/wasm-node/javascript/package.json | 3 +- .../src/compat/index-browser-overwrite.js | 32 --------------- .../javascript/src/compat/index.d.ts | 14 ------- bin/wasm-node/javascript/src/compat/index.js | 20 ---------- .../src/worker/spawn-browser-overwrite.js | 39 ------------------- .../javascript/src/worker/spawn.d.ts | 20 ---------- bin/wasm-node/javascript/src/worker/spawn.js | 32 --------------- 7 files changed, 1 insertion(+), 159 deletions(-) delete mode 100644 bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js delete mode 100644 bin/wasm-node/javascript/src/worker/spawn.d.ts delete mode 100644 bin/wasm-node/javascript/src/worker/spawn.js diff --git a/bin/wasm-node/javascript/package.json b/bin/wasm-node/javascript/package.json index 1d8ec8291f..817607d48d 100644 --- a/bin/wasm-node/javascript/package.json +++ b/bin/wasm-node/javascript/package.json @@ -23,8 +23,7 @@ "test": "node prepare.js --debug && rimraf ./dist && tsc && ava --timeout=2m --concurrency 2 --no-worker-threads" }, "browser": { - "./dist/compat/index.js": "./dist/compat/index-browser-overwrite.js", - "./dist/worker/spawn.js": "./dist/worker/spawn-browser-overwrite.js" + "./dist/compat/index.js": "./dist/compat/index-browser-overwrite.js" }, "dependencies": { "buffer": "^6.0.1", diff --git a/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js b/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js index 96844eb5d3..962cd65fc1 100644 --- a/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js +++ b/bin/wasm-node/javascript/src/compat/index-browser-overwrite.js @@ -17,38 +17,6 @@ // Overrides `index.js` when in a browser. -export function workerOnMessage(worker, callback) { - worker.onmessage = (event) => callback(event.data) -} - -export function workerOnError(worker, callback) { - worker.onerror = (event) => { - // For reference: - // https://html.spec.whatwg.org/multipage/indices.html#event-error - // https://html.spec.whatwg.org/multipage/webappapis.html#errorevent - - // If `event.error` exists, then it will likely be an instance of `Error`. - // However, that's not guaranteed by the spec and it could also be - // something else. So, our second best try is the `event.message` - // property. Finally, our last resort is to create an Error from the event. - if (event.error instanceof Error) callback(event.error); - else callback(new Error(event.message || event)); - } -} - -export function workerTerminate(worker) { - worker.terminate(); - return Promise.resolve(); -} - -export function postMessage(msg) { - self.postMessage(msg); -} - -export function setOnMessage(callback) { - self.onmessage = (event) => callback(event.data); -} - export function performanceNow() { return performance.now() } diff --git a/bin/wasm-node/javascript/src/compat/index.d.ts b/bin/wasm-node/javascript/src/compat/index.d.ts index f8abcd1a8e..5c752022c1 100644 --- a/bin/wasm-node/javascript/src/compat/index.d.ts +++ b/bin/wasm-node/javascript/src/compat/index.d.ts @@ -22,20 +22,6 @@ import type { Socket as TcpSocket, NetConnectOpts } from 'node:net'; export type WasmModuleImports = WebAssembly.ModuleImports; -export class CompatWorker { - postMessage(value: any, transferList?: ReadonlyArray): void; -} - -export function workerOnMessage(worker: CompatWorker, callback: (value: any) => void): void; - -export function workerOnError(worker: CompatWorker, callback: (err: Error) => void): void; - -export function workerTerminate(worker: CompatWorker): Promise; - -export function postMessage(message: any): void; - -export function setOnMessage(callback: (message: any) => void): void; - export function performanceNow(): number; export function isTcpAvailable(): boolean; diff --git a/bin/wasm-node/javascript/src/compat/index.js b/bin/wasm-node/javascript/src/compat/index.js index e3d7aa13da..c2ac374cb9 100644 --- a/bin/wasm-node/javascript/src/compat/index.js +++ b/bin/wasm-node/javascript/src/compat/index.js @@ -24,26 +24,6 @@ import { hrtime } from 'node:process'; import { createConnection as nodeCreateConnection } from 'node:net'; import { randomFillSync } from 'node:crypto'; -export function workerOnMessage(worker, callback) { - worker.on('message', callback); -} - -export function workerOnError(worker, callback) { - worker.on('error', callback); -} - -export function workerTerminate(worker) { - return worker.terminate().then(() => { }); -} - -export function postMessage(msg) { - parentPort.postMessage(msg); -} - -export function setOnMessage(callback) { - parentPort.on('message', callback); -} - export function performanceNow() { const time = hrtime(); return ((time[0] * 1e3) + (time[1] / 1e6)); diff --git a/bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js b/bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js deleted file mode 100644 index 8020c208fb..0000000000 --- a/bin/wasm-node/javascript/src/worker/spawn-browser-overwrite.js +++ /dev/null @@ -1,39 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -export default function () { - if (!window.Worker) - throw new Error("Workers not available"); - - // The line of code below (`new Worker(...)`) is designed to hopefully work across all - // platforms and bundlers. - // Because this line is precisely recognized by bundlers, we extract it to a separate - // JavaScript file. - // See also the README.md for more context. - - // Note that, at the time of writing, Firefox doesn't support the `type: "module"` option. - // Because browsers don't fully support modules yet, this code is expected to be run through - // a bundler (e.g. WebPack) before being given to a browser, which will remove all usage of - // modules in the worker code. It is thus also the role of this bundler to tweak or remove - // the value of this `type` property to indicate to the browser that modules aren't in use. - // - // WebPack in particular does this, but it is unclear whether *all* bundlers do it. - // Whether bundlers actually do this or not, it is nonetheless more correct to indicate - // `type: "module"` and doing so doesn't have any drawback. - const worker = new Worker(new URL('./worker.js', import.meta.url), { name: "smoldot", type: "module" }); - return worker; -} diff --git a/bin/wasm-node/javascript/src/worker/spawn.d.ts b/bin/wasm-node/javascript/src/worker/spawn.d.ts deleted file mode 100644 index 670e46b531..0000000000 --- a/bin/wasm-node/javascript/src/worker/spawn.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -import type { CompatWorker } from '../compat/index.js'; - -export default function (): CompatWorker; diff --git a/bin/wasm-node/javascript/src/worker/spawn.js b/bin/wasm-node/javascript/src/worker/spawn.js deleted file mode 100644 index 6b7fa06533..0000000000 --- a/bin/wasm-node/javascript/src/worker/spawn.js +++ /dev/null @@ -1,32 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -import { Worker } from 'node:worker_threads'; - -export default function () { - // The line of code below (`new Worker(...)`) is designed to hopefully work across all - // platforms and bundlers. - // Because this line is precisely recognized by bundlers, we extract it to a separate - // JavaScript file. - // See also the README.md for more context. - - // Note that at the time of writing of this comment, NodeJS doesn't support the - // `type: "module"` option, as modules "just work". But we put it anyway because Deno throws - // an exception if this option isn't present. - const worker = new Worker(new URL('./worker.js', import.meta.url), { type: "module" }); - return worker; -} From 3782fc18560d3e72159423847e08b5859952fea3 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:33:53 +0200 Subject: [PATCH 06/39] Provide the configuration when starting and not asychronously --- bin/wasm-node/javascript/src/client.ts | 34 +++++++++---------- bin/wasm-node/javascript/src/worker/worker.ts | 28 +++++---------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 92f428fb0f..4a97441379 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -390,7 +390,23 @@ export function start(options?: ClientOptions): Client { // The actual execution of Smoldot is performed in a worker thread. // Because this specific line of code is a bit sensitive, it is done in a separate file. let workerError: null | Error = null; - const worker = startWorker((message: messages.FromWorker): void => { + const worker = startWorker( + { + // Maximum level of log entries sent by the client. + // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace + maxLogLevel: options.maxLogLevel || 3, + // `enableCurrentTask` adds a small performance hit, but adds some additional information to + // crash reports. Whether this should be enabled is very opiniated and not that important. At + // the moment, we enable it all the time, except if the user has logging disabled altogether. + enableCurrentTask: options.maxLogLevel ? options.maxLogLevel >= 1 : true, + cpuRateLimit: options.cpuRateLimit || 1.0, + forbidTcp: options.forbidTcp || false, + forbidWs: options.forbidWs || false, + forbidNonLocalWs: options.forbidNonLocalWs || false, + forbidWss: options.forbidWss || false, + }, + + (message: messages.FromWorker): void => { // The worker can send us messages whose type is identified through a `kind` field. switch (message.kind) { case 'jsonrpc': { @@ -521,22 +537,6 @@ export function start(options?: ClientOptions): Client { chains.clear(); });*/ - // The first message expected by the worker contains the configuration. - worker.handleMessage({ - // Maximum level of log entries sent by the client. - // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace - maxLogLevel: options.maxLogLevel || 3, - // `enableCurrentTask` adds a small performance hit, but adds some additional information to - // crash reports. Whether this should be enabled is very opiniated and not that important. At - // the moment, we enable it all the time, except if the user has logging disabled altogether. - enableCurrentTask: options.maxLogLevel ? options.maxLogLevel >= 1 : true, - cpuRateLimit: options.cpuRateLimit || 1.0, - forbidTcp: options.forbidTcp || false, - forbidWs: options.forbidWs || false, - forbidNonLocalWs: options.forbidNonLocalWs || false, - forbidWss: options.forbidWss || false, - }); - return { addChain: (options: AddChainOptions): Promise => { if (workerError) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index a3dd42b779..c6a46f8a41 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -24,17 +24,15 @@ export interface Worker { handleMessage: (msg: messages.ToWorker) => void } -export function start(messagesCallback: (msg: messages.FromWorker) => void): Worker { +export function start(configMessage: messages.ToWorkerConfig, messagesCallback: (msg: messages.FromWorker) => void): Worker { -// This variable represents the state of the worker, and serves three different purposes: +// This variable represents the state of the worker, and serves two different purposes: // -// - At initialization, it is set to `null`. -// - Once the first message, containing the configuration, has been received from the parent, it -// becomes an array filled with the messages that are received while the Wasm VM is still -// initializing. +// - At initialization, it is an array filled with the messages that are received while the Wasm +// VM is still initializing. // - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. // -let state: null | messages.ToWorkerNonConfig[] | SmoldotWasmInstance = null; +let state: messages.ToWorkerNonConfig[] | SmoldotWasmInstance = []; // Inject a message coming from `index.js` to a running Wasm VM. function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorkerNonConfig): void { @@ -122,17 +120,6 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker } }; -function handleMessage(message: messages.ToWorker) { - // What to do depends on the type of `state`. - // See the documentation of the `state` variable for information. - if (state == null) { - // First ever message received by the worker. Always contains the initial configuration. - const configMessage = message as messages.ToWorkerConfig; - - // Transition to the next phase: an array during which messages are stored while the - // initialization is in progress. - state = []; - // Start initialization of the Wasm VM. const config: instance.Config = { logCallback: (level, target, message) => { @@ -178,7 +165,10 @@ function handleMessage(message: messages.ToWorker) { state = instance; }); - } else if (Array.isArray(state)) { +function handleMessage(message: messages.ToWorker) { + // What to do depends on the type of `state`. + // See the documentation of the `state` variable for information. + if (Array.isArray(state)) { // A message has been received while the Wasm VM is still initializing. Queue it for when // initialization is over. state.push(message as messages.ToWorkerNonConfig); From 4fe54bd0d6a5825e388a843ebda69754ba9f5022 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:55:27 +0200 Subject: [PATCH 07/39] Use a Promise for state rather than an array --- bin/wasm-node/javascript/src/worker/worker.ts | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index c6a46f8a41..ac064d0f9c 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -28,11 +28,10 @@ export function start(configMessage: messages.ToWorkerConfig, messagesCallback: // This variable represents the state of the worker, and serves two different purposes: // -// - At initialization, it is an array filled with the messages that are received while the Wasm -// VM is still initializing. +// - At initialization, it is a Promise containing the Wasm VM is still initializing. // - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. // -let state: messages.ToWorkerNonConfig[] | SmoldotWasmInstance = []; +let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; // Inject a message coming from `index.js` to a running Wasm VM. function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorkerNonConfig): void { @@ -141,7 +140,7 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker forbidWss: configMessage.forbidWss, }; - instance.startInstance(config).then((instance) => { + state = { initialized: false, promise: instance.startInstance(config).then((instance) => { // `config.cpuRateLimit` is a floating point that should be between 0 and 1, while the value // to pass as parameter must be between `0` and `2^32-1`. // The few lines of code below should handle all possible values of `number`, including @@ -155,27 +154,22 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker // configuration. instance.exports.init(configMessage.maxLogLevel, configMessage.enableCurrentTask ? 1 : 0, cpuRateLimit); - // Smoldot has finished initializing. - // Since this function is an asynchronous function, it is possible that messages have been - // received from the parent while it was executing. These messages are now handled. - (state as messages.ToWorkerNonConfig[]).forEach((message) => { - injectMessage(instance, message); - }); - - state = instance; - }); + return instance; + }) }; function handleMessage(message: messages.ToWorker) { // What to do depends on the type of `state`. // See the documentation of the `state` variable for information. - if (Array.isArray(state)) { + if (!state.initialized) { // A message has been received while the Wasm VM is still initializing. Queue it for when // initialization is over. - state.push(message as messages.ToWorkerNonConfig); + state.promise.then((instance) => { + injectMessage(instance, message as messages.ToWorkerNonConfig); + }) } else { // Everything is already initialized. Process the message synchronously. - injectMessage(state, message as messages.ToWorkerNonConfig); + injectMessage(state.instance, message as messages.ToWorkerNonConfig); } } From 01dfd4582e04009c0c346bb7fb81bee31ce56eeb Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 10:59:42 +0200 Subject: [PATCH 08/39] Use queueOperation instead of handleMessage --- bin/wasm-node/javascript/src/worker/worker.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index ac064d0f9c..645afcf602 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -157,24 +157,24 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker return instance; }) }; -function handleMessage(message: messages.ToWorker) { +function queueOperation(operation: (instance: SmoldotWasmInstance) => void) { // What to do depends on the type of `state`. // See the documentation of the `state` variable for information. if (!state.initialized) { // A message has been received while the Wasm VM is still initializing. Queue it for when // initialization is over. - state.promise.then((instance) => { - injectMessage(instance, message as messages.ToWorkerNonConfig); - }) + state.promise.then((instance) => { operation(instance) }) } else { // Everything is already initialized. Process the message synchronously. - injectMessage(state.instance, message as messages.ToWorkerNonConfig); + operation(state.instance) } } return { - handleMessage + handleMessage: (message: messages.ToWorker) => { + queueOperation((instance) => injectMessage(instance, message as messages.ToWorkerNonConfig)) + } } } From 34117ef5f5a3b80cc3e10de01fed8c38be1112e2 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:05:14 +0200 Subject: [PATCH 09/39] Turn generic handleMessage into individual functions --- bin/wasm-node/javascript/src/client.ts | 8 +- bin/wasm-node/javascript/src/worker/worker.ts | 153 +++++++++--------- 2 files changed, 76 insertions(+), 85 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 4a97441379..2b9de0f939 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -438,7 +438,7 @@ export function start(options?: ClientOptions): Client { throw new JsonRpcDisabledError(); if (request.length >= 8 * 1024 * 1024) return; - worker.handleMessage({ ty: 'request', request, chainId }); + worker.request({ ty: 'request', request, chainId }); }, databaseContent: (maxUtf8BytesSize) => { if (workerError) @@ -456,7 +456,7 @@ export function start(options?: ClientOptions): Client { const maxSize = maxUtf8BytesSize || (twoPower32 - 1); const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - worker.handleMessage({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); + worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); return promise; }, @@ -470,7 +470,7 @@ export function start(options?: ClientOptions): Client { throw new AlreadyDestroyedError(); console.assert(chainIds.has(newChain)); chainIds.delete(newChain); - worker.handleMessage({ ty: 'removeChain', chainId }); + worker.removeChain({ ty: 'removeChain', chainId }); }, }; @@ -575,7 +575,7 @@ export function start(options?: ClientOptions): Client { jsonRpcCallback: options.jsonRpcCallback, }); - worker.handleMessage({ + worker.addChain({ ty: 'addChain', chainSpec: options.chainSpec, databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 645afcf602..1a58ddf0aa 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -21,7 +21,10 @@ import * as messages from './messages.js'; import { SmoldotWasmInstance } from './bindings.js'; export interface Worker { - handleMessage: (msg: messages.ToWorker) => void + request: (message: messages.ToWorkerRpcRequest) => void + addChain: (message: messages.ToWorkerAddChain) => void + removeChain: (message: messages.ToWorkerRemoveChain) => void + databaseContent: (message: messages.ToWorkerDatabaseContent) => void } export function start(configMessage: messages.ToWorkerConfig, messagesCallback: (msg: messages.FromWorker) => void): Worker { @@ -33,18 +36,70 @@ export function start(configMessage: messages.ToWorkerConfig, messagesCallback: // let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; -// Inject a message coming from `index.js` to a running Wasm VM. -function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorkerNonConfig): void { - switch (message.ty) { - case 'request': { + // Start initialization of the Wasm VM. + const config: instance.Config = { + logCallback: (level, target, message) => { + messagesCallback({ kind: 'log', level, target, message }); + }, + jsonRpcCallback: (data, chainId) => { + messagesCallback({ kind: 'jsonrpc', data, chainId }); + }, + databaseContentCallback: (data, chainId) => { + messagesCallback({ kind: 'databaseContent', data, chainId }); + }, + currentTaskCallback: (_taskName) => { + // TODO: do something here? + }, + cpuRateLimit: configMessage.cpuRateLimit, + forbidTcp: configMessage.forbidTcp, + forbidWs: configMessage.forbidWs, + forbidNonLocalWs: configMessage.forbidNonLocalWs, + forbidWss: configMessage.forbidWss, + }; + + state = { initialized: false, promise: instance.startInstance(config).then((instance) => { + // `config.cpuRateLimit` is a floating point that should be between 0 and 1, while the value + // to pass as parameter must be between `0` and `2^32-1`. + // The few lines of code below should handle all possible values of `number`, including + // infinites and NaN. + let cpuRateLimit = Math.round(config.cpuRateLimit * 4294967295); // `2^32 - 1` + if (cpuRateLimit < 0) cpuRateLimit = 0; + if (cpuRateLimit > 4294967295) cpuRateLimit = 4294967295; + if (!Number.isFinite(cpuRateLimit)) cpuRateLimit = 4294967295; // User might have passed NaN + + // Smoldot requires an initial call to the `init` function in order to do its internal + // configuration. + instance.exports.init(configMessage.maxLogLevel, configMessage.enableCurrentTask ? 1 : 0, cpuRateLimit); + + return instance; + }) }; + +function queueOperation(operation: (instance: SmoldotWasmInstance) => void) { + // What to do depends on the type of `state`. + // See the documentation of the `state` variable for information. + if (!state.initialized) { + // A message has been received while the Wasm VM is still initializing. Queue it for when + // initialization is over. + state.promise.then((instance) => { operation(instance) }) + + } else { + // Everything is already initialized. Process the message synchronously. + operation(state.instance) + } +} + +return { + request: (message: messages.ToWorkerRpcRequest) => { + queueOperation((instance) => { const len = Buffer.byteLength(message.request, 'utf8'); const ptr = instance.exports.alloc(len) >>> 0; Buffer.from(instance.exports.memory.buffer).write(message.request, ptr); instance.exports.json_rpc_send(ptr, len, message.chainId); - break; - } + }) + }, - case 'addChain': { + addChain: (message: messages.ToWorkerAddChain) => { + queueOperation((instance) => { // Write the chain specification into memory. const chainSpecLen = Buffer.byteLength(message.chainSpec, 'utf8'); const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; @@ -86,16 +141,17 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker instance.exports.remove_chain(chainId); messagesCallback({ kind: 'chainAddedErr', error: errorMsg }); } + }) + }, - break; - } - - case 'removeChain': { + removeChain: (message: messages.ToWorkerRemoveChain) => { + queueOperation((instance) => { instance.exports.remove_chain(message.chainId); - break; - } + }) + }, - case 'databaseContent': { + databaseContent: (message: messages.ToWorkerDatabaseContent) => { + queueOperation((instance) => { // The value of `maxUtf8BytesSize` is guaranteed (by `index.js`) to always fit in 32 bits, in // other words, that `maxUtf8BytesSize < (1 << 32)`. // We need to perform a conversion in such a way that the the bits of the output of @@ -108,72 +164,7 @@ function injectMessage(instance: SmoldotWasmInstance, message: messages.ToWorker const converted = (message.maxUtf8BytesSize >= twoPower31) ? (message.maxUtf8BytesSize - (twoPower31 * 2)) : message.maxUtf8BytesSize; instance.exports.database_content(message.chainId, converted); - break; - } - - default: { - // Exhaustive check. - const _exhaustiveCheck: never = message; - return _exhaustiveCheck; - } - } -}; - - // Start initialization of the Wasm VM. - const config: instance.Config = { - logCallback: (level, target, message) => { - messagesCallback({ kind: 'log', level, target, message }); - }, - jsonRpcCallback: (data, chainId) => { - messagesCallback({ kind: 'jsonrpc', data, chainId }); - }, - databaseContentCallback: (data, chainId) => { - messagesCallback({ kind: 'databaseContent', data, chainId }); - }, - currentTaskCallback: (_taskName) => { - // TODO: do something here? - }, - cpuRateLimit: configMessage.cpuRateLimit, - forbidTcp: configMessage.forbidTcp, - forbidWs: configMessage.forbidWs, - forbidNonLocalWs: configMessage.forbidNonLocalWs, - forbidWss: configMessage.forbidWss, - }; - - state = { initialized: false, promise: instance.startInstance(config).then((instance) => { - // `config.cpuRateLimit` is a floating point that should be between 0 and 1, while the value - // to pass as parameter must be between `0` and `2^32-1`. - // The few lines of code below should handle all possible values of `number`, including - // infinites and NaN. - let cpuRateLimit = Math.round(config.cpuRateLimit * 4294967295); // `2^32 - 1` - if (cpuRateLimit < 0) cpuRateLimit = 0; - if (cpuRateLimit > 4294967295) cpuRateLimit = 4294967295; - if (!Number.isFinite(cpuRateLimit)) cpuRateLimit = 4294967295; // User might have passed NaN - - // Smoldot requires an initial call to the `init` function in order to do its internal - // configuration. - instance.exports.init(configMessage.maxLogLevel, configMessage.enableCurrentTask ? 1 : 0, cpuRateLimit); - - return instance; - }) }; - -function queueOperation(operation: (instance: SmoldotWasmInstance) => void) { - // What to do depends on the type of `state`. - // See the documentation of the `state` variable for information. - if (!state.initialized) { - // A message has been received while the Wasm VM is still initializing. Queue it for when - // initialization is over. - state.promise.then((instance) => { operation(instance) }) - - } else { - // Everything is already initialized. Process the message synchronously. - operation(state.instance) - } -} - -return { - handleMessage: (message: messages.ToWorker) => { - queueOperation((instance) => injectMessage(instance, message as messages.ToWorkerNonConfig)) + }) } } From 5ca6ba016a933f6735f1c67654f630d0d7c43442 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:20:47 +0200 Subject: [PATCH 10/39] Make worker.addChain return a Promise --- bin/wasm-node/javascript/src/client.ts | 168 +++++++----------- .../javascript/src/worker/messages.ts | 12 +- bin/wasm-node/javascript/src/worker/worker.ts | 16 +- 3 files changed, 74 insertions(+), 122 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 2b9de0f939..e1bea34377 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -364,12 +364,6 @@ export function start(options?: ClientOptions): Client { } }); - // Whenever an `addChain` or `removeChain` message is sent to the worker, a corresponding entry - // is pushed to this array. The worker needs to send back a confirmation, which pops the first - // element of this array. In the case of `addChain`, additional fields are stored in this array - // to finish the initialization of the chain. - let pendingConfirmations: PendingConfirmation[] = []; - // Contains the information of each chain that is currently. // Entries are instantly removed when the user desires to remove a chain even before the worker // has confirmed the removal. Doing so avoids a race condition where the worker sends back a @@ -415,78 +409,6 @@ export function start(options?: ClientOptions): Client { break; } - case 'chainAddedOk': { - const expected = pendingConfirmations.shift()!; - const chainId = message.chainId; - - if (chains.has(chainId)) // Sanity check. - throw 'Unexpected reuse of a chain ID'; - chains.set(chainId, { - jsonRpcCallback: expected.jsonRpcCallback, - databasePromises: new Array() - }); - - // `expected` was pushed by the `addChain` method. - // Resolve the promise that `addChain` returned to the user. - const newChain: Chain = { - sendJsonRpc: (request) => { - if (workerError) - throw workerError; - if (!chains.has(chainId)) - throw new AlreadyDestroyedError(); - if (!(chains.get(chainId)?.jsonRpcCallback)) - throw new JsonRpcDisabledError(); - if (request.length >= 8 * 1024 * 1024) - return; - worker.request({ ty: 'request', request, chainId }); - }, - databaseContent: (maxUtf8BytesSize) => { - if (workerError) - return Promise.reject(workerError); - - const databaseContentPromises = chains.get(chainId)?.databasePromises; - if (!databaseContentPromises) - return Promise.reject(new AlreadyDestroyedError()); - - const promise: Promise = new Promise((resolve, reject) => { - databaseContentPromises.push({ resolve, reject }); - }); - - const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. - const maxSize = maxUtf8BytesSize || (twoPower32 - 1); - const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - - worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); - - return promise; - }, - remove: () => { - if (workerError) - throw workerError; - // Because the `removeChain` message is asynchronous, it is possible for a JSON-RPC - // response or database content concerning that `chainId` to arrive after the `remove` - // function has returned. We solve that by removing the information immediately. - if (!chains.delete(chainId)) - throw new AlreadyDestroyedError(); - console.assert(chainIds.has(newChain)); - chainIds.delete(newChain); - worker.removeChain({ ty: 'removeChain', chainId }); - }, - }; - - chainIds.set(newChain, chainId); - expected.resolve(newChain); - break; - } - - case 'chainAddedErr': { - const expected = pendingConfirmations.shift()!; - // `expected` was pushed by the `addChain` method. - // Reject the promise that `addChain` returned to the user. - expected.reject(new AddChainError(message.error)); - break; - } - case 'databaseContent': { const promises = chains.get(message.chainId)?.databasePromises; if (promises) (promises.shift() as DatabasePromise).resolve(message.data); @@ -559,23 +481,7 @@ export function start(options?: ClientOptions): Client { } } - // Build a promise that will be resolved or rejected after the chain has been added. - // TODO: because of https://github.com/microsoft/TypeScript/issues/11498 we need to define the callbacks as possibly null, and go through `unknown` - let chainAddedPromiseResolve; - let chainAddedPromiseReject; - const chainAddedPromise: Promise = new Promise((resolve, reject) => { - chainAddedPromiseResolve = resolve; - chainAddedPromiseReject = reject; - }); - - pendingConfirmations.push({ - ty: 'chainAdded', - reject: chainAddedPromiseReject as unknown as (error: AddChainError) => void, - resolve: chainAddedPromiseResolve as unknown as (c: Chain) => void, - jsonRpcCallback: options.jsonRpcCallback, - }); - - worker.addChain({ + const promise = worker.addChain({ ty: 'addChain', chainSpec: options.chainSpec, databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", @@ -583,7 +489,70 @@ export function start(options?: ClientOptions): Client { jsonRpcRunning: !!options.jsonRpcCallback, }); - return chainAddedPromise; + return promise.then((outcome) => { + if (!outcome.success) + throw new AddChainError(outcome.error); + + const chainId = outcome.chainId; + + if (chains.has(chainId)) // Sanity check. + throw 'Unexpected reuse of a chain ID'; + chains.set(chainId, { + jsonRpcCallback: options.jsonRpcCallback, + databasePromises: new Array() + }); + + // `expected` was pushed by the `addChain` method. + // Resolve the promise that `addChain` returned to the user. + const newChain: Chain = { + sendJsonRpc: (request) => { + if (workerError) + throw workerError; + if (!chains.has(chainId)) + throw new AlreadyDestroyedError(); + if (!(chains.get(chainId)?.jsonRpcCallback)) + throw new JsonRpcDisabledError(); + if (request.length >= 8 * 1024 * 1024) + return; + worker.request({ ty: 'request', request, chainId }); + }, + databaseContent: (maxUtf8BytesSize) => { + if (workerError) + return Promise.reject(workerError); + + const databaseContentPromises = chains.get(chainId)?.databasePromises; + if (!databaseContentPromises) + return Promise.reject(new AlreadyDestroyedError()); + + const promise: Promise = new Promise((resolve, reject) => { + databaseContentPromises.push({ resolve, reject }); + }); + + const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. + const maxSize = maxUtf8BytesSize || (twoPower32 - 1); + const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; + + worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); + + return promise; + }, + remove: () => { + if (workerError) + throw workerError; + // Because the `removeChain` message is asynchronous, it is possible for a JSON-RPC + // response or database content concerning that `chainId` to arrive after the `remove` + // function has returned. We solve that by removing the information immediately. + if (!chains.delete(chainId)) + throw new AlreadyDestroyedError(); + console.assert(chainIds.has(newChain)); + chainIds.delete(newChain); + worker.removeChain({ ty: 'removeChain', chainId }); + }, + }; + + chainIds.set(newChain, chainId); + return newChain; + }) }, terminate: async () => { if (workerError) @@ -595,13 +564,6 @@ export function start(options?: ClientOptions): Client { } } -interface PendingConfirmation { - ty: 'chainAdded', - resolve: (c: Chain) => void, - reject: (error: AddChainError) => void, - jsonRpcCallback?: JsonRpcCallback, -} - interface DatabasePromise { resolve: (data: string) => void, reject: (error: Error) => void, diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts index 4800491ce2..8954752ce1 100644 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ b/bin/wasm-node/javascript/src/worker/messages.ts @@ -27,7 +27,7 @@ export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorker /** * Message that the worker can send to the outside. */ -export type FromWorker = FromWorkerChainAddedOk | FromWorkerChainAddedError | FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent; +export type FromWorker = FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent; /** * Contains the initial configuration of the worker. @@ -88,16 +88,6 @@ export interface ToWorkerDatabaseContent { maxUtf8BytesSize: number, } -export interface FromWorkerChainAddedOk { - kind: 'chainAddedOk', - chainId: number, -} - -export interface FromWorkerChainAddedError { - kind: 'chainAddedErr', - error: string, -} - export interface FromWorkerLog { kind: 'log', level: number, diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 1a58ddf0aa..8d4c364620 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -22,7 +22,7 @@ import { SmoldotWasmInstance } from './bindings.js'; export interface Worker { request: (message: messages.ToWorkerRpcRequest) => void - addChain: (message: messages.ToWorkerAddChain) => void + addChain: (message: messages.ToWorkerAddChain) => Promise<{ success: true, chainId: number } | { success: false, error: string }> removeChain: (message: messages.ToWorkerRemoveChain) => void databaseContent: (message: messages.ToWorkerDatabaseContent) => void } @@ -74,17 +74,17 @@ let state: { initialized: false, promise: Promise } | { ini return instance; }) }; -function queueOperation(operation: (instance: SmoldotWasmInstance) => void) { +async function queueOperation(operation: (instance: SmoldotWasmInstance) => T): Promise { // What to do depends on the type of `state`. // See the documentation of the `state` variable for information. if (!state.initialized) { // A message has been received while the Wasm VM is still initializing. Queue it for when // initialization is over. - state.promise.then((instance) => { operation(instance) }) + return state.promise.then((instance) => operation(instance)) } else { // Everything is already initialized. Process the message synchronously. - operation(state.instance) + return operation(state.instance) } } @@ -98,8 +98,8 @@ return { }) }, - addChain: (message: messages.ToWorkerAddChain) => { - queueOperation((instance) => { + addChain: (message: messages.ToWorkerAddChain): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { + return queueOperation((instance) => { // Write the chain specification into memory. const chainSpecLen = Buffer.byteLength(message.chainSpec, 'utf8'); const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; @@ -132,14 +132,14 @@ return { ); if (instance.exports.chain_is_ok(chainId) != 0) { - messagesCallback({ kind: 'chainAddedOk', chainId }); + return { success: true, chainId }; } else { const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; const errorMsgPtr = instance.exports.chain_error_ptr(chainId) >>> 0; const errorMsg = Buffer.from(instance.exports.memory.buffer) .toString('utf8', errorMsgPtr, errorMsgPtr + errorMsgLen); instance.exports.remove_chain(chainId); - messagesCallback({ kind: 'chainAddedErr', error: errorMsg }); + return { success: false, error: errorMsg }; } }) }, From e4ff8bd5c8aef66bc5dac20a4041bedd474fc588 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:23:16 +0200 Subject: [PATCH 11/39] addChain is now properly async --- bin/wasm-node/javascript/src/client.ts | 130 ++++++++++++------------- 1 file changed, 64 insertions(+), 66 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index e1bea34377..300da8222e 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -460,7 +460,7 @@ export function start(options?: ClientOptions): Client { });*/ return { - addChain: (options: AddChainOptions): Promise => { + addChain: async (options: AddChainOptions): Promise => { if (workerError) throw workerError; @@ -481,7 +481,7 @@ export function start(options?: ClientOptions): Client { } } - const promise = worker.addChain({ + const outcome = await worker.addChain({ ty: 'addChain', chainSpec: options.chainSpec, databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", @@ -489,70 +489,68 @@ export function start(options?: ClientOptions): Client { jsonRpcRunning: !!options.jsonRpcCallback, }); - return promise.then((outcome) => { - if (!outcome.success) - throw new AddChainError(outcome.error); - - const chainId = outcome.chainId; - - if (chains.has(chainId)) // Sanity check. - throw 'Unexpected reuse of a chain ID'; - chains.set(chainId, { - jsonRpcCallback: options.jsonRpcCallback, - databasePromises: new Array() - }); - - // `expected` was pushed by the `addChain` method. - // Resolve the promise that `addChain` returned to the user. - const newChain: Chain = { - sendJsonRpc: (request) => { - if (workerError) - throw workerError; - if (!chains.has(chainId)) - throw new AlreadyDestroyedError(); - if (!(chains.get(chainId)?.jsonRpcCallback)) - throw new JsonRpcDisabledError(); - if (request.length >= 8 * 1024 * 1024) - return; - worker.request({ ty: 'request', request, chainId }); - }, - databaseContent: (maxUtf8BytesSize) => { - if (workerError) - return Promise.reject(workerError); - - const databaseContentPromises = chains.get(chainId)?.databasePromises; - if (!databaseContentPromises) - return Promise.reject(new AlreadyDestroyedError()); - - const promise: Promise = new Promise((resolve, reject) => { - databaseContentPromises.push({ resolve, reject }); - }); - - const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. - const maxSize = maxUtf8BytesSize || (twoPower32 - 1); - const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - - worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); - - return promise; - }, - remove: () => { - if (workerError) - throw workerError; - // Because the `removeChain` message is asynchronous, it is possible for a JSON-RPC - // response or database content concerning that `chainId` to arrive after the `remove` - // function has returned. We solve that by removing the information immediately. - if (!chains.delete(chainId)) - throw new AlreadyDestroyedError(); - console.assert(chainIds.has(newChain)); - chainIds.delete(newChain); - worker.removeChain({ ty: 'removeChain', chainId }); - }, - }; - - chainIds.set(newChain, chainId); - return newChain; - }) + if (!outcome.success) + throw new AddChainError(outcome.error); + + const chainId = outcome.chainId; + + if (chains.has(chainId)) // Sanity check. + throw 'Unexpected reuse of a chain ID'; + chains.set(chainId, { + jsonRpcCallback: options.jsonRpcCallback, + databasePromises: new Array() + }); + + // `expected` was pushed by the `addChain` method. + // Resolve the promise that `addChain` returned to the user. + const newChain: Chain = { + sendJsonRpc: (request) => { + if (workerError) + throw workerError; + if (!chains.has(chainId)) + throw new AlreadyDestroyedError(); + if (!(chains.get(chainId)?.jsonRpcCallback)) + throw new JsonRpcDisabledError(); + if (request.length >= 8 * 1024 * 1024) + return; + worker.request({ ty: 'request', request, chainId }); + }, + databaseContent: (maxUtf8BytesSize) => { + if (workerError) + return Promise.reject(workerError); + + const databaseContentPromises = chains.get(chainId)?.databasePromises; + if (!databaseContentPromises) + return Promise.reject(new AlreadyDestroyedError()); + + const promise: Promise = new Promise((resolve, reject) => { + databaseContentPromises.push({ resolve, reject }); + }); + + const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. + const maxSize = maxUtf8BytesSize || (twoPower32 - 1); + const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; + + worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); + + return promise; + }, + remove: () => { + if (workerError) + throw workerError; + // Because the `removeChain` message is asynchronous, it is possible for a JSON-RPC + // response or database content concerning that `chainId` to arrive after the `remove` + // function has returned. We solve that by removing the information immediately. + if (!chains.delete(chainId)) + throw new AlreadyDestroyedError(); + console.assert(chainIds.has(newChain)); + chainIds.delete(newChain); + worker.removeChain({ ty: 'removeChain', chainId }); + }, + }; + + chainIds.set(newChain, chainId); + return newChain; }, terminate: async () => { if (workerError) From b478da1e85a69f17eed0ed7e3b3c462181904dd6 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:36:31 +0200 Subject: [PATCH 12/39] Make removeChain, request, and databaseContent always synchronous --- bin/wasm-node/javascript/src/worker/worker.ts | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 8d4c364620..22bc94fb53 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -90,12 +90,16 @@ async function queueOperation(operation: (instance: SmoldotWasmInstance) => T return { request: (message: messages.ToWorkerRpcRequest) => { - queueOperation((instance) => { - const len = Buffer.byteLength(message.request, 'utf8'); - const ptr = instance.exports.alloc(len) >>> 0; - Buffer.from(instance.exports.memory.buffer).write(message.request, ptr); - instance.exports.json_rpc_send(ptr, len, message.chainId); - }) + // Because `request` is passed as parameter an identifier returned by `addChain`, it is + // always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + + const len = Buffer.byteLength(message.request, 'utf8'); + const ptr = state.instance.exports.alloc(len) >>> 0; + Buffer.from(state.instance.exports.memory.buffer).write(message.request, ptr); + state.instance.exports.json_rpc_send(ptr, len, message.chainId); }, addChain: (message: messages.ToWorkerAddChain): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { @@ -145,26 +149,34 @@ return { }, removeChain: (message: messages.ToWorkerRemoveChain) => { - queueOperation((instance) => { - instance.exports.remove_chain(message.chainId); - }) + // Because `removeChain` is passed as parameter an identifier returned by `addChain`, it is + // always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + + state.instance.exports.remove_chain(message.chainId); }, databaseContent: (message: messages.ToWorkerDatabaseContent) => { - queueOperation((instance) => { - // The value of `maxUtf8BytesSize` is guaranteed (by `index.js`) to always fit in 32 bits, in - // other words, that `maxUtf8BytesSize < (1 << 32)`. - // We need to perform a conversion in such a way that the the bits of the output of - // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. - // See ToInt32 here: https://tc39.es/ecma262/#sec-toint32 - // Note that the code below has been tested against example values. Please be very careful - // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI - // layer between JS and Rust, writing unit tests would be extremely complicated. - const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. - const converted = (message.maxUtf8BytesSize >= twoPower31) ? - (message.maxUtf8BytesSize - (twoPower31 * 2)) : message.maxUtf8BytesSize; - instance.exports.database_content(message.chainId, converted); - }) + // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it + // is always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + + // The value of `maxUtf8BytesSize` is guaranteed (by `index.js`) to always fit in 32 bits, in + // other words, that `maxUtf8BytesSize < (1 << 32)`. + // We need to perform a conversion in such a way that the the bits of the output of + // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. + // See ToInt32 here: https://tc39.es/ecma262/#sec-toint32 + // Note that the code below has been tested against example values. Please be very careful + // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI + // layer between JS and Rust, writing unit tests would be extremely complicated. + const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. + const converted = (message.maxUtf8BytesSize >= twoPower31) ? + (message.maxUtf8BytesSize - (twoPower31 * 2)) : message.maxUtf8BytesSize; + state.instance.exports.database_content(message.chainId, converted); } } From 511c6c0651e90bc3ac729dbf9134b324bdc49717 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:39:04 +0200 Subject: [PATCH 13/39] Track the active chains within worker.ts --- bin/wasm-node/javascript/src/worker/worker.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 22bc94fb53..4cc5d2d836 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -36,6 +36,12 @@ export function start(configMessage: messages.ToWorkerConfig, messagesCallback: // let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; + // Contains the information of each chain that is currently alive. + let chains: Map void, + databasePromises: DatabasePromise[], + }> = new Map(); + // Start initialization of the Wasm VM. const config: instance.Config = { logCallback: (level, target, message) => { @@ -136,6 +142,12 @@ return { ); if (instance.exports.chain_is_ok(chainId) != 0) { + if (chains.has(chainId)) // Sanity check. + throw 'Unexpected reuse of a chain ID'; + chains.set(chainId, { + // TODO: jsonRpcCallback: options.jsonRpcCallback, + databasePromises: new Array() + }); return { success: true, chainId }; } else { const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; @@ -155,6 +167,10 @@ return { if (!state.initialized) throw new Error("Internal error"); + // Removing the chain synchronously avoids having to deal with race conditions such as a + // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. + // These kind of race conditions are already delt with within smoldot. + chains.delete(message.chainId); state.instance.exports.remove_chain(message.chainId); }, @@ -181,3 +197,8 @@ return { } } + +interface DatabasePromise { + resolve: (data: string) => void, + reject: (error: Error) => void, +} From 025d6abb4bb3d588ecb74ddd21ffbe1903da8b39 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:42:48 +0200 Subject: [PATCH 14/39] Fix state wasn't being set properly --- bin/wasm-node/javascript/src/worker/worker.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 4cc5d2d836..fabb392e1e 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -77,6 +77,7 @@ let state: { initialized: false, promise: Promise } | { ini // configuration. instance.exports.init(configMessage.maxLogLevel, configMessage.enableCurrentTask ? 1 : 0, cpuRateLimit); + state = { initialized: true, instance }; return instance; }) }; From f240fb4188efd3798e8d88969e6e1a362b32a7aa Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:46:14 +0200 Subject: [PATCH 15/39] Make databaseContent return a Promise --- bin/wasm-node/javascript/src/client.ts | 27 ++----------------- .../javascript/src/worker/messages.ts | 10 ++----- bin/wasm-node/javascript/src/worker/worker.ts | 14 +++++++--- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 300da8222e..b002507ec7 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -373,7 +373,6 @@ export function start(options?: ClientOptions): Client { // This map is also used in general as a way to check whether a chain still exists. let chains: Map = new Map(); // For each chain object returned by `addChain`, the associated internal chain id. @@ -409,12 +408,6 @@ export function start(options?: ClientOptions): Client { break; } - case 'databaseContent': { - const promises = chains.get(message.chainId)?.databasePromises; - if (promises) (promises.shift() as DatabasePromise).resolve(message.data); - break; - } - case 'log': { logCallback(message.level, message.target, message.message); break; @@ -497,8 +490,7 @@ export function start(options?: ClientOptions): Client { if (chains.has(chainId)) // Sanity check. throw 'Unexpected reuse of a chain ID'; chains.set(chainId, { - jsonRpcCallback: options.jsonRpcCallback, - databasePromises: new Array() + jsonRpcCallback: options.jsonRpcCallback }); // `expected` was pushed by the `addChain` method. @@ -519,21 +511,11 @@ export function start(options?: ClientOptions): Client { if (workerError) return Promise.reject(workerError); - const databaseContentPromises = chains.get(chainId)?.databasePromises; - if (!databaseContentPromises) - return Promise.reject(new AlreadyDestroyedError()); - - const promise: Promise = new Promise((resolve, reject) => { - databaseContentPromises.push({ resolve, reject }); - }); - const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. const maxSize = maxUtf8BytesSize || (twoPower32 - 1); const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); - - return promise; + return worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); }, remove: () => { if (workerError) @@ -561,8 +543,3 @@ export function start(options?: ClientOptions): Client { } } } - -interface DatabasePromise { - resolve: (data: string) => void, - reject: (error: Error) => void, -} diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts index 8954752ce1..4d99eb628a 100644 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ b/bin/wasm-node/javascript/src/worker/messages.ts @@ -22,12 +22,12 @@ * messages must be `ToWorkerNonConfig`s. */ export type ToWorker = ToWorkerConfig | ToWorkerNonConfig; -export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorkerRemoveChain | ToWorkerDatabaseContent; +export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorkerRemoveChain; /** * Message that the worker can send to the outside. */ -export type FromWorker = FromWorkerLog | FromWorkerJsonRpc | FromWorkerDatabaseContent; +export type FromWorker = FromWorkerLog | FromWorkerJsonRpc; /** * Contains the initial configuration of the worker. @@ -100,9 +100,3 @@ export interface FromWorkerJsonRpc { data: string, chainId: number, } - -export interface FromWorkerDatabaseContent { - kind: 'databaseContent', - data: string, - chainId: number, -} diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index fabb392e1e..01e7b969d3 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -24,7 +24,7 @@ export interface Worker { request: (message: messages.ToWorkerRpcRequest) => void addChain: (message: messages.ToWorkerAddChain) => Promise<{ success: true, chainId: number } | { success: false, error: string }> removeChain: (message: messages.ToWorkerRemoveChain) => void - databaseContent: (message: messages.ToWorkerDatabaseContent) => void + databaseContent: (message: messages.ToWorkerDatabaseContent) => Promise } export function start(configMessage: messages.ToWorkerConfig, messagesCallback: (msg: messages.FromWorker) => void): Worker { @@ -51,7 +51,8 @@ let state: { initialized: false, promise: Promise } | { ini messagesCallback({ kind: 'jsonrpc', data, chainId }); }, databaseContentCallback: (data, chainId) => { - messagesCallback({ kind: 'databaseContent', data, chainId }); + const promises = chains.get(chainId)?.databasePromises!; + (promises.shift() as DatabasePromise).resolve(data); }, currentTaskCallback: (_taskName) => { // TODO: do something here? @@ -175,13 +176,18 @@ return { state.instance.exports.remove_chain(message.chainId); }, - databaseContent: (message: messages.ToWorkerDatabaseContent) => { + databaseContent: (message: messages.ToWorkerDatabaseContent): Promise => { // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it // is always the case that the Wasm instance is already initialized. The only possibility for // it to not be the case is if the user completely invented the `chainId`. if (!state.initialized) throw new Error("Internal error"); + const databaseContentPromises = chains.get(message.chainId)?.databasePromises!; + const promise: Promise = new Promise((resolve, reject) => { + databaseContentPromises.push({ resolve, reject }); + }); + // The value of `maxUtf8BytesSize` is guaranteed (by `index.js`) to always fit in 32 bits, in // other words, that `maxUtf8BytesSize < (1 << 32)`. // We need to perform a conversion in such a way that the the bits of the output of @@ -194,6 +200,8 @@ return { const converted = (message.maxUtf8BytesSize >= twoPower31) ? (message.maxUtf8BytesSize - (twoPower31 * 2)) : message.maxUtf8BytesSize; state.instance.exports.database_content(message.chainId, converted); + + return promise; } } From c61a4a61709fdbc57043115c958ecf1b6fd41575 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:52:49 +0200 Subject: [PATCH 16/39] Provide logCallback in config and finish removing messages --- bin/wasm-node/javascript/src/client.ts | 29 ++----------------- .../javascript/src/worker/messages.ts | 21 ++------------ bin/wasm-node/javascript/src/worker/worker.ts | 13 +++++---- 3 files changed, 12 insertions(+), 51 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index b002507ec7..b1875ce8ec 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -15,7 +15,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -import * as messages from './worker/messages.js'; import { start as startWorker } from './worker/worker.js'; /** @@ -383,11 +382,11 @@ export function start(options?: ClientOptions): Client { // The actual execution of Smoldot is performed in a worker thread. // Because this specific line of code is a bit sensitive, it is done in a separate file. let workerError: null | Error = null; - const worker = startWorker( - { + const worker = startWorker({ // Maximum level of log entries sent by the client. // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace maxLogLevel: options.maxLogLevel || 3, + logCallback, // `enableCurrentTask` adds a small performance hit, but adds some additional information to // crash reports. Whether this should be enabled is very opiniated and not that important. At // the moment, we enable it all the time, except if the user has logging disabled altogether. @@ -397,28 +396,6 @@ export function start(options?: ClientOptions): Client { forbidWs: options.forbidWs || false, forbidNonLocalWs: options.forbidNonLocalWs || false, forbidWss: options.forbidWss || false, - }, - - (message: messages.FromWorker): void => { - // The worker can send us messages whose type is identified through a `kind` field. - switch (message.kind) { - case 'jsonrpc': { - const cb = chains.get(message.chainId)?.jsonRpcCallback; - if (cb) cb(message.data); - break; - } - - case 'log': { - logCallback(message.level, message.target, message.message); - break; - } - - default: { - // Exhaustive check. - const _exhaustiveCheck: never = message; - return _exhaustiveCheck; - } - } }); // TODO: restore @@ -479,7 +456,7 @@ export function start(options?: ClientOptions): Client { chainSpec: options.chainSpec, databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", potentialRelayChains: potentialRelayChainsIds, - jsonRpcRunning: !!options.jsonRpcCallback, + jsonRpcCallback: options.jsonRpcCallback, }); if (!outcome.success) diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts index 4d99eb628a..2ac1c9f361 100644 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ b/bin/wasm-node/javascript/src/worker/messages.ts @@ -24,11 +24,6 @@ export type ToWorker = ToWorkerConfig | ToWorkerNonConfig; export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorkerRemoveChain; -/** - * Message that the worker can send to the outside. - */ -export type FromWorker = FromWorkerLog | FromWorkerJsonRpc; - /** * Contains the initial configuration of the worker. * @@ -36,6 +31,7 @@ export type FromWorker = FromWorkerLog | FromWorkerJsonRpc; * worker. */ export interface ToWorkerConfig { + logCallback: (level: number, target: string, message: string) => void maxLogLevel: number; enableCurrentTask: boolean; cpuRateLimit: number, @@ -64,7 +60,7 @@ export interface ToWorkerAddChain { chainSpec: string, databaseContent: string, potentialRelayChains: number[], - jsonRpcRunning: boolean, + jsonRpcCallback?: (response: string) => void, } /** @@ -87,16 +83,3 @@ export interface ToWorkerDatabaseContent { chainId: number, maxUtf8BytesSize: number, } - -export interface FromWorkerLog { - kind: 'log', - level: number, - target: string, - message: string, -} - -export interface FromWorkerJsonRpc { - kind: 'jsonrpc', - data: string, - chainId: number, -} diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 01e7b969d3..e41e35c23b 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -27,7 +27,7 @@ export interface Worker { databaseContent: (message: messages.ToWorkerDatabaseContent) => Promise } -export function start(configMessage: messages.ToWorkerConfig, messagesCallback: (msg: messages.FromWorker) => void): Worker { +export function start(configMessage: messages.ToWorkerConfig): Worker { // This variable represents the state of the worker, and serves two different purposes: // @@ -38,17 +38,18 @@ let state: { initialized: false, promise: Promise } | { ini // Contains the information of each chain that is currently alive. let chains: Map void, + jsonRpcCallback?: (response: string) => void, databasePromises: DatabasePromise[], }> = new Map(); // Start initialization of the Wasm VM. const config: instance.Config = { logCallback: (level, target, message) => { - messagesCallback({ kind: 'log', level, target, message }); + configMessage.logCallback(level, target, message) }, jsonRpcCallback: (data, chainId) => { - messagesCallback({ kind: 'jsonrpc', data, chainId }); + const cb = chains.get(chainId)?.jsonRpcCallback; + if (cb) cb(data); }, databaseContentCallback: (data, chainId) => { const promises = chains.get(chainId)?.databasePromises!; @@ -139,7 +140,7 @@ return { const chainId = instance.exports.add_chain( chainSpecPtr, chainSpecLen, databaseContentPtr, databaseContentLen, - message.jsonRpcRunning ? 1 : 0, + !!message.jsonRpcCallback ? 1 : 0, potentialRelayChainsPtr, potentialRelayChainsLen ); @@ -147,7 +148,7 @@ return { if (chains.has(chainId)) // Sanity check. throw 'Unexpected reuse of a chain ID'; chains.set(chainId, { - // TODO: jsonRpcCallback: options.jsonRpcCallback, + jsonRpcCallback: message.jsonRpcCallback, databasePromises: new Array() }); return { success: true, chainId }; From 26d3c4530324621c7114f25376b2bb0e16e0b874 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:55:29 +0200 Subject: [PATCH 17/39] Remove mention of the worker --- bin/wasm-node/javascript/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/bin/wasm-node/javascript/README.md b/bin/wasm-node/javascript/README.md index 3c211dd6f4..c9df517ff5 100644 --- a/bin/wasm-node/javascript/README.md +++ b/bin/wasm-node/javascript/README.md @@ -73,22 +73,3 @@ chains must be passed as parameter to `addChain` as well. In situations where th specifications passed to `addChain` are not trusted, it is important for security reasons to not establish a parachain-relay-chain link between two chains that aren't part of the same "trust sandbox". - -# About the worker - -The code in this package uses a web worker (in browsers) or a worker thread (on NodeJS). The -line of JavaScript that creates the worker is of the following form: - -``` js -new Worker(new URL('./worker.js', import.meta.url), { type: "module" }); -``` - -This format is compatible [with Webpack 5](https://webpack.js.org/guides/web-workers/), meaning -that Webpack will be able to resolve the imports in `worker.js` and adjust this little snippet. - -This format also works in NodeJS without any issue. - -However, at the time of writing of this comment, this format doesn't work with Parcel (both 1 and -2) due to various bugs. - -As a general warning, be aware of the fact that this line might cause issues if you use a bundler. From ba4addcc776f02bd95729eaf55442b02943e55b0 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 11:55:40 +0200 Subject: [PATCH 18/39] Remove unused node:worker_threads import --- bin/wasm-node/javascript/src/compat/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/wasm-node/javascript/src/compat/index.js b/bin/wasm-node/javascript/src/compat/index.js index c2ac374cb9..bc86e6a468 100644 --- a/bin/wasm-node/javascript/src/compat/index.js +++ b/bin/wasm-node/javascript/src/compat/index.js @@ -19,7 +19,6 @@ // // A rule in the `package.json` overrides it with `index-browser-override.js` when in a browser. -import { parentPort } from 'node:worker_threads'; import { hrtime } from 'node:process'; import { createConnection as nodeCreateConnection } from 'node:net'; import { randomFillSync } from 'node:crypto'; From 97a39445c211aa1eeaa0d525557ea92ec6df92c9 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 12:00:53 +0200 Subject: [PATCH 19/39] Finish removing the ToWorker messages --- bin/wasm-node/javascript/src/client.ts | 14 ++--- .../javascript/src/worker/messages.ts | 52 ------------------- bin/wasm-node/javascript/src/worker/worker.ts | 52 +++++++++---------- 3 files changed, 30 insertions(+), 88 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index b1875ce8ec..13bbabed6c 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -451,13 +451,7 @@ export function start(options?: ClientOptions): Client { } } - const outcome = await worker.addChain({ - ty: 'addChain', - chainSpec: options.chainSpec, - databaseContent: typeof options.databaseContent === 'string' ? options.databaseContent : "", - potentialRelayChains: potentialRelayChainsIds, - jsonRpcCallback: options.jsonRpcCallback, - }); + const outcome = await worker.addChain(options.chainSpec, typeof options.databaseContent === 'string' ? options.databaseContent : "", potentialRelayChainsIds, options.jsonRpcCallback); if (!outcome.success) throw new AddChainError(outcome.error); @@ -482,7 +476,7 @@ export function start(options?: ClientOptions): Client { throw new JsonRpcDisabledError(); if (request.length >= 8 * 1024 * 1024) return; - worker.request({ ty: 'request', request, chainId }); + worker.request(request, chainId); }, databaseContent: (maxUtf8BytesSize) => { if (workerError) @@ -492,7 +486,7 @@ export function start(options?: ClientOptions): Client { const maxSize = maxUtf8BytesSize || (twoPower32 - 1); const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - return worker.databaseContent({ ty: 'databaseContent', chainId, maxUtf8BytesSize: cappedMaxSize }); + return worker.databaseContent(chainId, cappedMaxSize); }, remove: () => { if (workerError) @@ -504,7 +498,7 @@ export function start(options?: ClientOptions): Client { throw new AlreadyDestroyedError(); console.assert(chainIds.has(newChain)); chainIds.delete(newChain); - worker.removeChain({ ty: 'removeChain', chainId }); + worker.removeChain(chainId); }, }; diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts index 2ac1c9f361..3b01c389d1 100644 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ b/bin/wasm-node/javascript/src/worker/messages.ts @@ -15,15 +15,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -/** - * Message to the worker. - * - * The first ever message sent to the worker must be a `ToWorkerConfig`, then all subsequent - * messages must be `ToWorkerNonConfig`s. - */ -export type ToWorker = ToWorkerConfig | ToWorkerNonConfig; -export type ToWorkerNonConfig = ToWorkerRpcRequest | ToWorkerAddChain | ToWorkerRemoveChain; - /** * Contains the initial configuration of the worker. * @@ -40,46 +31,3 @@ export interface ToWorkerConfig { forbidNonLocalWs: boolean; forbidWss: boolean; } - -/** - * Start a JSON-RPC request. - */ -export interface ToWorkerRpcRequest { - ty: 'request', - request: string, - chainId: number, -} - -/** - * Add a new chain. - * - * The worker must reply with either a `FromWorkerChainAddedOk` or a `FromWorkerChainAddedError`. - */ -export interface ToWorkerAddChain { - ty: 'addChain', - chainSpec: string, - databaseContent: string, - potentialRelayChains: number[], - jsonRpcCallback?: (response: string) => void, -} - -/** - * Remove a chain. - * - * The worker must reply with a `FromWorkerChainRemoved`. - */ -export interface ToWorkerRemoveChain { - ty: 'removeChain', - chainId: number, -} - -/** - * Get the database content of a chain. - * - * The worker must reply with a `FromWorkerDatabaseContent`. - */ -export interface ToWorkerDatabaseContent { - ty: 'databaseContent', - chainId: number, - maxUtf8BytesSize: number, -} diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index e41e35c23b..c0d43be27f 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -21,10 +21,10 @@ import * as messages from './messages.js'; import { SmoldotWasmInstance } from './bindings.js'; export interface Worker { - request: (message: messages.ToWorkerRpcRequest) => void - addChain: (message: messages.ToWorkerAddChain) => Promise<{ success: true, chainId: number } | { success: false, error: string }> - removeChain: (message: messages.ToWorkerRemoveChain) => void - databaseContent: (message: messages.ToWorkerDatabaseContent) => Promise + request: (request: string, chainId: number) => void + addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void) => Promise<{ success: true, chainId: number } | { success: false, error: string }> + removeChain: (chainId: number) => void + databaseContent: (chainId: number, maxUtf8BytesSize: number) => Promise } export function start(configMessage: messages.ToWorkerConfig): Worker { @@ -98,39 +98,39 @@ async function queueOperation(operation: (instance: SmoldotWasmInstance) => T } return { - request: (message: messages.ToWorkerRpcRequest) => { + request: (request: string, chainId: number) => { // Because `request` is passed as parameter an identifier returned by `addChain`, it is // always the case that the Wasm instance is already initialized. The only possibility for // it to not be the case is if the user completely invented the `chainId`. if (!state.initialized) throw new Error("Internal error"); - const len = Buffer.byteLength(message.request, 'utf8'); + const len = Buffer.byteLength(request, 'utf8'); const ptr = state.instance.exports.alloc(len) >>> 0; - Buffer.from(state.instance.exports.memory.buffer).write(message.request, ptr); - state.instance.exports.json_rpc_send(ptr, len, message.chainId); + Buffer.from(state.instance.exports.memory.buffer).write(request, ptr); + state.instance.exports.json_rpc_send(ptr, len, chainId); }, - addChain: (message: messages.ToWorkerAddChain): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { + addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { return queueOperation((instance) => { // Write the chain specification into memory. - const chainSpecLen = Buffer.byteLength(message.chainSpec, 'utf8'); + const chainSpecLen = Buffer.byteLength(chainSpec, 'utf8'); const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; Buffer.from(instance.exports.memory.buffer) - .write(message.chainSpec, chainSpecPtr); + .write(chainSpec, chainSpecPtr); // Write the database content into memory. - const databaseContentLen = Buffer.byteLength(message.databaseContent, 'utf8'); + const databaseContentLen = Buffer.byteLength(databaseContent, 'utf8'); const databaseContentPtr = instance.exports.alloc(databaseContentLen) >>> 0; Buffer.from(instance.exports.memory.buffer) - .write(message.databaseContent, databaseContentPtr); + .write(databaseContent, databaseContentPtr); // Write the potential relay chains into memory. - const potentialRelayChainsLen = message.potentialRelayChains.length; + const potentialRelayChainsLen = potentialRelayChains.length; const potentialRelayChainsPtr = instance.exports.alloc(potentialRelayChainsLen * 4) >>> 0; - for (let idx = 0; idx < message.potentialRelayChains.length; ++idx) { + for (let idx = 0; idx < potentialRelayChains.length; ++idx) { Buffer.from(instance.exports.memory.buffer) - .writeUInt32LE(message.potentialRelayChains[idx]!, potentialRelayChainsPtr + idx * 4); + .writeUInt32LE(potentialRelayChains[idx]!, potentialRelayChainsPtr + idx * 4); } // `add_chain` unconditionally allocates a chain id. If an error occurs, however, this chain @@ -140,7 +140,7 @@ return { const chainId = instance.exports.add_chain( chainSpecPtr, chainSpecLen, databaseContentPtr, databaseContentLen, - !!message.jsonRpcCallback ? 1 : 0, + !!jsonRpcCallback ? 1 : 0, potentialRelayChainsPtr, potentialRelayChainsLen ); @@ -148,7 +148,7 @@ return { if (chains.has(chainId)) // Sanity check. throw 'Unexpected reuse of a chain ID'; chains.set(chainId, { - jsonRpcCallback: message.jsonRpcCallback, + jsonRpcCallback, databasePromises: new Array() }); return { success: true, chainId }; @@ -163,7 +163,7 @@ return { }) }, - removeChain: (message: messages.ToWorkerRemoveChain) => { + removeChain: (chainId: number) => { // Because `removeChain` is passed as parameter an identifier returned by `addChain`, it is // always the case that the Wasm instance is already initialized. The only possibility for // it to not be the case is if the user completely invented the `chainId`. @@ -173,18 +173,18 @@ return { // Removing the chain synchronously avoids having to deal with race conditions such as a // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. // These kind of race conditions are already delt with within smoldot. - chains.delete(message.chainId); - state.instance.exports.remove_chain(message.chainId); + chains.delete(chainId); + state.instance.exports.remove_chain(chainId); }, - databaseContent: (message: messages.ToWorkerDatabaseContent): Promise => { + databaseContent: (chainId: number, maxUtf8BytesSize: number): Promise => { // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it // is always the case that the Wasm instance is already initialized. The only possibility for // it to not be the case is if the user completely invented the `chainId`. if (!state.initialized) throw new Error("Internal error"); - const databaseContentPromises = chains.get(message.chainId)?.databasePromises!; + const databaseContentPromises = chains.get(chainId)?.databasePromises!; const promise: Promise = new Promise((resolve, reject) => { databaseContentPromises.push({ resolve, reject }); }); @@ -198,9 +198,9 @@ return { // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI // layer between JS and Rust, writing unit tests would be extremely complicated. const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. - const converted = (message.maxUtf8BytesSize >= twoPower31) ? - (message.maxUtf8BytesSize - (twoPower31 * 2)) : message.maxUtf8BytesSize; - state.instance.exports.database_content(message.chainId, converted); + const converted = (maxUtf8BytesSize >= twoPower31) ? + (maxUtf8BytesSize - (twoPower31 * 2)) : maxUtf8BytesSize; + state.instance.exports.database_content(chainId, converted); return promise; } From 48a26988208d0780f7dbe99237f73ddbbbb22e38 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 12:01:20 +0200 Subject: [PATCH 20/39] Get rid of messages module --- .../javascript/src/worker/messages.ts | 33 ------------------- bin/wasm-node/javascript/src/worker/worker.ts | 20 +++++++++-- 2 files changed, 18 insertions(+), 35 deletions(-) delete mode 100644 bin/wasm-node/javascript/src/worker/messages.ts diff --git a/bin/wasm-node/javascript/src/worker/messages.ts b/bin/wasm-node/javascript/src/worker/messages.ts deleted file mode 100644 index 3b01c389d1..0000000000 --- a/bin/wasm-node/javascript/src/worker/messages.ts +++ /dev/null @@ -1,33 +0,0 @@ -// Smoldot -// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -/** - * Contains the initial configuration of the worker. - * - * This message is only ever sent once, and it is always the first ever message sent to the - * worker. - */ -export interface ToWorkerConfig { - logCallback: (level: number, target: string, message: string) => void - maxLogLevel: number; - enableCurrentTask: boolean; - cpuRateLimit: number, - forbidTcp: boolean; - forbidWs: boolean; - forbidNonLocalWs: boolean; - forbidWss: boolean; -} diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index c0d43be27f..5280bc018a 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -17,9 +17,25 @@ import { Buffer } from 'buffer'; import * as instance from './raw-instance.js'; -import * as messages from './messages.js'; import { SmoldotWasmInstance } from './bindings.js'; +/** + * Contains the initial configuration of the worker. + * + * This message is only ever sent once, and it is always the first ever message sent to the + * worker. + */ + export interface Config { + logCallback: (level: number, target: string, message: string) => void + maxLogLevel: number; + enableCurrentTask: boolean; + cpuRateLimit: number, + forbidTcp: boolean; + forbidWs: boolean; + forbidNonLocalWs: boolean; + forbidWss: boolean; +} + export interface Worker { request: (request: string, chainId: number) => void addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void) => Promise<{ success: true, chainId: number } | { success: false, error: string }> @@ -27,7 +43,7 @@ export interface Worker { databaseContent: (chainId: number, maxUtf8BytesSize: number) => Promise } -export function start(configMessage: messages.ToWorkerConfig): Worker { +export function start(configMessage: Config): Worker { // This variable represents the state of the worker, and serves two different purposes: // From 2a9130d0d41d6d2617b772088d79425a1d6b97ac Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 12:02:25 +0200 Subject: [PATCH 21/39] Add some console.asserts --- bin/wasm-node/javascript/src/worker/worker.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 5280bc018a..4896834b66 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -161,8 +161,7 @@ return { ); if (instance.exports.chain_is_ok(chainId) != 0) { - if (chains.has(chainId)) // Sanity check. - throw 'Unexpected reuse of a chain ID'; + console.assert(!chains.has(chainId)); chains.set(chainId, { jsonRpcCallback, databasePromises: new Array() @@ -189,6 +188,7 @@ return { // Removing the chain synchronously avoids having to deal with race conditions such as a // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. // These kind of race conditions are already delt with within smoldot. + console.assert(chains.has(chainId)); chains.delete(chainId); state.instance.exports.remove_chain(chainId); }, @@ -200,6 +200,7 @@ return { if (!state.initialized) throw new Error("Internal error"); + console.assert(chains.has(chainId)); const databaseContentPromises = chains.get(chainId)?.databasePromises!; const promise: Promise = new Promise((resolve, reject) => { databaseContentPromises.push({ resolve, reject }); From a590f73069c7d4a8ecc7491f99c84b4d63ba5819 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 12:17:09 +0200 Subject: [PATCH 22/39] No longer track chains in client.ts --- bin/wasm-node/javascript/src/client.ts | 30 ++++++-------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 13bbabed6c..8a6cfabf99 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -363,17 +363,6 @@ export function start(options?: ClientOptions): Client { } }); - // Contains the information of each chain that is currently. - // Entries are instantly removed when the user desires to remove a chain even before the worker - // has confirmed the removal. Doing so avoids a race condition where the worker sends back a - // database content or a JSON-RPC response/notification even though we've already sent a - // `removeChain` message to it. - // - // This map is also used in general as a way to check whether a chain still exists. - let chains: Map = new Map(); - // For each chain object returned by `addChain`, the associated internal chain id. // // Immediately cleared when `remove()` is called on a chain. @@ -457,12 +446,7 @@ export function start(options?: ClientOptions): Client { throw new AddChainError(outcome.error); const chainId = outcome.chainId; - - if (chains.has(chainId)) // Sanity check. - throw 'Unexpected reuse of a chain ID'; - chains.set(chainId, { - jsonRpcCallback: options.jsonRpcCallback - }); + const wasDestroyed = { destroyed: false }; // `expected` was pushed by the `addChain` method. // Resolve the promise that `addChain` returned to the user. @@ -470,9 +454,9 @@ export function start(options?: ClientOptions): Client { sendJsonRpc: (request) => { if (workerError) throw workerError; - if (!chains.has(chainId)) + if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); - if (!(chains.get(chainId)?.jsonRpcCallback)) + if (!options.jsonRpcCallback) throw new JsonRpcDisabledError(); if (request.length >= 8 * 1024 * 1024) return; @@ -481,6 +465,8 @@ export function start(options?: ClientOptions): Client { databaseContent: (maxUtf8BytesSize) => { if (workerError) return Promise.reject(workerError); + if (wasDestroyed.destroyed) + throw new AlreadyDestroyedError(); const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. const maxSize = maxUtf8BytesSize || (twoPower32 - 1); @@ -491,11 +477,9 @@ export function start(options?: ClientOptions): Client { remove: () => { if (workerError) throw workerError; - // Because the `removeChain` message is asynchronous, it is possible for a JSON-RPC - // response or database content concerning that `chainId` to arrive after the `remove` - // function has returned. We solve that by removing the information immediately. - if (!chains.delete(chainId)) + if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); + wasDestroyed.destroyed = true; console.assert(chainIds.has(newChain)); chainIds.delete(newChain); worker.removeChain(chainId); From 338bd00f1a4cc5a3029ef3a38ec963d4678a5da9 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 12:19:46 +0200 Subject: [PATCH 23/39] Move the number handling of databaseContent to worker.ts --- bin/wasm-node/javascript/src/client.ts | 7 +------ bin/wasm-node/javascript/src/worker/worker.ts | 15 ++++++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 8a6cfabf99..aa45e69447 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -467,12 +467,7 @@ export function start(options?: ClientOptions): Client { return Promise.reject(workerError); if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); - - const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. - const maxSize = maxUtf8BytesSize || (twoPower32 - 1); - const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - - return worker.databaseContent(chainId, cappedMaxSize); + return worker.databaseContent(chainId, maxUtf8BytesSize); }, remove: () => { if (workerError) diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/worker/worker.ts index 4896834b66..2840b62eb6 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/worker/worker.ts @@ -40,7 +40,7 @@ export interface Worker { request: (request: string, chainId: number) => void addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void) => Promise<{ success: true, chainId: number } | { success: false, error: string }> removeChain: (chainId: number) => void - databaseContent: (chainId: number, maxUtf8BytesSize: number) => Promise + databaseContent: (chainId: number, maxUtf8BytesSize?: number) => Promise } export function start(configMessage: Config): Worker { @@ -193,7 +193,7 @@ return { state.instance.exports.remove_chain(chainId); }, - databaseContent: (chainId: number, maxUtf8BytesSize: number): Promise => { + databaseContent: (chainId: number, maxUtf8BytesSize?: number): Promise => { // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it // is always the case that the Wasm instance is already initialized. The only possibility for // it to not be the case is if the user completely invented the `chainId`. @@ -206,7 +206,12 @@ return { databaseContentPromises.push({ resolve, reject }); }); - // The value of `maxUtf8BytesSize` is guaranteed (by `index.js`) to always fit in 32 bits, in + // Cap `maxUtf8BytesSize` and set a default value. + const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. + const maxSize = maxUtf8BytesSize || (twoPower32 - 1); + const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; + + // The value of `maxUtf8BytesSize` is guaranteed to always fit in 32 bits, in // other words, that `maxUtf8BytesSize < (1 << 32)`. // We need to perform a conversion in such a way that the the bits of the output of // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. @@ -215,8 +220,8 @@ return { // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI // layer between JS and Rust, writing unit tests would be extremely complicated. const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. - const converted = (maxUtf8BytesSize >= twoPower31) ? - (maxUtf8BytesSize - (twoPower31 * 2)) : maxUtf8BytesSize; + const converted = (cappedMaxSize >= twoPower31) ? + (cappedMaxSize - twoPower32) : cappedMaxSize; state.instance.exports.database_content(chainId, converted); return promise; From 3aeba7741ea98846c782c346cb25ab70df47ea7a Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 11 Jul 2022 12:23:15 +0200 Subject: [PATCH 24/39] Rename worker to instance --- bin/wasm-node/javascript/prepare.js | 10 ++--- bin/wasm-node/javascript/src/client.ts | 44 +++++++++---------- .../{worker => instance}/autogen/.gitignore | 0 .../bindings-smoldot-light.ts | 0 .../src/{worker => instance}/bindings-wasi.ts | 0 .../src/{worker => instance}/bindings.ts | 0 .../src/{worker => instance}/connection.ts | 0 .../worker.ts => instance/instance.ts} | 11 ++--- .../src/{worker => instance}/raw-instance.ts | 0 9 files changed, 30 insertions(+), 35 deletions(-) rename bin/wasm-node/javascript/src/{worker => instance}/autogen/.gitignore (100%) rename bin/wasm-node/javascript/src/{worker => instance}/bindings-smoldot-light.ts (100%) rename bin/wasm-node/javascript/src/{worker => instance}/bindings-wasi.ts (100%) rename bin/wasm-node/javascript/src/{worker => instance}/bindings.ts (100%) rename bin/wasm-node/javascript/src/{worker => instance}/connection.ts (100%) rename bin/wasm-node/javascript/src/{worker/worker.ts => instance/instance.ts} (96%) rename bin/wasm-node/javascript/src/{worker => instance}/raw-instance.ts (100%) diff --git a/bin/wasm-node/javascript/prepare.js b/bin/wasm-node/javascript/prepare.js index f410aa2cf5..42acf56d49 100755 --- a/bin/wasm-node/javascript/prepare.js +++ b/bin/wasm-node/javascript/prepare.js @@ -69,13 +69,13 @@ child_process.execSync( { 'stdio': 'inherit' } ); -// The code below will write a variable number of files to the `src/worker/autogen` directory. +// The code below will write a variable number of files to the `src/instance/autogen` directory. // Start by clearing all existing files from this directory in case there are some left from past // builds. -const filesToRemove = fs.readdirSync('./src/worker/autogen'); +const filesToRemove = fs.readdirSync('./src/instance/autogen'); for (const file of filesToRemove) { if (!file.startsWith('.')) // Don't want to remove the `.gitignore` or `.npmignore` or similar - fs.unlinkSync(path.join("./src/worker/autogen", file)); + fs.unlinkSync(path.join("./src/instance/autogen", file)); } // We then do an optimization pass on the Wasm file, using `wasm-opt`. @@ -119,13 +119,13 @@ try { const chunk = base64Data.slice(0, 1024 * 1024); // We could simply export the chunk instead of a function that returns the chunk, but that // would cause TypeScript to generate a definitions file containing a copy of the entire chunk. - fs.writeFileSync('./src/worker/autogen/wasm' + fileNum + '.ts', 'export default function(): string { return "' + chunk + '"; }'); + fs.writeFileSync('./src/instance/autogen/wasm' + fileNum + '.ts', 'export default function(): string { return "' + chunk + '"; }'); imports += 'import { default as wasm' + fileNum + ' } from \'./wasm' + fileNum + '.js\';\n'; chunksSum += ' + wasm' + fileNum + '()'; fileNum += 1; base64Data = base64Data.slice(1024 * 1024); } - fs.writeFileSync('./src/worker/autogen/wasm.ts', imports + 'export default ' + chunksSum + ';'); + fs.writeFileSync('./src/instance/autogen/wasm.ts', imports + 'export default ' + chunksSum + ';'); } finally { fs.rmSync(tmpDir, { recursive: true }); diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index aa45e69447..69720b3882 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -import { start as startWorker } from './worker/worker.js'; +import { start as startWorker } from './instance/instance.js'; /** * Thrown in case of a problem when initializing the chain. @@ -368,10 +368,8 @@ export function start(options?: ClientOptions): Client { // Immediately cleared when `remove()` is called on a chain. let chainIds: WeakMap = new WeakMap(); - // The actual execution of Smoldot is performed in a worker thread. - // Because this specific line of code is a bit sensitive, it is done in a separate file. - let workerError: null | Error = null; - const worker = startWorker({ + let instanceError: null | Error = null; + const instance = startWorker({ // Maximum level of log entries sent by the client. // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace maxLogLevel: options.maxLogLevel || 3, @@ -388,8 +386,8 @@ export function start(options?: ClientOptions): Client { }); // TODO: restore - /*workerOnError(worker, (error) => { - // A worker error should only happen in case of a critical error as the result of a bug + /*workerOnError(instance, (error) => { + // An instance error should only happen in case of a critical error as the result of a bug // somewhere. Consequently, nothing is really in place to cleanly report the error. const errorToString = error.toString(); // TODO: restore after having updated `workerCurrentTask` @@ -420,8 +418,8 @@ export function start(options?: ClientOptions): Client { return { addChain: async (options: AddChainOptions): Promise => { - if (workerError) - throw workerError; + if (instanceError) + throw instanceError; // Passing a JSON object for the chain spec is an easy mistake, so we provide a more // readable error. @@ -440,7 +438,7 @@ export function start(options?: ClientOptions): Client { } } - const outcome = await worker.addChain(options.chainSpec, typeof options.databaseContent === 'string' ? options.databaseContent : "", potentialRelayChainsIds, options.jsonRpcCallback); + const outcome = await instance.addChain(options.chainSpec, typeof options.databaseContent === 'string' ? options.databaseContent : "", potentialRelayChainsIds, options.jsonRpcCallback); if (!outcome.success) throw new AddChainError(outcome.error); @@ -452,32 +450,32 @@ export function start(options?: ClientOptions): Client { // Resolve the promise that `addChain` returned to the user. const newChain: Chain = { sendJsonRpc: (request) => { - if (workerError) - throw workerError; + if (instanceError) + throw instanceError; if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); if (!options.jsonRpcCallback) throw new JsonRpcDisabledError(); if (request.length >= 8 * 1024 * 1024) return; - worker.request(request, chainId); + instance.request(request, chainId); }, databaseContent: (maxUtf8BytesSize) => { - if (workerError) - return Promise.reject(workerError); + if (instanceError) + return Promise.reject(instanceError); if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); - return worker.databaseContent(chainId, maxUtf8BytesSize); + return instance.databaseContent(chainId, maxUtf8BytesSize); }, remove: () => { - if (workerError) - throw workerError; + if (instanceError) + throw instanceError; if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); wasDestroyed.destroyed = true; console.assert(chainIds.has(newChain)); chainIds.delete(newChain); - worker.removeChain(chainId); + instance.removeChain(chainId); }, }; @@ -485,11 +483,11 @@ export function start(options?: ClientOptions): Client { return newChain; }, terminate: async () => { - if (workerError) - return Promise.reject(workerError) - workerError = new AlreadyDestroyedError(); + if (instanceError) + return Promise.reject(instanceError) + instanceError = new AlreadyDestroyedError(); // TODO: restore - //return workerTerminate(worker) + //return workerTerminate(instance) } } } diff --git a/bin/wasm-node/javascript/src/worker/autogen/.gitignore b/bin/wasm-node/javascript/src/instance/autogen/.gitignore similarity index 100% rename from bin/wasm-node/javascript/src/worker/autogen/.gitignore rename to bin/wasm-node/javascript/src/instance/autogen/.gitignore diff --git a/bin/wasm-node/javascript/src/worker/bindings-smoldot-light.ts b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/bindings-smoldot-light.ts rename to bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts diff --git a/bin/wasm-node/javascript/src/worker/bindings-wasi.ts b/bin/wasm-node/javascript/src/instance/bindings-wasi.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/bindings-wasi.ts rename to bin/wasm-node/javascript/src/instance/bindings-wasi.ts diff --git a/bin/wasm-node/javascript/src/worker/bindings.ts b/bin/wasm-node/javascript/src/instance/bindings.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/bindings.ts rename to bin/wasm-node/javascript/src/instance/bindings.ts diff --git a/bin/wasm-node/javascript/src/worker/connection.ts b/bin/wasm-node/javascript/src/instance/connection.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/connection.ts rename to bin/wasm-node/javascript/src/instance/connection.ts diff --git a/bin/wasm-node/javascript/src/worker/worker.ts b/bin/wasm-node/javascript/src/instance/instance.ts similarity index 96% rename from bin/wasm-node/javascript/src/worker/worker.ts rename to bin/wasm-node/javascript/src/instance/instance.ts index 2840b62eb6..dcaca3c5bf 100644 --- a/bin/wasm-node/javascript/src/worker/worker.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -20,10 +20,7 @@ import * as instance from './raw-instance.js'; import { SmoldotWasmInstance } from './bindings.js'; /** - * Contains the initial configuration of the worker. - * - * This message is only ever sent once, and it is always the first ever message sent to the - * worker. + * Contains the configuration of the instance. */ export interface Config { logCallback: (level: number, target: string, message: string) => void @@ -36,16 +33,16 @@ import { SmoldotWasmInstance } from './bindings.js'; forbidWss: boolean; } -export interface Worker { +export interface Instance { request: (request: string, chainId: number) => void addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void) => Promise<{ success: true, chainId: number } | { success: false, error: string }> removeChain: (chainId: number) => void databaseContent: (chainId: number, maxUtf8BytesSize?: number) => Promise } -export function start(configMessage: Config): Worker { +export function start(configMessage: Config): Instance { -// This variable represents the state of the worker, and serves two different purposes: +// This variable represents the state of the instance, and serves two different purposes: // // - At initialization, it is a Promise containing the Wasm VM is still initializing. // - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. diff --git a/bin/wasm-node/javascript/src/worker/raw-instance.ts b/bin/wasm-node/javascript/src/instance/raw-instance.ts similarity index 100% rename from bin/wasm-node/javascript/src/worker/raw-instance.ts rename to bin/wasm-node/javascript/src/instance/raw-instance.ts From b31d4630f643d663c61ab0273e45bcad8bc8d0df Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 13:28:44 +0200 Subject: [PATCH 25/39] Pass onProcExit rather than throwing directly --- bin/wasm-node/javascript/src/instance/bindings-wasi.ts | 9 ++++++++- bin/wasm-node/javascript/src/instance/raw-instance.ts | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/wasm-node/javascript/src/instance/bindings-wasi.ts b/bin/wasm-node/javascript/src/instance/bindings-wasi.ts index 915435f501..17b6c89c4b 100644 --- a/bin/wasm-node/javascript/src/instance/bindings-wasi.ts +++ b/bin/wasm-node/javascript/src/instance/bindings-wasi.ts @@ -37,6 +37,13 @@ export interface Config { * Must never be modified after the bindings have been initialized. */ envVars: string[], + + /** + * Closure to call when the Wasm instance calls `proc_exit`. + * + * This callback will always be invoked from within a binding called the Wasm instance. + */ + onProcExit: (retCode: number) => never, } export default (config: Config): compat.WasmModuleImports => { @@ -130,7 +137,7 @@ export default (config: Config): compat.WasmModuleImports => { // Used by Rust in catastrophic situations, such as a double panic. proc_exit: (retCode: number) => { - throw new Error(`proc_exit called: ${retCode}`); + config.onProcExit(retCode) }, // Return the number of environment variables and the total size of all environment diff --git a/bin/wasm-node/javascript/src/instance/raw-instance.ts b/bin/wasm-node/javascript/src/instance/raw-instance.ts index 36579a9107..1731f4c70c 100644 --- a/bin/wasm-node/javascript/src/instance/raw-instance.ts +++ b/bin/wasm-node/javascript/src/instance/raw-instance.ts @@ -53,6 +53,7 @@ export async function startInstance(config: Config): Promise { throw new Error(`proc_exit called: ${retCode}`); } }; // Start the Wasm virtual machine. From dea8db8311c03050a723fe9a6fbc4c18f65fe62d Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 14:15:20 +0200 Subject: [PATCH 26/39] Add onPanic to smoldot bindings --- .../javascript/src/instance/bindings-smoldot-light.ts | 10 +++++++++- bin/wasm-node/javascript/src/instance/raw-instance.ts | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts index 5feabf0134..4552323a9e 100644 --- a/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts +++ b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts @@ -27,6 +27,14 @@ import type { SmoldotWasmInstance } from './bindings.js'; export interface Config { instance?: SmoldotWasmInstance, + + /** + * Closure to call when the Wasm instance calls `panic`. + * + * This callback will always be invoked from within a binding called the Wasm instance. + */ + onPanic: (message: string) => never, + logCallback: (level: number, target: string, message: string) => void, jsonRpcCallback: (response: string, chainId: number) => void, databaseContentCallback: (data: string, chainId: number) => void, @@ -52,7 +60,7 @@ export default function (config: Config): compat.WasmModuleImports { len >>>= 0; const message = Buffer.from(instance.exports.memory.buffer).toString('utf8', ptr, ptr + len); - throw new Error(message); + config.onPanic(message); }, // Used by the Rust side to emit a JSON-RPC response or subscription notification. diff --git a/bin/wasm-node/javascript/src/instance/raw-instance.ts b/bin/wasm-node/javascript/src/instance/raw-instance.ts index 1731f4c70c..2f9e05b745 100644 --- a/bin/wasm-node/javascript/src/instance/raw-instance.ts +++ b/bin/wasm-node/javascript/src/instance/raw-instance.ts @@ -47,6 +47,9 @@ export async function startInstance(config: Config): Promise { + throw new Error(message); + }, ...config }; From 2476fa4745cb110a39cff62b5df1197c81f12c27 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 14:47:47 +0200 Subject: [PATCH 27/39] Clarify panic handling situation --- .../src/instance/bindings-smoldot-light.ts | 71 +++++++++++++++---- .../javascript/src/instance/instance.ts | 17 ++++- .../javascript/src/instance/raw-instance.ts | 33 +++++++-- 3 files changed, 103 insertions(+), 18 deletions(-) diff --git a/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts index 4552323a9e..7c3fae321a 100644 --- a/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts +++ b/bin/wasm-node/javascript/src/instance/bindings-smoldot-light.ts @@ -45,12 +45,25 @@ export interface Config { forbidWss: boolean, } -export default function (config: Config): compat.WasmModuleImports { +export default function (config: Config): { imports: compat.WasmModuleImports, killAll: () => void } { // Used below to store the list of all connections. // The indices within this array are chosen by the Rust code. let connections: Record = {}; - return { + // Object containing a boolean indicating whether the `killAll` function has been invoked by + // the user. + const killedTracked = { killed: false }; + + const killAll = () => { + killedTracked.killed = true; + // TODO: kill timers as well? + for (const connection in connections) { + connections[connection]!.close() + delete connections[connection] + } + }; + + const imports = { // Must exit with an error. A human-readable message can be found in the WebAssembly // memory in the given buffer. panic: (ptr: number, len: number) => { @@ -65,6 +78,8 @@ export default function (config: Config): compat.WasmModuleImports { // Used by the Rust side to emit a JSON-RPC response or subscription notification. json_rpc_respond: (ptr: number, len: number, chainId: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -78,6 +93,8 @@ export default function (config: Config): compat.WasmModuleImports { // Used by the Rust side in response to asking for the database content of a chain. database_content_ready: (ptr: number, len: number, chainId: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -92,6 +109,8 @@ export default function (config: Config): compat.WasmModuleImports { // Used by the Rust side to emit a log entry. // See also the `max_log_level` parameter in the configuration. log: (level: number, targetPtr: number, targetLen: number, messagePtr: number, messageLen: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; targetPtr >>>= 0; @@ -116,6 +135,8 @@ export default function (config: Config): compat.WasmModuleImports { // Must call `timer_finished` after the given number of milliseconds has elapsed. start_timer: (id: number, ms: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; // In both NodeJS and browsers, if `setTimeout` is called with a value larger than @@ -130,11 +151,17 @@ export default function (config: Config): compat.WasmModuleImports { // with `1`) and wants you to use `setImmediate` instead. if (ms == 0 && typeof setImmediate === "function") { setImmediate(() => { - instance.exports.timer_finished(id); + if (killedTracked.killed) return; + try { + instance.exports.timer_finished(id); + } catch(_error) {} }) } else { setTimeout(() => { - instance.exports.timer_finished(id); + if (killedTracked.killed) return; + try { + instance.exports.timer_finished(id); + } catch(_error) {} }, ms) } }, @@ -153,6 +180,9 @@ export default function (config: Config): compat.WasmModuleImports { } try { + if (killedTracked.killed) + throw new Error("killAll invoked"); + const address = Buffer.from(instance.exports.memory.buffer) .toString('utf8', addrPtr, addrPtr + addrLen); @@ -163,18 +193,27 @@ export default function (config: Config): compat.WasmModuleImports { forbidNonLocalWs: config.forbidNonLocalWs, forbidWss: config.forbidWss, onOpen: () => { - instance.exports.connection_open_single_stream(connectionId); + if (killedTracked.killed) return; + try { + instance.exports.connection_open_single_stream(connectionId); + } catch(_error) {} }, onClose: (message: string) => { - const len = Buffer.byteLength(message, 'utf8'); - const ptr = instance.exports.alloc(len) >>> 0; - Buffer.from(instance.exports.memory.buffer).write(message, ptr); - instance.exports.connection_closed(connectionId, ptr, len); + if (killedTracked.killed) return; + try { + const len = Buffer.byteLength(message, 'utf8'); + const ptr = instance.exports.alloc(len) >>> 0; + Buffer.from(instance.exports.memory.buffer).write(message, ptr); + instance.exports.connection_closed(connectionId, ptr, len); + } catch(_error) {} }, onMessage: (message: Buffer) => { - const ptr = instance.exports.alloc(message.length) >>> 0; - message.copy(Buffer.from(instance.exports.memory.buffer), ptr); - instance.exports.stream_message(connectionId, 0, ptr, message.length); + if (killedTracked.killed) return; + try { + const ptr = instance.exports.alloc(message.length) >>> 0; + message.copy(Buffer.from(instance.exports.memory.buffer), ptr); + instance.exports.stream_message(connectionId, 0, ptr, message.length); + } catch(_error) {} } }); @@ -200,6 +239,7 @@ export default function (config: Config): compat.WasmModuleImports { // Must close and destroy the connection object. connection_close: (connectionId: number) => { + if (killedTracked.killed) return; const connection = connections[connectionId]!; connection.close(); delete connections[connectionId]; @@ -220,6 +260,8 @@ export default function (config: Config): compat.WasmModuleImports { // Must queue the data found in the WebAssembly memory at the given pointer. It is assumed // that this function is called only when the connection is in an open state. stream_send: (connectionId: number, _streamId: number, ptr: number, len: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -231,6 +273,8 @@ export default function (config: Config): compat.WasmModuleImports { }, current_task_entered: (ptr: number, len: number) => { + if (killedTracked.killed) return; + const instance = config.instance!; ptr >>>= 0; @@ -242,8 +286,11 @@ export default function (config: Config): compat.WasmModuleImports { }, current_task_exit: () => { + if (killedTracked.killed) return; if (config.currentTaskCallback) config.currentTaskCallback(null); } }; + + return { imports, killAll } } diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index dcaca3c5bf..045534b761 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -49,6 +49,8 @@ export function start(configMessage: Config): Instance { // let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; +const workerCurrentTask: { name: string | null } = { name: null }; + // Contains the information of each chain that is currently alive. let chains: Map void, @@ -57,6 +59,17 @@ let state: { initialized: false, promise: Promise } | { ini // Start initialization of the Wasm VM. const config: instance.Config = { + onWasmPanic: (message) => { + // TODO: must remember that a panic happened and not invoke any other function + // TODO: must add try-catch blocks when invoking Wasm VM + console.error( + "Smoldot has panicked" + + (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + + ". This is a bug in smoldot. Please open an issue at " + + "https://github.com/paritytech/smoldot/issues with the following message:\n" + + message + ); + }, logCallback: (level, target, message) => { configMessage.logCallback(level, target, message) }, @@ -68,8 +81,8 @@ let state: { initialized: false, promise: Promise } | { ini const promises = chains.get(chainId)?.databasePromises!; (promises.shift() as DatabasePromise).resolve(data); }, - currentTaskCallback: (_taskName) => { - // TODO: do something here? + currentTaskCallback: (taskName) => { + workerCurrentTask.name = taskName }, cpuRateLimit: configMessage.cpuRateLimit, forbidTcp: configMessage.forbidTcp, diff --git a/bin/wasm-node/javascript/src/instance/raw-instance.ts b/bin/wasm-node/javascript/src/instance/raw-instance.ts index 2f9e05b745..a21487e1b0 100644 --- a/bin/wasm-node/javascript/src/instance/raw-instance.ts +++ b/bin/wasm-node/javascript/src/instance/raw-instance.ts @@ -19,7 +19,7 @@ import pako from 'pako'; import { Buffer } from 'buffer'; -import { Config as SmoldotBindingsConfig, default as smolditLightBindingsBuilder } from './bindings-smoldot-light.js'; +import { Config as SmoldotBindingsConfig, default as smoldotLightBindingsBuilder } from './bindings-smoldot-light.js'; import { Config as WasiConfig, default as wasiBindingsBuilder } from './bindings-wasi.js'; import { default as wasmBase64 } from './autogen/wasm.js'; @@ -27,6 +27,18 @@ import { default as wasmBase64 } from './autogen/wasm.js'; import { SmoldotWasmInstance } from './bindings.js'; export interface Config { + /** + * Closure to call when the Wasm instance panics. + * + * This callback will always be invoked from within a binding called the Wasm instance. + * + * After this callback has been called, it is forbidden to invoke any function from the Wasm + * VM. + * + * If this callback is called while invoking a function from the Wasm VM, this function will + * throw a dummy exception. + */ + onWasmPanic: (message: string) => void, logCallback: (level: number, target: string, message: string) => void, jsonRpcCallback: (response: string, chainId: number) => void, databaseContentCallback: (data: string, chainId: number) => void, @@ -45,10 +57,14 @@ export async function startInstance(config: Config): Promise void; + // Used to bind with the smoldot-light bindings. See the `bindings-smoldot-light.js` file. const smoldotJsConfig: SmoldotBindingsConfig = { onPanic: (message) => { - throw new Error(message); + killAll(); + config.onWasmPanic(message); + throw new Error(); }, ...config }; @@ -56,15 +72,24 @@ export async function startInstance(config: Config): Promise { throw new Error(`proc_exit called: ${retCode}`); } + onProcExit: (retCode) => { + killAll(); + config.onWasmPanic(`proc_exit called: ${retCode}`) + throw new Error(); + } }; + const { imports: smoldotBindings, killAll: smoldotBindingsKillAll } = + smoldotLightBindingsBuilder(smoldotJsConfig); + + killAll = smoldotBindingsKillAll; + // Start the Wasm virtual machine. // The Rust code defines a list of imports that must be fulfilled by the environment. The second // parameter provides their implementations. const result = await WebAssembly.instantiate(wasmBytecode, { // The functions with the "smoldot" prefix are specific to smoldot. - "smoldot": smolditLightBindingsBuilder(smoldotJsConfig), + "smoldot": smoldotBindings, // As the Rust code is compiled for wasi, some more wasi-specific imports exist. "wasi_snapshot_preview1": wasiBindingsBuilder(wasiConfig), }); From 77ab5a7da9f811901112dcccec0c26742ad65613 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:37:51 +0200 Subject: [PATCH 28/39] Finally properly handle crashes --- .../javascript/src/instance/instance.ts | 48 +++++++++++++++---- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index 045534b761..4b2987cd3d 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -18,6 +18,7 @@ import { Buffer } from 'buffer'; import * as instance from './raw-instance.js'; import { SmoldotWasmInstance } from './bindings.js'; +import { CrashError } from '../client.js'; /** * Contains the configuration of the instance. @@ -49,6 +50,8 @@ export function start(configMessage: Config): Instance { // let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; +const crashError: { error?: CrashError } = {}; + const workerCurrentTask: { name: string | null } = { name: null }; // Contains the information of each chain that is currently alive. @@ -60,8 +63,8 @@ const workerCurrentTask: { name: string | null } = { name: null }; // Start initialization of the Wasm VM. const config: instance.Config = { onWasmPanic: (message) => { - // TODO: must remember that a panic happened and not invoke any other function - // TODO: must add try-catch blocks when invoking Wasm VM + // TODO: consider obtaining a backtrace here + crashError.error = new CrashError(message); console.error( "Smoldot has panicked" + (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + @@ -130,15 +133,26 @@ return { // it to not be the case is if the user completely invented the `chainId`. if (!state.initialized) throw new Error("Internal error"); - - const len = Buffer.byteLength(request, 'utf8'); - const ptr = state.instance.exports.alloc(len) >>> 0; - Buffer.from(state.instance.exports.memory.buffer).write(request, ptr); - state.instance.exports.json_rpc_send(ptr, len, chainId); + if (crashError.error) + throw crashError.error; + + try { + const len = Buffer.byteLength(request, 'utf8'); + const ptr = state.instance.exports.alloc(len) >>> 0; + Buffer.from(state.instance.exports.memory.buffer).write(request, ptr); + state.instance.exports.json_rpc_send(ptr, len, chainId); + } catch(_error) { + console.assert(crashError.error); + throw crashError.error + } }, addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { return queueOperation((instance) => { + if (crashError.error) + throw crashError.error; + + try { // Write the chain specification into memory. const chainSpecLen = Buffer.byteLength(chainSpec, 'utf8'); const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; @@ -185,6 +199,10 @@ return { instance.exports.remove_chain(chainId); return { success: false, error: errorMsg }; } + } catch(_error) { + console.assert(crashError.error); + throw crashError.error + } }) }, @@ -200,7 +218,12 @@ return { // These kind of race conditions are already delt with within smoldot. console.assert(chains.has(chainId)); chains.delete(chainId); - state.instance.exports.remove_chain(chainId); + try { + state.instance.exports.remove_chain(chainId); + } catch(_error) { + console.assert(crashError.error); + throw crashError.error + } }, databaseContent: (chainId: number, maxUtf8BytesSize?: number): Promise => { @@ -232,9 +255,14 @@ return { const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. const converted = (cappedMaxSize >= twoPower31) ? (cappedMaxSize - twoPower32) : cappedMaxSize; - state.instance.exports.database_content(chainId, converted); - return promise; + try { + state.instance.exports.database_content(chainId, converted); + return promise; + } catch(_error) { + console.assert(crashError.error); + throw crashError.error + } } } From 750f848232376e2b46fe2e5716145b5b03c64a04 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:42:17 +0200 Subject: [PATCH 29/39] Properly format instance.ts --- .../javascript/src/instance/instance.ts | 374 +++++++++--------- 1 file changed, 188 insertions(+), 186 deletions(-) diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index 4b2987cd3d..49fec760ed 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -23,7 +23,7 @@ import { CrashError } from '../client.js'; /** * Contains the configuration of the instance. */ - export interface Config { +export interface Config { logCallback: (level: number, target: string, message: string) => void maxLogLevel: number; enableCurrentTask: boolean; @@ -43,16 +43,16 @@ export interface Instance { export function start(configMessage: Config): Instance { -// This variable represents the state of the instance, and serves two different purposes: -// -// - At initialization, it is a Promise containing the Wasm VM is still initializing. -// - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. -// -let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; + // This variable represents the state of the instance, and serves two different purposes: + // + // - At initialization, it is a Promise containing the Wasm VM is still initializing. + // - After the Wasm VM has finished initialization, contains the `WebAssembly.Instance` object. + // + let state: { initialized: false, promise: Promise } | { initialized: true, instance: SmoldotWasmInstance }; -const crashError: { error?: CrashError } = {}; + const crashError: { error?: CrashError } = {}; -const workerCurrentTask: { name: string | null } = { name: null }; + const workerCurrentTask: { name: string | null } = { name: null }; // Contains the information of each chain that is currently alive. let chains: Map = new Map(); - // Start initialization of the Wasm VM. - const config: instance.Config = { - onWasmPanic: (message) => { - // TODO: consider obtaining a backtrace here - crashError.error = new CrashError(message); - console.error( - "Smoldot has panicked" + - (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + - ". This is a bug in smoldot. Please open an issue at " + - "https://github.com/paritytech/smoldot/issues with the following message:\n" + - message - ); - }, - logCallback: (level, target, message) => { - configMessage.logCallback(level, target, message) - }, - jsonRpcCallback: (data, chainId) => { - const cb = chains.get(chainId)?.jsonRpcCallback; - if (cb) cb(data); - }, - databaseContentCallback: (data, chainId) => { - const promises = chains.get(chainId)?.databasePromises!; - (promises.shift() as DatabasePromise).resolve(data); - }, - currentTaskCallback: (taskName) => { - workerCurrentTask.name = taskName - }, - cpuRateLimit: configMessage.cpuRateLimit, - forbidTcp: configMessage.forbidTcp, - forbidWs: configMessage.forbidWs, - forbidNonLocalWs: configMessage.forbidNonLocalWs, - forbidWss: configMessage.forbidWss, - }; - - state = { initialized: false, promise: instance.startInstance(config).then((instance) => { + // Start initialization of the Wasm VM. + const config: instance.Config = { + onWasmPanic: (message) => { + // TODO: consider obtaining a backtrace here + crashError.error = new CrashError(message); + console.error( + "Smoldot has panicked" + + (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + + ". This is a bug in smoldot. Please open an issue at " + + "https://github.com/paritytech/smoldot/issues with the following message:\n" + + message + ); + }, + logCallback: (level, target, message) => { + configMessage.logCallback(level, target, message) + }, + jsonRpcCallback: (data, chainId) => { + const cb = chains.get(chainId)?.jsonRpcCallback; + if (cb) cb(data); + }, + databaseContentCallback: (data, chainId) => { + const promises = chains.get(chainId)?.databasePromises!; + (promises.shift() as DatabasePromise).resolve(data); + }, + currentTaskCallback: (taskName) => { + workerCurrentTask.name = taskName + }, + cpuRateLimit: configMessage.cpuRateLimit, + forbidTcp: configMessage.forbidTcp, + forbidWs: configMessage.forbidWs, + forbidNonLocalWs: configMessage.forbidNonLocalWs, + forbidWss: configMessage.forbidWss, + }; + + state = { + initialized: false, promise: instance.startInstance(config).then((instance) => { // `config.cpuRateLimit` is a floating point that should be between 0 and 1, while the value // to pass as parameter must be between `0` and `2^32-1`. // The few lines of code below should handle all possible values of `number`, including @@ -110,161 +111,162 @@ const workerCurrentTask: { name: string | null } = { name: null }; state = { initialized: true, instance }; return instance; - }) }; - -async function queueOperation(operation: (instance: SmoldotWasmInstance) => T): Promise { - // What to do depends on the type of `state`. - // See the documentation of the `state` variable for information. - if (!state.initialized) { - // A message has been received while the Wasm VM is still initializing. Queue it for when - // initialization is over. - return state.promise.then((instance) => operation(instance)) - - } else { - // Everything is already initialized. Process the message synchronously. - return operation(state.instance) - } -} - -return { - request: (request: string, chainId: number) => { - // Because `request` is passed as parameter an identifier returned by `addChain`, it is - // always the case that the Wasm instance is already initialized. The only possibility for - // it to not be the case is if the user completely invented the `chainId`. - if (!state.initialized) - throw new Error("Internal error"); - if (crashError.error) - throw crashError.error; - - try { - const len = Buffer.byteLength(request, 'utf8'); - const ptr = state.instance.exports.alloc(len) >>> 0; - Buffer.from(state.instance.exports.memory.buffer).write(request, ptr); - state.instance.exports.json_rpc_send(ptr, len, chainId); - } catch(_error) { - console.assert(crashError.error); - throw crashError.error + }) + }; + + async function queueOperation(operation: (instance: SmoldotWasmInstance) => T): Promise { + // What to do depends on the type of `state`. + // See the documentation of the `state` variable for information. + if (!state.initialized) { + // A message has been received while the Wasm VM is still initializing. Queue it for when + // initialization is over. + return state.promise.then((instance) => operation(instance)) + + } else { + // Everything is already initialized. Process the message synchronously. + return operation(state.instance) } - }, + } - addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { - return queueOperation((instance) => { + return { + request: (request: string, chainId: number) => { + // Because `request` is passed as parameter an identifier returned by `addChain`, it is + // always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); if (crashError.error) throw crashError.error; try { - // Write the chain specification into memory. - const chainSpecLen = Buffer.byteLength(chainSpec, 'utf8'); - const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; - Buffer.from(instance.exports.memory.buffer) - .write(chainSpec, chainSpecPtr); - - // Write the database content into memory. - const databaseContentLen = Buffer.byteLength(databaseContent, 'utf8'); - const databaseContentPtr = instance.exports.alloc(databaseContentLen) >>> 0; - Buffer.from(instance.exports.memory.buffer) - .write(databaseContent, databaseContentPtr); - - // Write the potential relay chains into memory. - const potentialRelayChainsLen = potentialRelayChains.length; - const potentialRelayChainsPtr = instance.exports.alloc(potentialRelayChainsLen * 4) >>> 0; - for (let idx = 0; idx < potentialRelayChains.length; ++idx) { - Buffer.from(instance.exports.memory.buffer) - .writeUInt32LE(potentialRelayChains[idx]!, potentialRelayChainsPtr + idx * 4); + const len = Buffer.byteLength(request, 'utf8'); + const ptr = state.instance.exports.alloc(len) >>> 0; + Buffer.from(state.instance.exports.memory.buffer).write(request, ptr); + state.instance.exports.json_rpc_send(ptr, len, chainId); + } catch (_error) { + console.assert(crashError.error); + throw crashError.error } - - // `add_chain` unconditionally allocates a chain id. If an error occurs, however, this chain - // id will refer to an *erroneous* chain. `chain_is_ok` is used below to determine whether it - // has succeeeded or not. - // Note that `add_chain` properly de-allocates buffers even if it failed. - const chainId = instance.exports.add_chain( - chainSpecPtr, chainSpecLen, - databaseContentPtr, databaseContentLen, - !!jsonRpcCallback ? 1 : 0, - potentialRelayChainsPtr, potentialRelayChainsLen - ); - - if (instance.exports.chain_is_ok(chainId) != 0) { - console.assert(!chains.has(chainId)); - chains.set(chainId, { - jsonRpcCallback, - databasePromises: new Array() - }); - return { success: true, chainId }; - } else { - const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; - const errorMsgPtr = instance.exports.chain_error_ptr(chainId) >>> 0; - const errorMsg = Buffer.from(instance.exports.memory.buffer) - .toString('utf8', errorMsgPtr, errorMsgPtr + errorMsgLen); - instance.exports.remove_chain(chainId); - return { success: false, error: errorMsg }; + }, + + addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void): Promise<{ success: true, chainId: number } | { success: false, error: string }> => { + return queueOperation((instance) => { + if (crashError.error) + throw crashError.error; + + try { + // Write the chain specification into memory. + const chainSpecLen = Buffer.byteLength(chainSpec, 'utf8'); + const chainSpecPtr = instance.exports.alloc(chainSpecLen) >>> 0; + Buffer.from(instance.exports.memory.buffer) + .write(chainSpec, chainSpecPtr); + + // Write the database content into memory. + const databaseContentLen = Buffer.byteLength(databaseContent, 'utf8'); + const databaseContentPtr = instance.exports.alloc(databaseContentLen) >>> 0; + Buffer.from(instance.exports.memory.buffer) + .write(databaseContent, databaseContentPtr); + + // Write the potential relay chains into memory. + const potentialRelayChainsLen = potentialRelayChains.length; + const potentialRelayChainsPtr = instance.exports.alloc(potentialRelayChainsLen * 4) >>> 0; + for (let idx = 0; idx < potentialRelayChains.length; ++idx) { + Buffer.from(instance.exports.memory.buffer) + .writeUInt32LE(potentialRelayChains[idx]!, potentialRelayChainsPtr + idx * 4); + } + + // `add_chain` unconditionally allocates a chain id. If an error occurs, however, this chain + // id will refer to an *erroneous* chain. `chain_is_ok` is used below to determine whether it + // has succeeeded or not. + // Note that `add_chain` properly de-allocates buffers even if it failed. + const chainId = instance.exports.add_chain( + chainSpecPtr, chainSpecLen, + databaseContentPtr, databaseContentLen, + !!jsonRpcCallback ? 1 : 0, + potentialRelayChainsPtr, potentialRelayChainsLen + ); + + if (instance.exports.chain_is_ok(chainId) != 0) { + console.assert(!chains.has(chainId)); + chains.set(chainId, { + jsonRpcCallback, + databasePromises: new Array() + }); + return { success: true, chainId }; + } else { + const errorMsgLen = instance.exports.chain_error_len(chainId) >>> 0; + const errorMsgPtr = instance.exports.chain_error_ptr(chainId) >>> 0; + const errorMsg = Buffer.from(instance.exports.memory.buffer) + .toString('utf8', errorMsgPtr, errorMsgPtr + errorMsgLen); + instance.exports.remove_chain(chainId); + return { success: false, error: errorMsg }; + } + } catch (_error) { + console.assert(crashError.error); + throw crashError.error + } + }) + }, + + removeChain: (chainId: number) => { + // Because `removeChain` is passed as parameter an identifier returned by `addChain`, it is + // always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + + // Removing the chain synchronously avoids having to deal with race conditions such as a + // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. + // These kind of race conditions are already delt with within smoldot. + console.assert(chains.has(chainId)); + chains.delete(chainId); + try { + state.instance.exports.remove_chain(chainId); + } catch (_error) { + console.assert(crashError.error); + throw crashError.error } - } catch(_error) { + }, + + databaseContent: (chainId: number, maxUtf8BytesSize?: number): Promise => { + // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it + // is always the case that the Wasm instance is already initialized. The only possibility for + // it to not be the case is if the user completely invented the `chainId`. + if (!state.initialized) + throw new Error("Internal error"); + + console.assert(chains.has(chainId)); + const databaseContentPromises = chains.get(chainId)?.databasePromises!; + const promise: Promise = new Promise((resolve, reject) => { + databaseContentPromises.push({ resolve, reject }); + }); + + // Cap `maxUtf8BytesSize` and set a default value. + const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. + const maxSize = maxUtf8BytesSize || (twoPower32 - 1); + const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; + + // The value of `maxUtf8BytesSize` is guaranteed to always fit in 32 bits, in + // other words, that `maxUtf8BytesSize < (1 << 32)`. + // We need to perform a conversion in such a way that the the bits of the output of + // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. + // See ToInt32 here: https://tc39.es/ecma262/#sec-toint32 + // Note that the code below has been tested against example values. Please be very careful + // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI + // layer between JS and Rust, writing unit tests would be extremely complicated. + const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. + const converted = (cappedMaxSize >= twoPower31) ? + (cappedMaxSize - twoPower32) : cappedMaxSize; + + try { + state.instance.exports.database_content(chainId, converted); + return promise; + } catch (_error) { console.assert(crashError.error); throw crashError.error } - }) - }, - - removeChain: (chainId: number) => { - // Because `removeChain` is passed as parameter an identifier returned by `addChain`, it is - // always the case that the Wasm instance is already initialized. The only possibility for - // it to not be the case is if the user completely invented the `chainId`. - if (!state.initialized) - throw new Error("Internal error"); - - // Removing the chain synchronously avoids having to deal with race conditions such as a - // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. - // These kind of race conditions are already delt with within smoldot. - console.assert(chains.has(chainId)); - chains.delete(chainId); - try { - state.instance.exports.remove_chain(chainId); - } catch(_error) { - console.assert(crashError.error); - throw crashError.error - } - }, - - databaseContent: (chainId: number, maxUtf8BytesSize?: number): Promise => { - // Because `databaseContent` is passed as parameter an identifier returned by `addChain`, it - // is always the case that the Wasm instance is already initialized. The only possibility for - // it to not be the case is if the user completely invented the `chainId`. - if (!state.initialized) - throw new Error("Internal error"); - - console.assert(chains.has(chainId)); - const databaseContentPromises = chains.get(chainId)?.databasePromises!; - const promise: Promise = new Promise((resolve, reject) => { - databaseContentPromises.push({ resolve, reject }); - }); - - // Cap `maxUtf8BytesSize` and set a default value. - const twoPower32 = (1 << 30) * 4; // `1 << 31` and `1 << 32` in JavaScript don't give the value that you expect. - const maxSize = maxUtf8BytesSize || (twoPower32 - 1); - const cappedMaxSize = (maxSize >= twoPower32) ? (twoPower32 - 1) : maxSize; - - // The value of `maxUtf8BytesSize` is guaranteed to always fit in 32 bits, in - // other words, that `maxUtf8BytesSize < (1 << 32)`. - // We need to perform a conversion in such a way that the the bits of the output of - // `ToInt32(converted)`, when interpreted as u32, is equal to `maxUtf8BytesSize`. - // See ToInt32 here: https://tc39.es/ecma262/#sec-toint32 - // Note that the code below has been tested against example values. Please be very careful - // if you decide to touch it. Ideally it would be unit-tested, but since it concerns the FFI - // layer between JS and Rust, writing unit tests would be extremely complicated. - const twoPower31 = (1 << 30) * 2; // `1 << 31` in JavaScript doesn't give the value that you expect. - const converted = (cappedMaxSize >= twoPower31) ? - (cappedMaxSize - twoPower32) : cappedMaxSize; - - try { - state.instance.exports.database_content(chainId, converted); - return promise; - } catch(_error) { - console.assert(crashError.error); - throw crashError.error } } -} } From f1ed3e31ac7bdb16ce9eb394fffe0c7398925644 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:42:57 +0200 Subject: [PATCH 30/39] Remove obsolete code --- bin/wasm-node/javascript/src/client.ts | 31 -------------------------- 1 file changed, 31 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 69720b3882..6fbf0ad53a 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -385,37 +385,6 @@ export function start(options?: ClientOptions): Client { forbidWss: options.forbidWss || false, }); - // TODO: restore - /*workerOnError(instance, (error) => { - // An instance error should only happen in case of a critical error as the result of a bug - // somewhere. Consequently, nothing is really in place to cleanly report the error. - const errorToString = error.toString(); - // TODO: restore after having updated `workerCurrentTask` - console.error( - "Smoldot has panicked" + - (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + - ". This is a bug in smoldot. Please open an issue at " + - "https://github.com/paritytech/smoldot/issues with the following message:\n" + - errorToString - ); - workerError = new CrashError(errorToString); - - // Reject all promises returned by `addChain`. - for (var pending of pendingConfirmations) { - if (pending.ty == 'chainAdded') - pending.reject(workerError); - } - pendingConfirmations = []; - - // Reject all promises for database contents. - for (const chain of chains) { - for (const promise of chain[1].databasePromises) { - promise.reject(workerError) - } - } - chains.clear(); - });*/ - return { addChain: async (options: AddChainOptions): Promise => { if (instanceError) From 8126cf62f01bd4602e12aaed0dcf18be3514b88f Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:51:15 +0200 Subject: [PATCH 31/39] Implement clean shutdown of the instance --- bin/wasm-node/javascript/src/client.ts | 5 ++--- .../javascript/src/instance/bindings.ts | 1 + .../javascript/src/instance/instance.ts | 20 +++++++++++++++++++ bin/wasm-node/rust/src/bindings.rs | 9 +++++++++ bin/wasm-node/rust/src/lib.rs | 5 +++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 6fbf0ad53a..d114be8cde 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -453,10 +453,9 @@ export function start(options?: ClientOptions): Client { }, terminate: async () => { if (instanceError) - return Promise.reject(instanceError) + throw instanceError instanceError = new AlreadyDestroyedError(); - // TODO: restore - //return workerTerminate(instance) + instance.startShutdown() } } } diff --git a/bin/wasm-node/javascript/src/instance/bindings.ts b/bin/wasm-node/javascript/src/instance/bindings.ts index dfe54985e9..17b6b8435e 100644 --- a/bin/wasm-node/javascript/src/instance/bindings.ts +++ b/bin/wasm-node/javascript/src/instance/bindings.ts @@ -24,6 +24,7 @@ export interface SmoldotWasmExports extends WebAssembly.Exports { memory: WebAssembly.Memory, init: (maxLogLevel: number, enableCurrentTask: number, cpuRateLimit: number) => void, + start_shutdown: () => void, alloc: (len: number) => number, add_chain: (chainSpecPointer: number, chainSpecLen: number, databaseContentPointer: number, databaseContentLen: number, jsonRpcRunning: number, potentialRelayChainsPtr: number, potentialRelayChainsLen: number) => number; remove_chain: (chainId: number) => void, diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index 49fec760ed..66ff117cfa 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -39,6 +39,7 @@ export interface Instance { addChain: (chainSpec: string, databaseContent: string, potentialRelayChains: number[], jsonRpcCallback?: (response: string) => void) => Promise<{ success: true, chainId: number } | { success: false, error: string }> removeChain: (chainId: number) => void databaseContent: (chainId: number, maxUtf8BytesSize?: number) => Promise + startShutdown: () => void } export function start(configMessage: Config): Instance { @@ -54,6 +55,8 @@ export function start(configMessage: Config): Instance { const workerCurrentTask: { name: string | null } = { name: null }; + const printError = { printError: true } + // Contains the information of each chain that is currently alive. let chains: Map void, @@ -65,6 +68,8 @@ export function start(configMessage: Config): Instance { onWasmPanic: (message) => { // TODO: consider obtaining a backtrace here crashError.error = new CrashError(message); + if (!printError.printError) + return; console.error( "Smoldot has panicked" + (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + @@ -265,6 +270,21 @@ export function start(configMessage: Config): Instance { console.assert(crashError.error); throw crashError.error } + }, + + startShutdown: () => { + return queueOperation((instance) => { + if (crashError.error) + throw crashError.error; + + try { + instance.exports.start_shutdown() + printError.printError = false + } catch (_error) { + console.assert(crashError.error); + throw crashError.error + } + }) } } diff --git a/bin/wasm-node/rust/src/bindings.rs b/bin/wasm-node/rust/src/bindings.rs index d40a50a5d8..5080df387a 100644 --- a/bin/wasm-node/rust/src/bindings.rs +++ b/bin/wasm-node/rust/src/bindings.rs @@ -280,6 +280,15 @@ pub extern "C" fn init(max_log_level: u32, enable_current_task: u32, cpu_rate_li crate::init(max_log_level, enable_current_task, cpu_rate_limit) } +/// Instructs the client to start shutting down. +/// +/// Later, the client will use `exit` to stop. +// TODO: can this be called multiple times? +#[no_mangle] +pub extern "C" fn start_shutdown() { + crate::start_shutdown() +} + /// Allocates a buffer of the given length, with an alignment of 1. /// /// This must be used in the context of [`add_chain`] and other functions that similarly require diff --git a/bin/wasm-node/rust/src/lib.rs b/bin/wasm-node/rust/src/lib.rs index 0d513b1e66..2ded7e32c1 100644 --- a/bin/wasm-node/rust/src/lib.rs +++ b/bin/wasm-node/rust/src/lib.rs @@ -125,6 +125,11 @@ fn init(max_log_level: u32, enable_current_task: u32, cpu_rate_limit: u32) { *client_lock = Some(init_out); } +fn start_shutdown() { + // TODO: do this in a clean way + std::process::exit(0) +} + fn add_chain( chain_spec_pointer: u32, chain_spec_len: u32, From 15af6cde4557630293eac9e5f48b6ea0366b1f9e Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:52:10 +0200 Subject: [PATCH 32/39] Fix removeChain and databaseContent still callable --- bin/wasm-node/javascript/src/instance/instance.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index 66ff117cfa..0915a22696 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -219,6 +219,8 @@ export function start(configMessage: Config): Instance { // it to not be the case is if the user completely invented the `chainId`. if (!state.initialized) throw new Error("Internal error"); + if (crashError.error) + throw crashError.error; // Removing the chain synchronously avoids having to deal with race conditions such as a // JSON-RPC response corresponding to a chain that is going to be deleted but hasn't been yet. @@ -240,6 +242,9 @@ export function start(configMessage: Config): Instance { if (!state.initialized) throw new Error("Internal error"); + if (crashError.error) + throw crashError.error; + console.assert(chains.has(chainId)); const databaseContentPromises = chains.get(chainId)?.databasePromises!; const promise: Promise = new Promise((resolve, reject) => { From ebe5764fe11d2ea349f9c3f0b48bd44878b2fb01 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:54:08 +0200 Subject: [PATCH 33/39] Document start_shutdown --- bin/wasm-node/rust/src/bindings.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/wasm-node/rust/src/bindings.rs b/bin/wasm-node/rust/src/bindings.rs index 5080df387a..f72b3368fc 100644 --- a/bin/wasm-node/rust/src/bindings.rs +++ b/bin/wasm-node/rust/src/bindings.rs @@ -283,6 +283,10 @@ pub extern "C" fn init(max_log_level: u32, enable_current_task: u32, cpu_rate_li /// Instructs the client to start shutting down. /// /// Later, the client will use `exit` to stop. +/// +/// It is still legal to call all the other functions of these bindings. The client continues to +/// operate normally until the call to `exit`, which happens at some undeterminate point in the +/// future. // TODO: can this be called multiple times? #[no_mangle] pub extern "C" fn start_shutdown() { From 9532b5f72aa73cbd55121fb4b58862876a0beef0 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 15:56:37 +0200 Subject: [PATCH 34/39] Clarify alreadyDestroyedError --- bin/wasm-node/javascript/src/client.ts | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index d114be8cde..6a0283cfcc 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -368,7 +368,10 @@ export function start(options?: ClientOptions): Client { // Immediately cleared when `remove()` is called on a chain. let chainIds: WeakMap = new WeakMap(); - let instanceError: null | Error = null; + // If `Client.terminate()̀ is called, this error is set to a value. + // All the functions of the public API check if this contains a value. + let alreadyDestroyedError: null | AlreadyDestroyedError = null; + const instance = startWorker({ // Maximum level of log entries sent by the client. // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace @@ -387,8 +390,8 @@ export function start(options?: ClientOptions): Client { return { addChain: async (options: AddChainOptions): Promise => { - if (instanceError) - throw instanceError; + if (alreadyDestroyedError) + throw alreadyDestroyedError; // Passing a JSON object for the chain spec is an easy mistake, so we provide a more // readable error. @@ -419,8 +422,8 @@ export function start(options?: ClientOptions): Client { // Resolve the promise that `addChain` returned to the user. const newChain: Chain = { sendJsonRpc: (request) => { - if (instanceError) - throw instanceError; + if (alreadyDestroyedError) + throw alreadyDestroyedError; if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); if (!options.jsonRpcCallback) @@ -430,15 +433,15 @@ export function start(options?: ClientOptions): Client { instance.request(request, chainId); }, databaseContent: (maxUtf8BytesSize) => { - if (instanceError) - return Promise.reject(instanceError); + if (alreadyDestroyedError) + return Promise.reject(alreadyDestroyedError); if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); return instance.databaseContent(chainId, maxUtf8BytesSize); }, remove: () => { - if (instanceError) - throw instanceError; + if (alreadyDestroyedError) + throw alreadyDestroyedError; if (wasDestroyed.destroyed) throw new AlreadyDestroyedError(); wasDestroyed.destroyed = true; @@ -452,9 +455,9 @@ export function start(options?: ClientOptions): Client { return newChain; }, terminate: async () => { - if (instanceError) - throw instanceError - instanceError = new AlreadyDestroyedError(); + if (alreadyDestroyedError) + throw alreadyDestroyedError + alreadyDestroyedError = new AlreadyDestroyedError(); instance.startShutdown() } } From aa3ca9491ccc28277b6505c4a3c7f1d3b8e55760 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 16:03:14 +0200 Subject: [PATCH 35/39] CHANGELOG --- bin/wasm-node/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/wasm-node/CHANGELOG.md b/bin/wasm-node/CHANGELOG.md index da8f195b4f..015645f2ca 100644 --- a/bin/wasm-node/CHANGELOG.md +++ b/bin/wasm-node/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Changed + +- No WebWorker/worker thread is spawned anymore by the JavaScript code. The WebAssembly virtual machine that runs smoldot is now directly instantiated by the `start` function. This should fix compatibility issues with various JavaScript bundlers. ([#2498](https://github.com/paritytech/smoldot/pull/2498)) + ## 0.6.23 - 2022-07-11 ### Fixed From 13bc1662ae8e5d68221576b98ae46a254d146029 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 16:53:49 +0200 Subject: [PATCH 36/39] Fix startShutdown handling of errors --- bin/wasm-node/javascript/src/instance/instance.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index 0915a22696..4b3dfc88ee 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -279,15 +279,19 @@ export function start(configMessage: Config): Instance { startShutdown: () => { return queueOperation((instance) => { + // `startShutdown` is a bit special in its handling of crashes. + // Shutting down will lead to `onWasmPanic` being called at some point, possibly during + // the call to `start_shutdown` itself. As such, we move into "don't print errors anymore" + // mode even before calling `start_shutdown`. + // + // Furthermore, if a crash happened in the past, there is no point in throwing an + // exception when the user wants the shutdown to happen. if (crashError.error) - throw crashError.error; - + return; try { - instance.exports.start_shutdown() printError.printError = false + instance.exports.start_shutdown() } catch (_error) { - console.assert(crashError.error); - throw crashError.error } }) } From 9dce352c51bba34389ab35766403cb922d1d74c2 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 16:58:26 +0200 Subject: [PATCH 37/39] Spellcheck --- bin/wasm-node/rust/src/bindings.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/wasm-node/rust/src/bindings.rs b/bin/wasm-node/rust/src/bindings.rs index f72b3368fc..5be2385223 100644 --- a/bin/wasm-node/rust/src/bindings.rs +++ b/bin/wasm-node/rust/src/bindings.rs @@ -285,8 +285,7 @@ pub extern "C" fn init(max_log_level: u32, enable_current_task: u32, cpu_rate_li /// Later, the client will use `exit` to stop. /// /// It is still legal to call all the other functions of these bindings. The client continues to -/// operate normally until the call to `exit`, which happens at some undeterminate point in the -/// future. +/// operate normally until the call to `exit`, which happens at some point in the future. // TODO: can this be called multiple times? #[no_mangle] pub extern "C" fn start_shutdown() { From 4e224355fee4a2bf3ba4ac8978959c5dd627b19e Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 17:48:09 +0200 Subject: [PATCH 38/39] Rename --- bin/wasm-node/javascript/src/client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/wasm-node/javascript/src/client.ts b/bin/wasm-node/javascript/src/client.ts index 6a0283cfcc..fdfa774c3d 100644 --- a/bin/wasm-node/javascript/src/client.ts +++ b/bin/wasm-node/javascript/src/client.ts @@ -15,7 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -import { start as startWorker } from './instance/instance.js'; +import { start as startInstance } from './instance/instance.js'; /** * Thrown in case of a problem when initializing the chain. @@ -372,7 +372,7 @@ export function start(options?: ClientOptions): Client { // All the functions of the public API check if this contains a value. let alreadyDestroyedError: null | AlreadyDestroyedError = null; - const instance = startWorker({ + const instance = startInstance({ // Maximum level of log entries sent by the client. // 0 = Logging disabled, 1 = Error, 2 = Warn, 3 = Info, 4 = Debug, 5 = Trace maxLogLevel: options.maxLogLevel || 3, From fd00321ed90339335ad5700777f6e49a4fe4bf5c Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 12 Jul 2022 17:48:33 +0200 Subject: [PATCH 39/39] More rename --- bin/wasm-node/javascript/src/instance/instance.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/wasm-node/javascript/src/instance/instance.ts b/bin/wasm-node/javascript/src/instance/instance.ts index 4b3dfc88ee..b3d5d498ec 100644 --- a/bin/wasm-node/javascript/src/instance/instance.ts +++ b/bin/wasm-node/javascript/src/instance/instance.ts @@ -53,7 +53,7 @@ export function start(configMessage: Config): Instance { const crashError: { error?: CrashError } = {}; - const workerCurrentTask: { name: string | null } = { name: null }; + const currentTask: { name: string | null } = { name: null }; const printError = { printError: true } @@ -72,7 +72,7 @@ export function start(configMessage: Config): Instance { return; console.error( "Smoldot has panicked" + - (workerCurrentTask.name ? (" while executing task `" + workerCurrentTask.name + "`") : "") + + (currentTask.name ? (" while executing task `" + currentTask.name + "`") : "") + ". This is a bug in smoldot. Please open an issue at " + "https://github.com/paritytech/smoldot/issues with the following message:\n" + message @@ -90,7 +90,7 @@ export function start(configMessage: Config): Instance { (promises.shift() as DatabasePromise).resolve(data); }, currentTaskCallback: (taskName) => { - workerCurrentTask.name = taskName + currentTask.name = taskName }, cpuRateLimit: configMessage.cpuRateLimit, forbidTcp: configMessage.forbidTcp,