From 078d09b4bee1be88c990bc8ee38644e27f3a0f16 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 18 Jul 2022 12:17:45 +0200 Subject: [PATCH 1/7] Add support for Deno with an index-deno.ts --- bin/wasm-node/CHANGELOG.md | 4 + bin/wasm-node/javascript/demo/demo-deno.ts | 87 +++++ bin/wasm-node/javascript/package.json | 2 +- bin/wasm-node/javascript/src/index-deno.ts | 408 +++++++++++++++++++++ 4 files changed, 500 insertions(+), 1 deletion(-) create mode 100644 bin/wasm-node/javascript/demo/demo-deno.ts create mode 100644 bin/wasm-node/javascript/src/index-deno.ts diff --git a/bin/wasm-node/CHANGELOG.md b/bin/wasm-node/CHANGELOG.md index 0b9f4973c7..996de1b3cd 100644 --- a/bin/wasm-node/CHANGELOG.md +++ b/bin/wasm-node/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Added + +- Add support for Deno. Smoldot is now available on the deno.land/x package registry. This doesn't modify anything to the behaviour of the smoldot NPM package. + ## 0.6.25 - 2022-07-18 ### Added diff --git a/bin/wasm-node/javascript/demo/demo-deno.ts b/bin/wasm-node/javascript/demo/demo-deno.ts new file mode 100644 index 0000000000..f46f1c7c68 --- /dev/null +++ b/bin/wasm-node/javascript/demo/demo-deno.ts @@ -0,0 +1,87 @@ +// Smoldot +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// This file launches a WebSocket server that exposes JSON-RPC functions. + +import * as smoldot from '../dist/mjs/index-deno.js'; + +// Load the chain spec file. +const chainSpec = new TextDecoder("utf-8").decode(await Deno.readFile("../../westend.json")); + +const client = smoldot.start({ + maxLogLevel: 3, // Can be increased for more verbosity + forbidTcp: false, + forbidWs: false, + forbidNonLocalWs: false, + forbidWss: false, + cpuRateLimit: 0.5, + logCallback: (_level, target, message) => { + // As incredible as it seems, there is currently no better way to print the current time + // formatted in a certain way. + const now = new Date(); + const hours = ("0" + now.getHours()).slice(-2); + const minutes = ("0" + now.getMinutes()).slice(-2); + const seconds = ("0" + now.getSeconds()).slice(-2); + const milliseconds = ("00" + now.getMilliseconds()).slice(-3); + console.log( + "[%s:%s:%s.%s] [%s] %s", + hours, minutes, seconds, milliseconds, target, message + ); + } +}); + +// We add the chain ahead of time in order to preload it. +// Once a client connects, the chain is added again, but smoldot is smart enough to not connect +// a second time. +client.addChain({ chainSpec }); + +// Now spawn a WebSocket server in order to handle JSON-RPC clients. +console.log('JSON-RPC server now listening on port 9944'); +console.log('Please visit: https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944'); + +const conn = Deno.listen({ port: 9944 }); +const httpConn = Deno.serveHttp(await conn.accept()); + +while(true) { + const event = await httpConn.nextRequest(); + if (!event) + continue; + + console.log('(demo) New JSON-RPC client connected.'); + + const { socket, response } = Deno.upgradeWebSocket(event.request); + + const chain = await client.addChain({ + chainSpec, + jsonRpcCallback: (response) => socket.send(response) + }); + + socket.onclose = () => { + console.log("(demo) JSON-RPC client disconnected."); + chain.remove(); + }; + + socket.onmessage = (event: Deno.MessageEvent) => { + if (typeof event.data === 'string') { + chain.sendJsonRpc(event.data); + } else { + socket.close(1002); // Protocol error + } + }; + + event.respondWith(response); +} diff --git a/bin/wasm-node/javascript/package.json b/bin/wasm-node/javascript/package.json index 4b3a934e8a..48ac685e37 100644 --- a/bin/wasm-node/javascript/package.json +++ b/bin/wasm-node/javascript/package.json @@ -31,7 +31,7 @@ "prepublishOnly": "node prepare.mjs --release && rimraf ./dist && npm run buildModules", "build": "node prepare.mjs --release && rimraf ./dist && npm run buildModules", "start": "node prepare.mjs --debug && rimraf ./dist && npm run buildModules && node demo/demo.mjs", - "test": "node prepare.mjs --debug && rimraf ./dist && npm run buildModules && ava --timeout=2m --concurrency 2 --no-worker-threads" + "test": "node prepare.mjs --debug && rimraf ./dist && npm run buildModules && deno run ./dist/mjs/index-deno.js && ava --timeout=2m --concurrency 2 --no-worker-threads" }, "dependencies": { "websocket": "^1.0.32" diff --git a/bin/wasm-node/javascript/src/index-deno.ts b/bin/wasm-node/javascript/src/index-deno.ts new file mode 100644 index 0000000000..f53169a47d --- /dev/null +++ b/bin/wasm-node/javascript/src/index-deno.ts @@ -0,0 +1,408 @@ +// Smoldot +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import { Client, ClientOptions, start as innerStart } from './client.js' +import { Connection, ConnectionError, ConnectionConfig } from './instance/instance.js'; + +export { + AddChainError, + AddChainOptions, + AlreadyDestroyedError, + Chain, + Client, + ClientOptions, + CrashError, + JsonRpcCallback, + JsonRpcDisabledError, + LogCallback +} from './client.js'; + +/** + * Initializes a new client. This is a pre-requisite to connecting to a blockchain. + * + * Can never fail. + * + * @param options Configuration of the client. Defaults to `{}`. + */ +export function start(options?: ClientOptions): Client { + options = options || {}; + + return innerStart(options || {}, { + performanceNow: () => { + return performance.now() + }, + getRandomValues: (buffer) => { + const crypto = globalThis.crypto; + if (!crypto) + throw new Error('randomness not available'); + crypto.getRandomValues(buffer); + }, + connect: (config) => { + return connect(config, options?.forbidTcp || false, options?.forbidWs || false, options?.forbidNonLocalWs || false, options?.forbidWss || false) + } + }) +} + +/** + * Tries to open a new connection using the given configuration. + * + * @see Connection + * @throws ConnectionError If the multiaddress couldn't be parsed or contains an invalid protocol. + */ +function connect(config: ConnectionConfig, forbidTcp: boolean, forbidWs: boolean, forbidNonLocalWs: boolean, forbidWss: boolean): Connection { + let connection: TcpWrapped | WebSocketWrapped; + + // Attempt to parse the multiaddress. + // TODO: remove support for `/wss` in a long time (https://github.com/paritytech/smoldot/issues/1940) + const wsParsed = config.address.match(/^\/(ip4|ip6|dns4|dns6|dns)\/(.*?)\/tcp\/(.*?)\/(ws|wss|tls\/ws)$/); + const tcpParsed = config.address.match(/^\/(ip4|ip6|dns4|dns6|dns)\/(.*?)\/tcp\/(.*?)$/); + + if (wsParsed != null) { + const proto = (wsParsed[4] == 'ws') ? 'ws' : 'wss'; + if ( + (proto == 'ws' && forbidWs) || + (proto == 'ws' && wsParsed[2] != 'localhost' && wsParsed[2] != '127.0.0.1' && forbidNonLocalWs) || + (proto == 'wss' && forbidWss) + ) { + throw new ConnectionError('Connection type not allowed'); + } + + const url = (wsParsed[1] == 'ip6') ? + (proto + "://[" + wsParsed[2] + "]:" + wsParsed[3]) : + (proto + "://" + wsParsed[2] + ":" + wsParsed[3]); + + connection = { + ty: 'websocket', + socket: new WebSocket(url) + }; + connection.socket.binaryType = 'arraybuffer'; + + connection.socket.onopen = () => { + config.onOpen(); + }; + connection.socket.onclose = (event) => { + const message = "Error code " + event.code + (!!event.reason ? (": " + event.reason) : ""); + config.onClose(message); + }; + connection.socket.onmessage = (msg) => { + config.onMessage(new Uint8Array(msg.data as ArrayBuffer)); + }; + + } else if (tcpParsed != null) { + // `net` module will be missing when we're not in NodeJS. + if (forbidTcp) { + throw new ConnectionError('TCP connections not available'); + } + + const socket = { + destroyed: false, + inner: Deno.connect({ + hostname: tcpParsed[2], + port: parseInt(tcpParsed[3]!, 10), + }) + }; + + connection = { ty: 'tcp', socket }; + + socket.inner = socket.inner.then((established) => { + // TODO: at the time of writing of this comment, `setNoDelay` is still unstable + //established.setNoDelay(); + + if (socket.destroyed) + return established; + config.onOpen(); + + // Spawns an asynchronous task that continuously reads from the socket. + // Every time data is read, the task re-executes itself in order to continue reading. + // The task ends automatically if an EOF or error is detected, which should also happen + // if the user calls `close()`. + const read = async (readBuffer: Uint8Array): Promise => { + if (socket.destroyed) + return; + let outcome: null | number | string = null; + try { + outcome = await established.read(readBuffer); + } catch(error) { + // The type of `error` is unclear, but we assume that it implements `Error` + outcome = (error as Error).toString() + } + if (socket.destroyed) + return; + if (typeof outcome !== 'number' || outcome === null) { + // The socket is reported closed, but `socket.destroyed` is still `false` (see + // check above). As such, we must inform the inner layers. + socket.destroyed = true; + config.onClose(outcome === null ? "EOF when reading socket" : outcome); + return; + } + console.assert(outcome !== 0); // `read` guarantees to return a non-zero value. + config.onMessage(readBuffer.slice(0, outcome)); + return read(readBuffer) + } + ;read(new Uint8Array(1024)); + + return established; + }); + + } else { + throw new ConnectionError('Unrecognized multiaddr format'); + } + + return { + close: (): void => { + if (connection.ty == 'websocket') { + // WebSocket + // We can't set these fields to null because the TypeScript definitions don't + // allow it, but we can set them to dummy values. + connection.socket.onopen = () => { }; + connection.socket.onclose = () => { }; + connection.socket.onmessage = () => { }; + connection.socket.onerror = () => { }; + connection.socket.close(); + } else { + // TCP + connection.socket.destroyed = true; + connection.socket.inner.then((connec) => connec.close()); + } + }, + + send: (data: Uint8Array): void => { + if (connection.ty == 'websocket') { + // WebSocket + connection.socket.send(data); + } else { + // TCP + // TODO: at the moment, sending data doesn't have any back-pressure mechanism; as such, we just buffer data indefinitely + let dataCopy = Uint8Array.from(data) // Deep copy of the data + const socket = connection.socket; + connection.socket.inner = connection.socket.inner.then(async (c) => { + while (dataCopy.length > 0) { + if (socket.destroyed) + return c; + let outcome: number | string; + try { + outcome = await c.write(dataCopy); + } catch(error) { + // The type of `error` is unclear, but we assume that it implements `Error` + outcome = (error as Error).toString() + } + if (typeof outcome !== 'number') { + // The socket is reported closed, but `socket.destroyed` is still + // `false` (see check above). As such, we must inform the inner layers. + socket.destroyed = true; + config.onClose(outcome); + return c; + } + // Note that, contrary to `read`, it is possible for `outcome` to be 0. + // This happen if the write had to be interrupted, and the only thing + // we have to do is try writing again. + dataCopy = dataCopy.slice(outcome); + } + return c; + }); + } + } + }; +} + +interface TcpWrapped { + ty: 'tcp', + socket: TcpConnection, +} + +interface WebSocketWrapped { + ty: 'websocket', + socket: WebSocket, +} + +interface TcpConnection { + // `Promise` that resolves when the connection is ready to accept more data to send, or when + // the connection is closed. Check `destroyed` in order to know whether the connection + // is closed. + inner: Promise, + destroyed: boolean, +} + + + +// Deno type definitions copy-pasted below, filtered to keep only what is necessary. +// The code below is under MIT license. + +/* +MIT License + +Copyright 2018-2022 the Deno authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +// Original can be found here: https://github.com/denoland/deno/blob/main/cli/dts/lib.deno.ns.d.ts +declare namespace Deno { + export interface Reader { + /** Reads up to `p.byteLength` bytes into `p`. It resolves to the number of + * bytes read (`0` < `n` <= `p.byteLength`) and rejects if any error + * encountered. Even if `read()` resolves to `n` < `p.byteLength`, it may + * use all of `p` as scratch space during the call. If some data is + * available but not `p.byteLength` bytes, `read()` conventionally resolves + * to what is available instead of waiting for more. + * + * When `read()` encounters end-of-file condition, it resolves to EOF + * (`null`). + * + * When `read()` encounters an error, it rejects with an error. + * + * Callers should always process the `n` > `0` bytes returned before + * considering the EOF (`null`). Doing so correctly handles I/O errors that + * happen after reading some bytes and also both of the allowed EOF + * behaviors. + * + * Implementations should not retain a reference to `p`. + * + * Use `itereateReader` from from https://deno.land/std/streams/conversion.ts to + * turn a Reader into an AsyncIterator. + */ + read(p: Uint8Array): Promise; + } + + export interface ReaderSync { + /** Reads up to `p.byteLength` bytes into `p`. It resolves to the number + * of bytes read (`0` < `n` <= `p.byteLength`) and rejects if any error + * encountered. Even if `readSync()` returns `n` < `p.byteLength`, it may use + * all of `p` as scratch space during the call. If some data is available + * but not `p.byteLength` bytes, `readSync()` conventionally returns what is + * available instead of waiting for more. + * + * When `readSync()` encounters end-of-file condition, it returns EOF + * (`null`). + * + * When `readSync()` encounters an error, it throws with an error. + * + * Callers should always process the `n` > `0` bytes returned before + * considering the EOF (`null`). Doing so correctly handles I/O errors that happen + * after reading some bytes and also both of the allowed EOF behaviors. + * + * Implementations should not retain a reference to `p`. + * + * Use `iterateReaderSync()` from from https://deno.land/std/streams/conversion.ts + * to turn a ReaderSync into an Iterator. + */ + readSync(p: Uint8Array): number | null; + } + + export interface Writer { + /** Writes `p.byteLength` bytes from `p` to the underlying data stream. It + * resolves to the number of bytes written from `p` (`0` <= `n` <= + * `p.byteLength`) or reject with the error encountered that caused the + * write to stop early. `write()` must reject with a non-null error if + * would resolve to `n` < `p.byteLength`. `write()` must not modify the + * slice data, even temporarily. + * + * Implementations should not retain a reference to `p`. + */ + write(p: Uint8Array): Promise; + } + + export interface WriterSync { + /** Writes `p.byteLength` bytes from `p` to the underlying data + * stream. It returns the number of bytes written from `p` (`0` <= `n` + * <= `p.byteLength`) and any error encountered that caused the write to + * stop early. `writeSync()` must throw a non-null error if it returns `n` < + * `p.byteLength`. `writeSync()` must not modify the slice data, even + * temporarily. + * + * Implementations should not retain a reference to `p`. + */ + writeSync(p: Uint8Array): number; + } + + export interface Closer { + close(): void; + } +} + +// Original can be found here: https://github.com/denoland/deno/blob/main/ext/net/lib.deno_net.d.ts +declare namespace Deno { + export interface NetAddr { + transport: "tcp" | "udp"; + hostname: string; + port: number; + } + + export interface UnixAddr { + transport: "unix" | "unixpacket"; + path: string; + } + + export type Addr = NetAddr | UnixAddr; + + export interface Conn extends Reader, Writer, Closer { + /** The local address of the connection. */ + readonly localAddr: Addr; + /** The remote address of the connection. */ + readonly remoteAddr: Addr; + /** The resource ID of the connection. */ + readonly rid: number; + /** Shuts down (`shutdown(2)`) the write side of the connection. Most + * callers should just use `close()`. */ + closeWrite(): Promise; + + readonly readable: ReadableStream; + readonly writable: WritableStream; + } + + export interface ConnectOptions { + /** The port to connect to. */ + port: number; + /** A literal IP address or host name that can be resolved to an IP address. + * If not specified, defaults to `127.0.0.1`. */ + hostname?: string; + transport?: "tcp"; + } + + /** + * Connects to the hostname (default is "127.0.0.1") and port on the named + * transport (default is "tcp"), and resolves to the connection (`Conn`). + * + * ```ts + * const conn1 = await Deno.connect({ port: 80 }); + * const conn2 = await Deno.connect({ hostname: "192.0.2.1", port: 80 }); + * const conn3 = await Deno.connect({ hostname: "[2001:db8::1]", port: 80 }); + * const conn4 = await Deno.connect({ hostname: "golang.org", port: 80, transport: "tcp" }); + * ``` + * + * Requires `allow-net` permission for "tcp". */ + export function connect(options: ConnectOptions): Promise; + + export interface TcpConn extends Conn { + /** + * **UNSTABLE**: new API, see https://github.com/denoland/deno/issues/13617. + * + * Enable/disable the use of Nagle's algorithm. Defaults to true. + */ + setNoDelay(nodelay?: boolean): void; + /** + * **UNSTABLE**: new API, see https://github.com/denoland/deno/issues/13617. + * + * Enable/disable keep-alive functionality. + */ + setKeepAlive(keepalive?: boolean): void; + } +} From 8b971a5daed4c02f09c7a3e4ebec05caff0ead84 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 18 Jul 2022 12:27:27 +0200 Subject: [PATCH 2/7] PR number --- bin/wasm-node/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/wasm-node/CHANGELOG.md b/bin/wasm-node/CHANGELOG.md index 996de1b3cd..55ec9b8b9d 100644 --- a/bin/wasm-node/CHANGELOG.md +++ b/bin/wasm-node/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- Add support for Deno. Smoldot is now available on the deno.land/x package registry. This doesn't modify anything to the behaviour of the smoldot NPM package. +- Add support for Deno. Smoldot is now available on the deno.land/x package registry. This doesn't modify anything to the behaviour of the smoldot NPM package. ([#2522](https://github.com/paritytech/smoldot/pull/2522)) ## 0.6.25 - 2022-07-18 From c20a8cbc83fbfc9df821168d5b5f8c70cb5ee880 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 18 Jul 2022 12:28:08 +0200 Subject: [PATCH 3/7] Install Deno in the CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 245a6058ae..6e81811c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,9 @@ jobs: - uses: actions/setup-node@v3.4.1 with: node-version: '14' + - uses: denoland/setup-deno@v1 + with: + deno-version: v1.x - run: cd bin/wasm-node/javascript && RUSTFLAGS=-Dwarnings npm install-ci-test wasm-node-size-diff: From 6c29bcd257a690133c3ecadfd74f388440adad63 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 19 Jul 2022 15:26:54 +0200 Subject: [PATCH 4/7] Add `zlibInflate` after merge --- bin/wasm-node/javascript/src/index-browser.ts | 2 +- bin/wasm-node/javascript/src/index-deno.ts | 39 +++++++++++++++++-- bin/wasm-node/javascript/src/index-nodejs.ts | 2 +- .../javascript/src/instance/raw-instance.ts | 7 +++- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/bin/wasm-node/javascript/src/index-browser.ts b/bin/wasm-node/javascript/src/index-browser.ts index 4a62e50a54..4b28613c52 100644 --- a/bin/wasm-node/javascript/src/index-browser.ts +++ b/bin/wasm-node/javascript/src/index-browser.ts @@ -46,7 +46,7 @@ export function start(options?: ClientOptions): Client { return innerStart(options, { zlibInflate: (buffer) => { - return pako.inflate(buffer) + return Promise.resolve(pako.inflate(buffer)) }, performanceNow: () => { return performance.now() diff --git a/bin/wasm-node/javascript/src/index-deno.ts b/bin/wasm-node/javascript/src/index-deno.ts index f53169a47d..fd2ccf45ac 100644 --- a/bin/wasm-node/javascript/src/index-deno.ts +++ b/bin/wasm-node/javascript/src/index-deno.ts @@ -42,6 +42,11 @@ export function start(options?: ClientOptions): Client { options = options || {}; return innerStart(options || {}, { + zlibInflate: async (buffer) => { + const decompressedStream = new Blob([buffer.buffer]).stream() + .pipeThrough(new DecompressionStream('deflate')); + return new Uint8Array(await new Response(decompressedStream).arrayBuffer()); + }, performanceNow: () => { return performance.now() }, @@ -136,7 +141,7 @@ function connect(config: ConnectionConfig, forbidTcp: boolean, forbidWs: boolean let outcome: null | number | string = null; try { outcome = await established.read(readBuffer); - } catch(error) { + } catch (error) { // The type of `error` is unclear, but we assume that it implements `Error` outcome = (error as Error).toString() } @@ -153,7 +158,7 @@ function connect(config: ConnectionConfig, forbidTcp: boolean, forbidWs: boolean config.onMessage(readBuffer.slice(0, outcome)); return read(readBuffer) } - ;read(new Uint8Array(1024)); + ; read(new Uint8Array(1024)); return established; }); @@ -196,7 +201,7 @@ function connect(config: ConnectionConfig, forbidTcp: boolean, forbidWs: boolean let outcome: number | string; try { outcome = await c.write(dataCopy); - } catch(error) { + } catch (error) { // The type of `error` is unclear, but we assume that it implements `Error` outcome = (error as Error).toString() } @@ -406,3 +411,31 @@ declare namespace Deno { setKeepAlive(keepalive?: boolean): void; } } + +// Original can be found here: https://github.com/denoland/deno/blob/main/ext/web/lib.deno_web.d.ts +/** + * An API for decompressing a stream of data. + * + * @example + * ```ts + * const input = await Deno.open("./file.txt.gz"); + * const output = await Deno.create("./file.txt"); + * + * await input.readable + * .pipeThrough(new DecompressionStream("gzip")) + * .pipeTo(output.writable); + * ``` + */ +declare class DecompressionStream { + /** + * Creates a new `DecompressionStream` object which decompresses a stream of + * data. + * + * Throws a `TypeError` if the format passed to the constructor is not + * supported. + */ + constructor(format: string); + + readonly readable: ReadableStream; + readonly writable: WritableStream; +} diff --git a/bin/wasm-node/javascript/src/index-nodejs.ts b/bin/wasm-node/javascript/src/index-nodejs.ts index 3ddb3ebd15..2a9ca68767 100644 --- a/bin/wasm-node/javascript/src/index-nodejs.ts +++ b/bin/wasm-node/javascript/src/index-nodejs.ts @@ -51,7 +51,7 @@ export function start(options?: ClientOptions): Client { return innerStart(options || {}, { zlibInflate: (buffer) => { - return pako.inflate(buffer) + return Promise.resolve(pako.inflate(buffer)) }, performanceNow: () => { const time = hrtime(); diff --git a/bin/wasm-node/javascript/src/instance/raw-instance.ts b/bin/wasm-node/javascript/src/instance/raw-instance.ts index 5fc5d8aded..51961fe5a8 100644 --- a/bin/wasm-node/javascript/src/instance/raw-instance.ts +++ b/bin/wasm-node/javascript/src/instance/raw-instance.ts @@ -52,8 +52,11 @@ export interface Config { export interface PlatformBindings { /** * Decompresses the given buffer using the inflate algorithm with zlib header. + * + * Note that this function is asynchronous because for whatever reason the compression streams + * Web API is asynchronous. */ - zlibInflate: (buffer: Uint8Array) => Uint8Array, + zlibInflate: (buffer: Uint8Array) => Promise, /** * Returns the number of milliseconds since an arbitrary epoch. @@ -79,7 +82,7 @@ export async function startInstance(config: Config, platformBindings: PlatformBi // different file. // This is suboptimal compared to using `instantiateStreaming`, but it is the most // cross-platform cross-bundler approach. - const wasmBytecode = platformBindings.zlibInflate(buffer.trustedBase64Decode(wasmBase64)) + const wasmBytecode = await platformBindings.zlibInflate(buffer.trustedBase64Decode(wasmBase64)) let killAll: () => void; From 4d6ea752b0fb98f3fe04b214c56c285f5c22d448 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 19 Jul 2022 16:59:10 +0200 Subject: [PATCH 5/7] Try make it work --- bin/wasm-node/javascript/src/index-deno.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/wasm-node/javascript/src/index-deno.ts b/bin/wasm-node/javascript/src/index-deno.ts index fd2ccf45ac..622d693d7f 100644 --- a/bin/wasm-node/javascript/src/index-deno.ts +++ b/bin/wasm-node/javascript/src/index-deno.ts @@ -43,8 +43,8 @@ export function start(options?: ClientOptions): Client { return innerStart(options || {}, { zlibInflate: async (buffer) => { - const decompressedStream = new Blob([buffer.buffer]).stream() - .pipeThrough(new DecompressionStream('deflate')); + const inputStream = new Blob([buffer.buffer]).stream(); + const decompressedStream = ((inputStream as any)['pipeThrough'] as (trans: TransformStream) => O)(new DecompressionStream('deflate')); return new Uint8Array(await new Response(decompressedStream).arrayBuffer()); }, performanceNow: () => { From 2418cac6467bf74c41329449422bacefd9fb1e1e Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 19 Jul 2022 17:59:11 +0200 Subject: [PATCH 6/7] Fix the inflate code --- bin/wasm-node/javascript/src/index-deno.ts | 24 +++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/bin/wasm-node/javascript/src/index-deno.ts b/bin/wasm-node/javascript/src/index-deno.ts index 622d693d7f..301c4e26a1 100644 --- a/bin/wasm-node/javascript/src/index-deno.ts +++ b/bin/wasm-node/javascript/src/index-deno.ts @@ -43,9 +43,27 @@ export function start(options?: ClientOptions): Client { return innerStart(options || {}, { zlibInflate: async (buffer) => { - const inputStream = new Blob([buffer.buffer]).stream(); - const decompressedStream = ((inputStream as any)['pipeThrough'] as (trans: TransformStream) => O)(new DecompressionStream('deflate')); - return new Uint8Array(await new Response(decompressedStream).arrayBuffer()); + const ds = new DecompressionStream('deflate'); + const writer = ds.writable.getWriter(); + writer.write(buffer); + writer.close(); + const output = []; + const reader = ds.readable.getReader(); + let totalSize = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) + break; + output.push(value); + totalSize += value.byteLength; + } + const concatenated = new Uint8Array(totalSize); + let offset = 0; + for (const array of output) { + concatenated.set(array, offset); + offset += array.byteLength; + } + return concatenated; }, performanceNow: () => { return performance.now() From 1abed8b9e378937be6047ae438e268a5ace5a9a3 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Tue, 19 Jul 2022 17:59:22 +0200 Subject: [PATCH 7/7] Comment --- bin/wasm-node/javascript/src/index-deno.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/wasm-node/javascript/src/index-deno.ts b/bin/wasm-node/javascript/src/index-deno.ts index 301c4e26a1..958622827f 100644 --- a/bin/wasm-node/javascript/src/index-deno.ts +++ b/bin/wasm-node/javascript/src/index-deno.ts @@ -43,6 +43,8 @@ export function start(options?: ClientOptions): Client { return innerStart(options || {}, { zlibInflate: async (buffer) => { + // This code has been copy-pasted from the official streams draft specification. + // At the moment, it is found here: https://wicg.github.io/compression/#example-deflate-compress const ds = new DecompressionStream('deflate'); const writer = ds.writable.getWriter(); writer.write(buffer);